Add files using upload-large-folder tool

README.md

@@ -56,6 +56,7 @@ Ring-mini-sparse-2.0-exp achieves high inference efficiency through highly spars
 Installation requirements:
 
 ```shell
+pip install flash-attn==2.6.3
 pip install transformers==4.56.1
 ```
 
@@ -71,6 +72,7 @@ model = AutoModelForCausalLM.from_pretrained(
     dtype="auto",
     device_map="auto",
     trust_remote_code=True,
+    attn_implementation="flash_attention_2",
 )
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 

configuration_bailing_moe_v2.py

@@ -1,45 +1,48 @@
-"""Bailing MoE model configuration"""
+"""Bailing MoE V2 model configuration"""
 
 from transformers.configuration_utils import PretrainedConfig
 
 
 class BailingMoeV2Config(PretrainedConfig):
-    model_type = "bailing_moe_v2"
 
     def __init__(
         self,
-        vocab_size=30592,
-        hidden_size=1024,
-        intermediate_size=None,
-        num_hidden_layers=24,
+        vocab_size=157184,
+        hidden_size=2048,
+        intermediate_size=5120,
+        num_hidden_layers=20,
         num_attention_heads=16,
-        num_key_value_heads=0,
+        num_key_value_heads=4,
         hidden_act="silu",
         use_qkv_bias=False,  # bailing only
-        use_bias=True,  # bailing only
-        rms_norm_eps=1e-05,
-        norm_head=False,  # bailing only
+        use_bias=False,  # bailing only
+        rms_norm_eps=1e-06,
         tie_word_embeddings=False,  # PretrainedConfig key, here change default value.
-        embedding_dropout=0.1,
-        attention_dropout=0.1,
-        output_dropout=0.1,
+        embedding_dropout=0.0,
+        attention_dropout=0.0,
+        output_dropout=0.0,
         initializer_range=0.02,
-        max_position_embeddings=16384,
-        rope_theta=10000.0,
+        max_position_embeddings=32768,
+        rope_theta=600000.0,
         use_cache=True,
-        use_sliding_window=False,
-        sliding_window=4096,
-        max_window_layers=28,
+        max_window_layers=20,
         rope_scaling=None,
-        pad_token_id=126081,
-        num_experts=16,
-        num_shared_experts=0,
-        num_experts_per_tok=2,
-        norm_topk_prob=True,
-        moe_intermediate_size=None,
-        first_k_dense_replace=0,
-        head_dim=None,
+        pad_token_id=156892,
+        eos_token_id=156892,
+        num_experts=256,
+        num_shared_experts=1,
+        num_experts_per_tok=8,
+        n_group=8,
+        topk_group=4,
+        moe_intermediate_size=512,
+        first_k_dense_replace=1,
+        head_dim=128,
         output_router_logits=False,
+        use_qk_norm=True,
+        num_nextn_predict_layers=0,
+        mtp_loss_scaling_factor=0,
+        moe_router_enable_expert_bias=True,
+        routed_scaling_factor=1.0,
         **kwargs,
     ):
         self.num_hidden_layers = num_hidden_layers
@@ -51,28 +54,31 @@ class BailingMoeV2Config(PretrainedConfig):
         self.hidden_act = hidden_act
         self.use_qkv_bias = use_qkv_bias
         self.use_bias = use_bias
-        self.norm_head = norm_head
         self.rms_norm_eps = rms_norm_eps
         self.embedding_dropout = embedding_dropout
         self.attention_dropout = attention_dropout
         self.output_dropout = output_dropout
+        self.num_nextn_predict_layers = num_nextn_predict_layers
+        self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
         self.initializer_range = initializer_range
         self.max_position_embeddings = max_position_embeddings
         self.rope_theta = rope_theta
         self.use_cache = use_cache
-        self.use_sliding_window = use_sliding_window
-        self.sliding_window = sliding_window
         self.max_window_layers = max_window_layers
         self.head_dim = head_dim or self.hidden_size // self.num_attention_heads
         self.rope_scaling = rope_scaling
+        self.use_qk_norm = use_qk_norm
+        self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
+        self.routed_scaling_factor = routed_scaling_factor
 
         # MoE configs
         self.num_experts = num_experts
         self.num_shared_experts = num_shared_experts
         self.num_experts_per_tok = num_experts_per_tok
-        self.norm_topk_prob = norm_topk_prob
+        self.n_group = n_group
+        self.topk_group = topk_group
         self.moe_intermediate_size = moe_intermediate_size
         self.first_k_dense_replace = first_k_dense_replace
         self.output_router_logits = output_router_logits
 
-        super().__init__(pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)
+        super().__init__(pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)

modeling_bailing_moe_v2.py

@@ -1,14 +1,5 @@
-#!/usr/bin/python
-#****************************************************************#
-# ScriptName: modeling_bailing_moe_v2.py
-# Author: $SHTERM_REAL_USER@alibaba-inc.com
-# Create Date: 2025-08-12 20:22
-# Modify Author: $SHTERM_REAL_USER@alibaba-inc.com
-# Modify Date: 2025-08-12 20:22
-# Function: 
-#***************************************************************#
 # coding=utf-8
-# Copyright 2023 Antgroup and The HuggingFace Inc. team. All rights reserved.
+# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved.
 #
 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
 # and OPT implementations in this library. It has been modified from its
@@ -27,15 +18,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """PyTorch BailingMoE model."""
+
 import math
 import warnings
 from typing import List, Optional, Tuple, Union
 
 import torch
 import torch.nn.functional as F
-import torch.utils.checkpoint
 from torch import nn
-from torch.nn import CrossEntropyLoss
 
 from transformers.activations import ACT2FN
 from transformers.cache_utils import Cache, DynamicCache
@@ -45,10 +35,8 @@ from transformers.modeling_attn_mask_utils import (
     _prepare_4d_causal_attention_mask,
     _prepare_4d_causal_attention_mask_for_sdpa,
 )
-from transformers.modeling_outputs import (
-    MoeModelOutputWithPast,
-    MoeCausalLMOutputWithPast,
-)
+from transformers.modeling_outputs import MoeModelOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
 from transformers.modeling_utils import PreTrainedModel
 from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, is_torch_greater_or_equal_than_1_13
 from transformers.utils import (
@@ -61,7 +49,11 @@ from transformers.utils import (
 )
 from transformers.utils.import_utils import is_torch_fx_available
 from .configuration_bailing_moe_v2 import BailingMoeV2Config
-
+from transformers.generation.utils import GenerationMixin
+from dataclasses import dataclass
+from transformers.utils import ModelOutput
+from einops import rearrange
+from functools import lru_cache
 
 if is_flash_attn_2_available():
     from flash_attn import flash_attn_func, flash_attn_varlen_func
@@ -82,6 +74,383 @@ logger = logging.get_logger(__name__)
 _CONFIG_FOR_DOC = "BailingMoeV2Config"
 
 
+def nonzero(x):
+    return x.nonzero(as_tuple=True)
+
+
+@lru_cache(maxsize=16)
+def calc_chunks(cu_seqlen, moba_chunk_size):
+    """calc chunks that needs moba attention"""
+
+    # batch_sizes[batch_idx] = batch size ( seqlen ) of batch idx
+    batch_sizes = cu_seqlen[1:] - cu_seqlen[:-1]
+    # batch_num_chunk[batch_idx] = how many chunk in batch idx
+    batch_num_chunk = (batch_sizes + (moba_chunk_size - 1)) // moba_chunk_size
+    # cu_num_chunk[batch_idx] = first chunk id of this batch
+    cu_num_chunk = torch.ones(
+        batch_num_chunk.numel() + 1,
+        device=cu_seqlen.device,
+        dtype=batch_num_chunk.dtype,
+    )
+    cu_num_chunk[1:] = batch_num_chunk.cumsum(dim=0)
+    # total chunk ( for all batch )
+    num_chunk = cu_num_chunk[-1]
+    # chunk_sizes[chunk_idx] = chunk_size of chunk idx
+    chunk_sizes = torch.full((num_chunk + 1,), moba_chunk_size, dtype=torch.int32, device=cu_seqlen.device)
+    chunk_sizes[0] = 0  # for calc cu chunk
+    batch_last_chunk_size = batch_sizes - (batch_num_chunk - 1) * moba_chunk_size
+    chunk_sizes[cu_num_chunk[1:]] = batch_last_chunk_size
+    # cu_chunk[chunk_idx] = the start chunk offset of chunk idx
+    cu_chunk = chunk_sizes.cumsum(dim=-1, dtype=torch.int32)
+    # chunk_to_batch[chunk_idx] = batch idx of the chunk idx
+    chunk_to_batch = torch.zeros((num_chunk,), dtype=torch.int32, device=cu_seqlen.device)
+    chunk_to_batch[cu_num_chunk[1:-1]] = 1
+    chunk_to_batch = chunk_to_batch.cumsum(dim=0, dtype=torch.int32)
+
+    """ filter chunks that need moba attn """
+
+    # filter chunks ( remove last chunk of each batch )
+    # filtered_chunk_indices: chunk index list that excludes the last chunk of each batch
+    chunk_to_remove = cu_num_chunk[1:] - 1
+    chunk_to_remain = torch.ones((num_chunk,), dtype=torch.bool, device=cu_seqlen.device)
+    chunk_to_remain[chunk_to_remove] = False
+    filtered_chunk_indices = chunk_to_remain.nonzero(as_tuple=True)[0]
+    num_filtered_chunk = len(filtered_chunk_indices)
+
+    return (
+        cu_chunk,
+        filtered_chunk_indices,
+        num_filtered_chunk,
+        chunk_to_batch,
+    )
+
+
+def _prepare_for_moba(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    cu_seqlens: torch.Tensor,
+    max_seqlen: int,
+    moba_chunk_size: int,
+    moba_topk: int,
+    is_decode: bool = False
+) -> torch.Tensor:
+    """An efficient version of moba implementation with triton kernels and flash-attn, the core logic:
+    1. Calculate the chunks and the number of chunks, n = floor(data_size / chunk_size)
+       - tokens in the tail chunk are reserved for self attn
+       - tokens in other chunks will be processed in later steps
+    2. K in each chunk will calculate mean value as the representative k, and Q will attend to these representative
+    k to get the gate logit, which will be used to select topk chunks
+    3. Select the topk chunks and get the dense q for each kv chunk pair and do the varlen attention
+    4. Combine the varlen attn and self attn results via online softmax to get the final result
+
+    Args:
+        q (torch.Tensor): [seqlen, head, head_dim]
+        k (torch.Tensor): [seqlen, head, head_dim]
+        v (torch.Tensor): [seqlen, head, head_dim]
+        cu_seqlens (torch.Tensor): the cumulative sequence length tensor, same definition in flash attn
+        max_seqlen (int): the max sequence length of the batch, same definition in flash attn
+
+    Returns:
+        attn_output (torch.Tensor): [seqlen, head, head_dim]
+    """
+
+    kv = torch.stack((k, v), dim=1)
+
+    """ some basic variables """
+    # qkv shape = [ S, H, D ]
+    seqlen_q, num_head, head_dim = q.shape
+    seqlen_kv, num_head_kv, _ = k.shape
+    replicas = num_head // num_head_kv
+
+    """ prepare chunk meta """
+    (
+        cu_chunk,
+        filtered_chunk_indices,
+        num_filtered_chunk,
+        chunk_to_batch,
+    ) = calc_chunks(cu_seqlens, moba_chunk_size)
+    # cu_chunk: [num_chunks + 1], the start position of each chunk
+    # filtered_chunk_indices: [num_filtered_chunk], the indices of filtered chunk (filter out last in each batch)
+    # chunk_to_batch: [total_num_chunks], chunk_to_batch[i] stands for the batch index of i-th chunk
+
+    self_attn_cu_seqlen = cu_chunk
+    # filtered_kv is a dense matrix that only contains filtered chunk of kv
+    filtered_kv_indices = torch.arange(0, moba_chunk_size, dtype=torch.int64, device=q.device)[None, :].repeat(
+        num_filtered_chunk, 1
+    )
+    filtered_kv_indices += cu_chunk[filtered_chunk_indices][:, None]
+    index_expanded = filtered_kv_indices.view(-1).view(-1, 1, 1, 1).expand(-1, 2, kv.shape[-2], kv.shape[-1])
+    filtered_kv = torch.gather(kv, 0, index_expanded)
+
+    """ calc key_gate_weight and gate """
+
+    # key_gate_weight [ F_N_CHUNK, HEAD, HEAD_DIM ]
+    key_gate_weight = (
+        filtered_kv[:, 0].view(num_filtered_chunk, moba_chunk_size, num_head_kv, head_dim).mean(dim=1)
+    )
+
+    # we will adjust selective topk to moba_topk - 1, as the last chunk is always chosen
+    moba_topk = min(moba_topk - 1, num_filtered_chunk)
+    need_moba_attn = moba_topk > 0
+    # corner case: if no moba attn needed, just return self attn
+    if not need_moba_attn:
+        return None, None, None, None, None, None, None
+
+    query_gate_weight = q.view(seqlen_q, num_head_kv, replicas, head_dim).mean(dim=2).float()
+    key_gate_weight = key_gate_weight.type(torch.float32)  # float logit for better gate logit perception
+    gate = torch.einsum("nhd,shd->nhs", key_gate_weight, query_gate_weight)  # gate [ F_N_CHUNK, HEAD, SEQ ]
+    key_gate_weight = key_gate_weight.type_as(k)
+    q = q.type_as(k)
+
+    # pose process gate, masking unchosen batch and apply causal mask to current chunk
+    gate_seq_idx = torch.arange(0, seqlen_q, device=q.device, dtype=torch.int32)[None, :].repeat(num_filtered_chunk, 1)
+    chunk_end = cu_chunk[filtered_chunk_indices + 1]
+    batch_end = cu_seqlens[chunk_to_batch[filtered_chunk_indices] + 1]
+    gate_chunk_end_mask = gate_seq_idx < chunk_end[:, None]
+    gate_batch_end_mask = gate_seq_idx >= batch_end[:, None]
+    gate_inf_mask = gate_chunk_end_mask | gate_batch_end_mask
+    gate.masked_fill_(gate_inf_mask.unsqueeze(1), -float("inf"))
+
+    """ find moba q that needs moba attn """
+    # find topk chunks
+    # gate_mask [ N_CHUNK, HEAD, SEQ ], true indicates that needs attention
+    _, gate_top_k_idx = torch.topk(gate, k=moba_topk, dim=0, largest=True, sorted=False)
+    # apply causal mask
+    gate_mask = torch.logical_not(gate.isinf())
+    # select topk chunks
+    gate_idx_mask = torch.zeros(gate_mask.shape, dtype=torch.bool, device=q.device)
+    gate_idx_mask = gate_idx_mask.scatter_(dim=0, index=gate_top_k_idx, value=True)
+    gate_mask = torch.logical_and(gate_mask, gate_idx_mask)
+
+    moba_q_indices = nonzero(gate_mask.reshape(gate_mask.shape[0], -1))[-1]  # .nonzero(as_tuple=True)[
+    #     -1
+    # ]  # [ HS indices ] * N
+    # moba_seqlen_q indicates that how many q chunks are selected for each kv chunk - head
+    moba_seqlen_q = gate_mask.sum(dim=-1).flatten()
+    # select all q that needs moba attn based on the moba_q_indices
+
+    # GQA
+    # moba_q_pre = q.transpose(0, 1).reshape(-1, q.size(-1))
+    moba_q = q.view(seqlen_q, num_head_kv, replicas, head_dim)
+    moba_q_pre = moba_q.transpose(0, 1).reshape(-1, *moba_q.shape[2:])
+
+    # GQA
+    index_expanded = moba_q_indices.view(-1, 1, 1).expand(-1, replicas, moba_q_pre.size(-1))
+
+    moba_q = torch.gather(moba_q_pre, 0, index_expanded)
+
+    # moba_q_sh_indices represents the position in the origin q tensor of each q token inside moba_q
+    # GQA
+    moba_q_sh_indices = moba_q_indices % seqlen_q * num_head_kv + moba_q_indices // seqlen_q
+
+    """ prepare moba kv """
+    # Since moba_q is organized as HS * N, we need to reorganize kv to adapt to q
+
+    # cut off zero experts
+    q_zero_mask = moba_seqlen_q == 0
+    valid_expert_mask = ~q_zero_mask
+    zero_expert_count = q_zero_mask.sum()
+    # only keep the kv that has q select > 0
+    if zero_expert_count > 0:
+        moba_seqlen_q = moba_seqlen_q[valid_expert_mask]
+    # moba cu_seqlen for flash attn
+    moba_cu_seqlen_q = torch.cat(
+        (
+            torch.tensor([0], device=q.device, dtype=moba_seqlen_q.dtype),
+            moba_seqlen_q.cumsum(dim=0),
+        ),
+        dim=0,
+    ).to(torch.int32)
+    moba_kv = filtered_kv.permute(2, 0, 1, 3)
+    moba_kv = moba_kv.split(moba_chunk_size, dim=1)
+    moba_kv = torch.cat(moba_kv, dim=0)
+
+    if zero_expert_count > 0:
+        assert valid_expert_mask.sum() == moba_kv.shape[0] - zero_expert_count
+        moba_kv = moba_kv[valid_expert_mask]  # cut off zero Q expert from kv , or the grad may be nan
+    moba_kv = moba_kv.flatten(start_dim=0, end_dim=1).unsqueeze(2)
+    moba_cu_seqlen_kv = (
+        torch.arange(
+            0,
+            num_filtered_chunk * num_head_kv + 1 - zero_expert_count,
+            dtype=torch.int32,
+            device=q.device,
+        )
+        * moba_chunk_size
+    )
+        
+    return self_attn_cu_seqlen, moba_q, moba_kv, moba_cu_seqlen_q, moba_cu_seqlen_kv, moba_chunk_size, moba_q_sh_indices
+
+
+def _moba_attn_varlen_prefill(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    cu_seqlens: torch.Tensor,
+    max_seqlen: int,
+    moba_chunk_size: int,
+    moba_topk: int,
+) -> torch.Tensor:
+    """An efficient version of moba implementation with triton kernels and flash-attn, the core logic:
+    1. Calculate the chunks and the number of chunks, n = floor(data_size / chunk_size)
+       - tokens in the tail chunk are reserved for self attn
+       - tokens in other chunks will be processed in later steps
+    2. K in each chunk will calculate mean value as the representative k, and Q will attend to these representative
+    k to get the gate logit, which will be used to select topk chunks
+    3. Select the topk chunks and get the dense q for each kv chunk pair and do the varlen attention
+    4. Combine the varlen attn and self attn results via online softmax to get the final result
+
+    Args:
+        q (torch.Tensor): [seqlen, head, head_dim]
+        k (torch.Tensor): [seqlen, head, head_dim]
+        v (torch.Tensor): [seqlen, head, head_dim]
+        cu_seqlens (torch.Tensor): the cumulative sequence length tensor, same definition in flash attn
+        max_seqlen (int): the max sequence length of the batch, same definition in flash attn
+
+    Returns:
+        attn_output (torch.Tensor): [seqlen, head, head_dim]
+    """
+
+    self_attn_cu_seqlen, moba_q, moba_kv, moba_cu_seqlen_q, moba_cu_seqlen_kv, moba_chunk_size, moba_q_sh_indices = (
+        _prepare_for_moba(q, k, v, cu_seqlens, max_seqlen, moba_chunk_size, moba_topk)
+    )
+
+    if moba_q is None:
+        return flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, causal=True)
+    softmax_scale = q.shape[-1] ** (-0.5)
+
+    # self attn
+    self_attn_out_sh, self_attn_lse_hs, *rest = flash_attn_varlen_func(
+        q=q,
+        k=k,
+        v=v,
+        cu_seqlens_q=self_attn_cu_seqlen,
+        cu_seqlens_k=self_attn_cu_seqlen,
+        max_seqlen_q=max_seqlen,
+        max_seqlen_k=max_seqlen,
+        softmax_scale=softmax_scale,
+        causal=True,
+        return_attn_probs=True
+    )
+
+    # moba attn
+    moba_attn_out, moba_attn_lse_hs, *rest = flash_attn_varlen_func(
+        q=moba_q,
+        k=moba_kv[:, 0],
+        v=moba_kv[:, 1],
+        cu_seqlens_q=moba_cu_seqlen_q,
+        cu_seqlens_k=moba_cu_seqlen_kv,
+        max_seqlen_q=max_seqlen,
+        max_seqlen_k=moba_chunk_size,
+        softmax_scale=softmax_scale,
+        causal=False,
+        return_attn_probs=True
+    )
+
+    kv_replicas = q.shape[1] // k.shape[1]
+    h, s = self_attn_lse_hs.shape
+
+    # convert lse shape hs -> sh ( follow the legacy mix attn logic )
+    self_attn_lse_sh = self_attn_lse_hs.t().view(s, k.shape[1], kv_replicas).contiguous()
+    moba_attn_lse = moba_attn_lse_hs.t().contiguous()
+
+    max_lse_1d = self_attn_lse_sh.view(-1, kv_replicas)
+    max_lse_1d = max_lse_1d.index_reduce(0, moba_q_sh_indices, moba_attn_lse, "amax")
+    self_attn_lse_sh = self_attn_lse_sh - max_lse_1d.view_as(self_attn_lse_sh)
+
+    moba_attn_lse = (
+        moba_attn_lse.view(-1, kv_replicas).sub(max_lse_1d.index_select(0, moba_q_sh_indices)).reshape_as(moba_attn_lse)
+    )
+
+    mixed_attn_se_sh = self_attn_lse_sh.exp()
+    moba_attn_se = moba_attn_lse.exp()
+
+    mixed_view = mixed_attn_se_sh.view(-1, kv_replicas)
+    result_view = mixed_view.index_add(0, moba_q_sh_indices, moba_attn_se.view(-1, kv_replicas))
+
+    mixed_attn_se_sh = result_view.view_as(mixed_attn_se_sh)
+    mixed_attn_lse_sh = mixed_attn_se_sh.log()
+
+    # add attn output
+    factor = (self_attn_lse_sh - mixed_attn_lse_sh).exp()  # [ vS, H ]
+    self_attn_out_sh = self_attn_out_sh * factor.view(self_attn_out_sh.shape[0], self_attn_out_sh.shape[1], 1)
+    output_2d = self_attn_out_sh.reshape(q.shape[0] * k.shape[1], kv_replicas, q.shape[2])
+
+    # add moba output
+    mixed_attn_lse = mixed_attn_lse_sh.view(-1, kv_replicas).index_select(0, moba_q_sh_indices).view_as(moba_attn_lse)
+    factor = (moba_attn_lse - mixed_attn_lse).exp()  # [ vS, H ]
+    moba_attn_out = moba_attn_out * factor.unsqueeze(-1)
+    raw_attn_out = moba_attn_out.view(-1, kv_replicas, moba_attn_out.shape[-1])
+    output_2d.index_add_(0, moba_q_sh_indices, raw_attn_out)
+
+    # add back max lse
+    mixed_attn_lse_sh = mixed_attn_lse_sh + max_lse_1d.view_as(mixed_attn_se_sh)
+
+    return output_2d.view(q.shape[0], q.shape[1], q.shape[2]).to(q.dtype)
+
+
+def roll_tensor(tensor, shifts=-1, dims=-1, fill_value=0):
+    """Roll the tensor input along the given dimension(s).
+    Inserted elements are set to be 0.0.
+    """
+    rolled_tensor = torch.roll(tensor, shifts=shifts, dims=dims)
+    rolled_tensor.select(dims, shifts).fill_(fill_value)
+    return rolled_tensor, rolled_tensor.sum()
+
+
+@dataclass
+class MoEV2CausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for causal language model (or autoregressive) outputs as well as Mixture of Expert's router hidden
+    states terms, to train a MoE model.
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        z_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
+            z_loss for the sparse modules.
+        aux_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
+            aux_loss for the sparse modules.
+        router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_logits=True` is passed or when `config.add_router_probs=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.
+            Router logits of the encoder model, useful to compute the auxiliary loss and the z_loss for the sparse
+            modules.
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: Optional[torch.FloatTensor] = None
+    past_key_values: Optional[Cache] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+    z_loss: Optional[torch.FloatTensor] = None
+    aux_loss: Optional[torch.FloatTensor] = None
+    router_logits: Optional[tuple[torch.FloatTensor]] = None
+    mtp_loss: Optional[torch.FloatTensor] = None
+    mtp_logits: Optional[tuple[torch.FloatTensor, ...]] = None
+
+
+class MoeV2ModelOutputWithPast(MoeModelOutputWithPast):
+
+    def __init__(self, mtp_hidden_states=None, **kwargs):
+        super().__init__(**kwargs)
+        self.mtp_hidden_states = mtp_hidden_states
+
+
 def _get_unpad_data(attention_mask):
     seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
     indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
@@ -133,171 +502,37 @@ ALL_LAYERNORM_LAYERS.append(BailingMoeV2RMSNorm)
 
 
 class BailingMoeV2RotaryEmbedding(nn.Module):
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+    def __init__(self, config: BailingMoeV2Config, device=None):
         super().__init__()
+        # BC: "rope_type" was originally "type"
+        if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+        else:
+            self.rope_type = "default"
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
 
-        self.dim = dim
-        self.max_position_embeddings = max_position_embeddings
-        self.base = base
-        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
-
-        # Build here to make `torch.jit.trace` work.
-        self._set_cos_sin_cache(
-            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
-        )
-        self.max_seq_len_cached = None
-
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
-
-        freqs = torch.outer(t, self.inv_freq.to(t.device))
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
-        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
-
-    def forward(self, x, seq_len=None):
-        # x: [bs, num_attention_heads, seq_len, head_size]
-        if self.max_seq_len_cached is None or seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
-
-        return (
-            self.cos_cached[:seq_len].to(dtype=x.dtype),
-            self.sin_cached[:seq_len].to(dtype=x.dtype),
-        )
-
-
-# Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->BailingMoeV2
-class BailingMoeV2LinearScalingRotaryEmbedding(BailingMoeV2RotaryEmbedding):
-    """BailingMoeV2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
-
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
-        self.scaling_factor = scaling_factor
-        super().__init__(dim, max_position_embeddings, base, device)
-
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
-        t = t / self.scaling_factor
-
-        freqs = torch.outer(t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
-        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
-
-
-# Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->BailingMoeV2
-class BailingMoeV2DynamicNTKScalingRotaryEmbedding(BailingMoeV2RotaryEmbedding):
-    """BailingMoeV2RotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
-
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
-        self.scaling_factor = scaling_factor
-        super().__init__(dim, max_position_embeddings, base, device)
-
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-
-        if seq_len > self.max_position_embeddings:
-            base = self.base * (
-                (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
-            ) ** (self.dim / (self.dim - 2))
-            inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
-            self.register_buffer("inv_freq", inv_freq, persistent=False)
-
-        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
-
-        freqs = torch.outer(t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
-        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
-
-
-# Inverse dim formula to find dim based on number of rotations
-def yarn_find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048):
-    return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base))
-
-
-# Find dim range bounds based on rotations
-def yarn_find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embeddings=2048):
-    low = math.floor(yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
-    high = math.ceil(yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings))
-    return max(low, 0), min(high, dim - 1)  # Clamp values just in case
-
-
-def yarn_get_mscale(scale=1, mscale=1):
-    if scale <= 1:
-        return 1.0
-    return 0.1 * mscale * math.log(scale) + 1.0
-
-
-def yarn_linear_ramp_mask(min, max, dim):
-    if min == max:
-        max += 0.001  # Prevent singularity
-
-    linear_func = (torch.arange(dim, dtype=torch.float32) - min) / (max - min)
-    ramp_func = torch.clamp(linear_func, 0, 1)
-    return ramp_func
-
-
-class BailingMoeV2YarnRotaryEmbedding(BailingMoeV2RotaryEmbedding):
-
-    def __init__(
-        self,
-        dim,
-        max_position_embeddings=2048,
-        base=10000,
-        device=None,
-        scaling_factor=1.0,
-        original_max_position_embeddings=4096,
-        beta_fast=32,
-        beta_slow=1,
-        mscale=1,
-        mscale_all_dim=0,
-    ):
-        self.scaling_factor = scaling_factor
-        self.original_max_position_embeddings = original_max_position_embeddings
-        self.beta_fast = beta_fast
-        self.beta_slow = beta_slow
-        self.mscale = mscale
-        self.mscale_all_dim = mscale_all_dim
-        super().__init__(dim, max_position_embeddings, base, device)
-
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        dim = self.dim
-
-        freq_extra = 1.0 / (self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))
-        freq_inter = 1.0 / (
-            self.scaling_factor * self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)
-        )
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
 
-        low, high = yarn_find_correction_range(
-            self.beta_fast,
-            self.beta_slow,
-            dim,
-            self.base,
-            self.original_max_position_embeddings,
-        )
-        inv_freq_mask = 1.0 - yarn_linear_ramp_mask(low, high, dim // 2).to(device=device, dtype=torch.float32)
-        inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
         self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
 
-        t = torch.arange(seq_len, device=device, dtype=torch.float32)
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
 
-        freqs = torch.outer(t, inv_freq)
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
 
-        _mscale = float(
-            yarn_get_mscale(self.scaling_factor, self.mscale)
-            / yarn_get_mscale(self.scaling_factor, self.mscale_all_dim)
-        )
-
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer("cos_cached", (emb.cos() * _mscale).to(dtype), persistent=False)
-        self.register_buffer("sin_cached", (emb.sin() * _mscale).to(dtype), persistent=False)
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
 
 
 # Copied from transformers.models.llama.modeling_llama.rotate_half
@@ -309,17 +544,13 @@ def rotate_half(x):
 
 
 # Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
-def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
     """Applies Rotary Position Embedding to the query and key tensors.
-
     Args:
         q (`torch.Tensor`): The query tensor.
         k (`torch.Tensor`): The key tensor.
         cos (`torch.Tensor`): The cosine part of the rotary embedding.
         sin (`torch.Tensor`): The sine part of the rotary embedding.
-        position_ids (`torch.Tensor`):
-            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
-            used to pass offsetted position ids when working with a KV-cache.
         unsqueeze_dim (`int`, *optional*, defaults to 1):
             The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
             sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
@@ -330,10 +561,21 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
     Returns:
         `tuple(torch.Tensor)` comprising the query and key tensors rotated using the Rotary Position Embedding.
     """
-    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
-    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
-    q_embed = (q * cos) + (rotate_half(q) * sin)
-    k_embed = (k * cos) + (rotate_half(k) * sin)
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+
+    # Keep half or full tensor for later concatenation
+    rotary_dim = cos.shape[-1]
+    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+
+    # Apply rotary embeddings on the first half or full tensor
+    q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)
+    k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)
+
+    # Concatenate back to full shape
+    q_embed = torch.cat([q_embed, q_pass], dim=-1)
+    k_embed = torch.cat([k_embed, k_pass], dim=-1)
     return q_embed, k_embed
 
 
@@ -360,14 +602,15 @@ class BailingMoeV2Gate(nn.Module):
         self.top_k = config.num_experts_per_tok
         self.num_experts = config.num_experts
 
+        self.n_group = config.n_group
+        self.topk_group = config.topk_group
+
         # topk selection algorithm
-        self.norm_topk_prob = config.norm_topk_prob
         self.gating_dim = config.hidden_size
         self.weight = nn.Parameter(torch.empty((self.num_experts, self.gating_dim)))
-        self.moe_router_topk_scaling_factor = config.moe_router_topk_scaling_factor
+        self.routed_scaling_factor = config.routed_scaling_factor
 
-        if self.config.use_expert_bias:
-            self.register_buffer("expert_bias", torch.zeros((self.num_experts)))
+        self.register_buffer("expert_bias", torch.zeros((self.num_experts)))
         self.reset_parameters()
 
     def reset_parameters(self) -> None:
@@ -375,39 +618,45 @@ class BailingMoeV2Gate(nn.Module):
 
         init.kaiming_uniform_(self.weight, a=math.sqrt(5))
 
+    def group_limited_topk(
+        self,
+        scores: torch.Tensor,
+    ):
+        num_tokens, _ = scores.size()
+        # Organize the experts into groups
+        group_scores = scores.view(num_tokens, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
+        group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+        group_mask = torch.zeros_like(group_scores)
+        group_mask.scatter_(1, group_idx, 1)
+
+        # Mask the experts based on selection groups
+        score_mask = (
+            group_mask.unsqueeze(-1)
+            .expand(num_tokens, self.n_group, self.num_experts // self.n_group)
+            .reshape(num_tokens, -1)
+        )
+
+        masked_scores = scores.masked_fill(~score_mask.bool(), float('-inf'))
+        probs, top_indices = torch.topk(masked_scores, k=self.top_k, dim=-1)
+
+        return probs, top_indices
+
     def forward(self, hidden_states):
         # compute gating score
         hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
-        logits = F.linear(hidden_states, self.weight, None)
-
-        if self.config.gate_score_function == 'softmax':
-            scores = logits.softmax(dim=-1, dtype=torch.float32)
+        logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32))
 
-            # select top-k experts
-            topk_weight, topk_idx = torch.topk(scores, k=self.top_k, dim=-1)
+        scores = torch.sigmoid(logits.float()).type_as(logits)
 
-            # norm gate to sum 1
-            if self.top_k > 1 and self.norm_topk_prob:
-                denominator = topk_weight.sum(dim=-1, keepdim=True)
-                topk_weight = topk_weight / denominator
-            topk_weight = topk_weight * self.moe_router_topk_scaling_factor
+        scores_for_routing = scores + self.expert_bias
+        _, topk_idx = self.group_limited_topk(scores_for_routing)
 
-            return topk_idx, topk_weight, logits
-        elif self.config.gate_score_function == 'sigmoid':
-            scores = torch.sigmoid(logits)
+        scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
 
-            if self.config.use_expert_bias:
-                scores_for_routing = scores + self.expert_bias
-                _, topk_idx = torch.topk(scores_for_routing, k=self.top_k, dim=-1)
-                scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
-            else:
-                scores, topk_idx = torch.topk(scores, k=self.top_k, dim=-1)
-            topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
-            topk_weight = topk_weight * self.moe_router_topk_scaling_factor
+        topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
+        topk_weight = topk_weight * self.routed_scaling_factor
 
-            return topk_idx, topk_weight, logits
-        else:
-            raise ValueError(f"Unsupported gate_score_function: {self.config.gate_score_function}")
+        return topk_idx, topk_weight, logits
 
 
 class BailingMoeV2SparseMoeBlock(nn.Module):
@@ -460,7 +709,6 @@ class BailingMoeV2SparseMoeBlock(nn.Module):
         tokens_per_expert = cnts.sum(dim=0)
         idxs = topk_ids.view(-1).argsort()
         sorted_tokens = x[idxs // topk_ids.shape[1]]
-        sorted_tokens_shape = sorted_tokens.shape
         tokens_per_expert = tokens_per_expert.cpu().numpy()
         outputs = []
         start_idx = 0
@@ -471,7 +719,7 @@ class BailingMoeV2SparseMoeBlock(nn.Module):
             expert = self.experts[i]
             tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
             expert_out = expert(tokens_for_this_expert)
-            outputs.append(expert_out)
+            outputs.append(expert_out.to(x.device))
             start_idx = end_idx
 
         outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
@@ -519,6 +767,8 @@ class BailingMoeV2Attention(nn.Module):
         self.hidden_size = config.hidden_size
         self.num_heads = config.num_attention_heads
         self.head_dim = config.head_dim or self.hidden_size // self.num_heads
+        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+        self.rope_dim = int(self.head_dim * partial_rotary_factor)
         self.num_key_value_heads = config.num_key_value_heads
         self.num_key_value_groups = self.num_heads // self.num_key_value_heads
         self.max_position_embeddings = config.max_position_embeddings
@@ -532,56 +782,9 @@ class BailingMoeV2Attention(nn.Module):
         )
 
         if self.config.use_qk_norm:
-            self.q_norm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
-            self.k_norm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+            self.query_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+            self.key_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
         self.dense = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.use_bias)
-        self._init_rope()
-
-    def _init_rope(self):
-        if self.config.rope_scaling is None:
-            self.rotary_emb = BailingMoeV2RotaryEmbedding(
-                self.head_dim,
-                max_position_embeddings=self.max_position_embeddings,
-                base=self.rope_theta,
-            )
-        else:
-            scaling_type = self.config.rope_scaling["type"]
-            scaling_factor = self.config.rope_scaling["factor"]
-            if scaling_type == "linear":
-                self.rotary_emb = BailingMoeV2LinearScalingRotaryEmbedding(
-                    self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
-                    scaling_factor=scaling_factor,
-                    base=self.rope_theta,
-                )
-            elif scaling_type == "dynamic":
-                self.rotary_emb = BailingMoeV2DynamicNTKScalingRotaryEmbedding(
-                    self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
-                    scaling_factor=scaling_factor,
-                    base=self.rope_theta,
-                )
-            elif scaling_type == "yarn":
-                kwargs = {
-                    key: self.config.rope_scaling[key]
-                    for key in [
-                        "original_max_position_embeddings",
-                        "beta_fast",
-                        "beta_slow",
-                        "mscale",
-                        "mscale_all_dim",
-                    ]
-                    if key in self.config.rope_scaling
-                }
-                self.rotary_emb = BailingMoeV2YarnRotaryEmbedding(
-                    self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
-                    scaling_factor=scaling_factor,
-                    base=self.rope_theta,
-                    **kwargs,
-                )
-            else:
-                raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
 
     def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
         return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
@@ -594,12 +797,9 @@ class BailingMoeV2Attention(nn.Module):
         past_key_value: Optional[Cache] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
         **kwargs,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        if "padding_mask" in kwargs:
-            warnings.warn(
-                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
-            )
 
         bsz, q_len, _ = hidden_states.size()
 
@@ -614,10 +814,12 @@ class BailingMoeV2Attention(nn.Module):
         value_states = value_states.transpose(1, 2)
 
         if self.config.use_qk_norm:
-            query_states = self.q_norm(query_states)
-            key_states = self.k_norm(key_states)
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
 
-        kv_seq_len = key_states.shape[-2]
         if past_key_value is not None:
             if self.layer_idx is None:
                 raise ValueError(
@@ -625,19 +827,15 @@ class BailingMoeV2Attention(nn.Module):
                     "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
                     "with a layer index."
                 )
-            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
-
-        if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            cache_kwargs = {"sin": sin, "cos": cos}
             key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
 
         key_states = repeat_kv(key_states, self.num_key_value_groups)
         value_states = repeat_kv(value_states, self.num_key_value_groups)
 
-        attn_weights = torch.matmul(query_states / math.sqrt(self.head_dim), key_states.transpose(2, 3))
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
 
+        kv_seq_len = key_states.shape[-2]
         if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
             raise ValueError(
                 f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
@@ -698,17 +896,10 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
         past_key_value: Optional[Cache] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
         **kwargs,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         # BailingMoeV2FlashAttention2 attention does not support output_attentions
-        if "padding_mask" in kwargs:
-            warnings.warn(
-                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
-            )
-
-            # overwrite attention_mask with padding_mask
-            attention_mask = kwargs.pop("padding_mask")
-
         output_attentions = False
 
         bsz, q_len, _ = hidden_states.size()
@@ -728,17 +919,14 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
         value_states = value_states.transpose(1, 2)
 
         if self.config.use_qk_norm:
-            query_states = self.q_norm(query_states)
-            key_states = self.k_norm(key_states)
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
 
-        kv_seq_len = key_states.shape[-2]
-        if past_key_value is not None:
-            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
 
         if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            cache_kwargs = {"sin": sin, "cos": cos}
             key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
 
         # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
@@ -763,7 +951,7 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
             elif torch.is_autocast_enabled():
                 target_dtype = torch.get_autocast_gpu_dtype()
             else:
-                target_dtype = self.q_proj.weight.dtype
+                target_dtype = self.query_key_value.weight.dtype
 
             logger.warning_once(
                 f"The input hidden states seems to be silently casted in float32, this might be related to"
@@ -774,10 +962,14 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
             query_states = query_states.to(target_dtype)
             key_states = key_states.to(target_dtype)
             value_states = value_states.to(target_dtype)
-
-        attn_output = self._flash_attention_forward(
-            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
-        )
+        if hasattr(self.config, "moba_topk"):
+            attn_output = self._mixture_attention_forward(
+                query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+            )
+        else:
+            attn_output = self._flash_attention_forward(
+                query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+            )
 
         attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
         attn_output = self.dense(attn_output)
@@ -786,6 +978,85 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
             attn_weights = None
 
         return attn_output, attn_weights, past_key_value
+    
+    def _mixture_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+            query_length (`int`):
+                The length of the query sequence in terms of tokens. This represents the number of tokens in the
+                `query_states` tensor along the sequence dimension. It is used to determine the effective sequence
+                length for attention computations.
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in BailingMoeV2FlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        if query_length != 1:
+          # prefill
+          # Contains at least one padding token in the sequence
+          if attention_mask is not None:
+              batch_size = query_states.shape[0]
+              query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                  query_states, key_states, value_states, attention_mask, query_length
+              )
+
+              cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+              max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+              attn_output_unpad = _moba_attn_varlen_prefill(
+                  query_states,
+                  key_states,
+                  value_states,
+                  cu_seqlens=cu_seqlens_k,
+                  max_seqlen=max_seqlen_in_batch_k,
+                  moba_chunk_size=self.config.moba_block_size,
+                  moba_topk=self.config.moba_topk
+              )
+
+              attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+          else:
+              batch_size = query_states.shape[0]
+              cu_seqlens_k = torch.cumsum(
+                  torch.tensor([0] + [query_length] * batch_size, device=query_states.device),
+                  dim=0,
+                  dtype=torch.int32,
+              )
+              query_states = query_states.view(-1, self.num_heads, self.head_dim)
+              key_states = key_states.view(-1, self.num_key_value_heads, self.head_dim)
+              value_states = value_states.view(-1, self.num_key_value_heads, self.head_dim)
+              attn_output = _moba_attn_varlen_prefill(
+                  query_states,
+                  key_states,
+                  value_states,
+                  cu_seqlens=cu_seqlens_k,
+                  max_seqlen=query_length,
+                  moba_chunk_size=self.config.moba_block_size,
+                  moba_topk=self.config.moba_topk
+              ).view(batch_size, query_length, -1)
+        else:
+            # decode
+            attn_output = self._flash_attention_forward(
+              query_states, key_states, value_states, attention_mask, query_length, dropout, softmax_scale
+            )
+        return attn_output
 
     def _flash_attention_forward(
         self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
@@ -793,7 +1064,6 @@ class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
         """
         Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
         first unpad the input, then computes the attention scores and pad the final attention scores.
-
         Args:
             query_states (`torch.Tensor`):
                 Input query states to be passed to Flash Attention API
@@ -906,6 +1176,7 @@ class BailingMoeV2SdpaAttention(BailingMoeV2Attention):
         past_key_value: Optional[Cache] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
         **kwargs,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         if output_attentions:
@@ -936,24 +1207,21 @@ class BailingMoeV2SdpaAttention(BailingMoeV2Attention):
         value_states = value_states.transpose(1, 2)
 
         if self.config.use_qk_norm:
-            query_states = self.q_norm(query_states)
-            key_states = self.k_norm(key_states)
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
 
-        kv_seq_len = key_states.shape[-2]
-        if past_key_value is not None:
-            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
-
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
 
         if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            cache_kwargs = {"sin": sin, "cos": cos}
             key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
 
         key_states = repeat_kv(key_states, self.num_key_value_groups)
         value_states = repeat_kv(value_states, self.num_key_value_groups)
 
         if attention_mask is not None:
+            kv_seq_len = key_states.shape[-2]
             if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
                 raise ValueError(
                     f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
@@ -991,6 +1259,78 @@ ATTENTION_CLASSES = {
 }
 
 
+class BailingMoeV2MTPLayer(nn.Module):
+    def __init__(self, config: BailingMoeV2Config, layer_idx: int):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.input_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.enorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
+        self.post_attention_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.attention = ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
+        self.mlp = BailingMoeV2SparseMoeBlock(config)
+
+        self.hnorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.final_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        input_embeds,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        output_router_logits: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        input_embeds = self.enorm(input_embeds)
+        hidden_states = self.hnorm(hidden_states)
+        hidden_states = self.eh_proj(torch.cat([input_embeds, hidden_states], dim=-1))
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.attention(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            position_embeddings=position_embeddings,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        if isinstance(hidden_states, tuple):
+            hidden_states, router_logits = hidden_states
+        else:
+            router_logits = None
+        hidden_states = residual + hidden_states.to(residual.device)
+        hidden_states = self.final_layernorm(hidden_states)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        if output_router_logits:
+            outputs += (router_logits,)
+
+        return outputs
+
+
 class BailingMoeV2DecoderLayer(nn.Module):
     def __init__(self, config: BailingMoeV2Config, layer_idx: int):
         super().__init__()
@@ -1015,6 +1355,7 @@ class BailingMoeV2DecoderLayer(nn.Module):
         output_attentions: Optional[bool] = False,
         output_router_logits: Optional[bool] = False,
         use_cache: Optional[bool] = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
         **kwargs,
     ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
         """
@@ -1038,10 +1379,6 @@ class BailingMoeV2DecoderLayer(nn.Module):
                 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                 (see `past_key_values`).
         """
-        if "padding_mask" in kwargs:
-            warnings.warn(
-                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
-            )
         residual = hidden_states
 
         hidden_states = self.input_layernorm(hidden_states)
@@ -1053,6 +1390,7 @@ class BailingMoeV2DecoderLayer(nn.Module):
             position_ids=position_ids,
             past_key_value=past_key_value,
             output_attentions=output_attentions,
+            position_embeddings=position_embeddings,
             use_cache=use_cache,
         )
         hidden_states = residual + hidden_states
@@ -1065,7 +1403,7 @@ class BailingMoeV2DecoderLayer(nn.Module):
             hidden_states, router_logits = hidden_states
         else:
             router_logits = None
-        hidden_states = residual + hidden_states
+        hidden_states = residual + hidden_states.to(residual.device)
 
         outputs = (hidden_states,)
 
@@ -1085,11 +1423,9 @@ BAILINGMOEV2_START_DOCSTRING = r"""
     This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
     library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
     etc.)
-
     This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
     Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
     and behavior.
-
     Parameters:
         config ([`BailingMoeV2Config`]):
             Model configuration class with all the parameters of the model. Initializing with a config file does not
@@ -1129,50 +1465,38 @@ BAILINGMOEV2_INPUTS_DOCSTRING = r"""
         input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
             Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
             it.
-
             Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
             [`PreTrainedTokenizer.__call__`] for details.
-
             [What are input IDs?](../glossary#input-ids)
         attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
             Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
             - 1 for tokens that are **not masked**,
             - 0 for tokens that are **masked**.
-
             [What are attention masks?](../glossary#attention-mask)
-
             Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
             [`PreTrainedTokenizer.__call__`] for details.
-
             If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
             `past_key_values`).
-
             If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
             and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
             information on the default strategy.
-
             - 1 indicates the head is **not masked**,
             - 0 indicates the head is **masked**.
         position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
             Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
             config.n_positions - 1]`.
-
             [What are position IDs?](../glossary#position-ids)
         past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
             Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
             blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
             returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
-
             Two formats are allowed:
             - a [`~cache_utils.Cache`] instance;
             - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
             shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
             cache format.
-
             The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
             legacy cache format will be returned.
-
             If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
             have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
             of shape `(batch_size, sequence_length)`.
@@ -1201,7 +1525,6 @@ BAILINGMOEV2_INPUTS_DOCSTRING = r"""
 class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
     """
     Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`BailingMoeV2DecoderLayer`]
-
     Args:
         config: BailingMoeV2Config
     """
@@ -1210,15 +1533,20 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
         super().__init__(config)
         self.padding_idx = config.pad_token_id
         self.vocab_size = config.vocab_size
+        self.num_nextn_predict_layers = config.num_nextn_predict_layers
 
         self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
-        self.layers = nn.ModuleList(
-            [BailingMoeV2DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
-        )
+        self.layers = []
+        for layer_idx in range(config.num_hidden_layers + config.num_nextn_predict_layers):
+            layer_cls = BailingMoeV2DecoderLayer if layer_idx < config.num_hidden_layers else BailingMoeV2MTPLayer
+            self.layers.append(layer_cls(config, layer_idx))
+
+        self.layers = nn.ModuleList(self.layers)
+
         self._use_sdpa = config._attn_implementation == "sdpa"
         self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
         self.norm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-
+        self.rotary_emb = BailingMoeV2RotaryEmbedding(config=config)
         self.gradient_checkpointing = False
         # Initialize weights and apply final processing
         self.post_init()
@@ -1243,7 +1571,7 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
         output_router_logits: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         **kwargs,
-    ) -> Union[Tuple, MoeModelOutputWithPast]:
+    ) -> Union[Tuple, MoeV2ModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
@@ -1272,23 +1600,20 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
                 )
                 use_cache = False
 
-        past_key_values_length = 0
-        if use_cache:
-            use_legacy_cache = not isinstance(past_key_values, Cache)
-            if use_legacy_cache:
-                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
-            past_key_values_length = past_key_values.get_usable_length(seq_length)
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache()
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+
+        past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
 
         if position_ids is None:
-            device = input_ids.device if input_ids is not None else inputs_embeds.device
             position_ids = torch.arange(
-                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
             )
             position_ids = position_ids.unsqueeze(0)
 
-        if inputs_embeds is None:
-            inputs_embeds = self.word_embeddings(input_ids)
-
         if self._use_flash_attention_2:
             # 2d mask is passed through the layers
             attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
@@ -1299,24 +1624,29 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
                 attention_mask,
                 (batch_size, seq_length),
                 inputs_embeds,
-                past_key_values_length,
+                past_seen_tokens,
             )
         else:
             # 4d mask is passed through the layers
             attention_mask = _prepare_4d_causal_attention_mask(
-                attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
+                attention_mask, (batch_size, seq_length), inputs_embeds, past_seen_tokens
             )
 
         # embed positions
         hidden_states = inputs_embeds
 
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
         # decoder layers
         all_hidden_states = () if output_hidden_states else None
         all_self_attns = () if output_attentions else None
         all_router_logits = () if output_router_logits else None
         next_decoder_cache = None
+        layers = self.layers[: -self.num_nextn_predict_layers] if self.num_nextn_predict_layers > 0 else self.layers
+        mtp_layers = self.layers[-self.num_nextn_predict_layers :] if self.num_nextn_predict_layers > 0 else None
 
-        for decoder_layer in self.layers:
+        for decoder_layer in layers:
             if output_hidden_states:
                 all_hidden_states += (hidden_states,)
 
@@ -1330,6 +1660,7 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
                     output_attentions,
                     output_router_logits,
                     use_cache,
+                    position_embeddings,
                 )
             else:
                 layer_outputs = decoder_layer(
@@ -1340,6 +1671,7 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
                     output_attentions=output_attentions,
                     output_router_logits=output_router_logits,
                     use_cache=use_cache,
+                    position_embeddings=position_embeddings,
                 )
             hidden_states = layer_outputs[0]
 
@@ -1353,38 +1685,90 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
                 all_router_logits += (layer_outputs[-1],)
 
         hidden_states = self.norm(hidden_states)
+        main_hidden_states = hidden_states
 
         # add hidden states from the last decoder layer
         if output_hidden_states:
-            all_hidden_states += (hidden_states,)
+            all_hidden_states += (main_hidden_states,)
+
+        mtp_hidden_states = None
+
+        if mtp_layers:
+            for decoder_layer in mtp_layers:
+                input_ids, _ = roll_tensor(input_ids, shifts=-1, dims=-1)
+                inputs_embeds = self.word_embeddings(input_ids)
+
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        decoder_layer.__call__,
+                        inputs_embeds,
+                        hidden_states,
+                        attention_mask,
+                        position_ids,
+                        past_key_values,
+                        output_attentions,
+                        output_router_logits,
+                        use_cache,
+                        position_embeddings,
+                    )
+                else:
+                    layer_outputs = decoder_layer(
+                        inputs_embeds,
+                        hidden_states,
+                        attention_mask=attention_mask,
+                        position_ids=position_ids,
+                        past_key_value=past_key_values,
+                        output_attentions=output_attentions,
+                        output_router_logits=output_router_logits,
+                        use_cache=use_cache,
+                        position_embeddings=position_embeddings,
+                    )
+                if mtp_hidden_states is None:
+                    mtp_hidden_states = []
+                hidden_states = layer_outputs[0]
+                mtp_hidden_states.append(hidden_states)
+
+                if output_hidden_states:
+                    all_hidden_states += (hidden_states,)
+
+                if use_cache:
+                    next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+                if output_attentions:
+                    all_self_attns += (layer_outputs[1],)
+
+                if output_router_logits and layer_outputs[-1] is not None:
+                    all_router_logits += (layer_outputs[-1],)
 
         next_cache = None
         if use_cache:
-            next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
+            next_cache = next_decoder_cache
         if not return_dict:
             return tuple(
                 v
-                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits]
+                for v in [main_hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits]
                 if v is not None
             )
-        return MoeModelOutputWithPast(
-            last_hidden_state=hidden_states,
+        return MoeV2ModelOutputWithPast(
+            last_hidden_state=main_hidden_states,
             past_key_values=next_cache,
             hidden_states=all_hidden_states,
+            mtp_hidden_states=mtp_hidden_states,
             attentions=all_self_attns,
             router_logits=all_router_logits,
         )
 
 
-class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel):
+class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel, GenerationMixin):
     _tied_weights_keys = ["lm_head.weight"]
 
     def __init__(self, config: BailingMoeV2Config):
         super().__init__(config)
         self.model = BailingMoeV2Model(config)
         self.vocab_size = config.vocab_size
-        self.norm_head = config.norm_head
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.num_nextn_predict_layers = config.num_nextn_predict_layers
+        self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor
 
         # Initialize weights and apply final processing
         self.post_init()
@@ -1407,26 +1791,8 @@ class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel):
     def get_decoder(self):
         return self.model
 
-    def compute_logit(self, hidden_states):
-        if self.norm_head:
-            if self.training:
-                norm_weight = (
-                    self.lm_head.weight / (torch.norm(self.lm_head.weight, p=2, dim=0, keepdim=True) + 1e-7).detach()
-                )
-                logits = F.linear(hidden_states, norm_weight, None)
-            else:
-                self.lm_head.weight.data = (
-                    self.lm_head.weight.data.float()
-                    / (torch.norm(self.lm_head.weight.data.float(), p=2, dim=0, keepdim=True) + 1e-7)
-                ).to(hidden_states.dtype)
-                logits = F.linear(hidden_states, self.lm_head.weight.data, None)
-                self.norm_head = False
-        else:
-            logits = self.lm_head(hidden_states)
-        return logits
-
     @add_start_docstrings_to_model_forward(BAILINGMOEV2_INPUTS_DOCSTRING)
-    @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    @replace_return_docstrings(output_type=MoEV2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
     def forward(
         self,
         input_ids: torch.LongTensor = None,
@@ -1441,27 +1807,21 @@ class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel):
         output_router_logits: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         **kwargs,
-    ) -> Union[Tuple, MoeCausalLMOutputWithPast]:
+    ) -> Union[Tuple, MoEV2CausalLMOutputWithPast]:
         r"""
         Args:
             labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
-
         Returns:
-
         Example:
-
         ```python
         >>> from transformers import AutoTokenizer
-
         >>> model = BailingMoeV2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
         >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
-
         >>> prompt = "Hey, are you conscious? Can you talk to me?"
         >>> inputs = tokenizer(prompt, return_tensors="pt")
-
         >>> # Generate
         >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
         >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
@@ -1490,25 +1850,40 @@ class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel):
             **kwargs,
         )
 
-        hidden_states = outputs[0]
-
-        logits = self.compute_logit(hidden_states=hidden_states)
-        logits = logits.float()
-
         loss = None
+        all_mtp_loss = None
         aux_loss = None
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
 
         if labels is not None:
-            # Shift so that tokens < n predict n
-            shift_logits = logits[..., :-1, :].contiguous()
-            shift_labels = labels[..., 1:].contiguous()
-            # Flatten the tokens
-            loss_fct = CrossEntropyLoss()
-            shift_logits = shift_logits.view(-1, self.config.vocab_size)
-            shift_labels = shift_labels.view(-1)
-            # Enable model parallelism
-            shift_labels = shift_labels.to(shift_logits.device)
-            loss = loss_fct(shift_logits, shift_labels)
+            loss = self.loss_function(logits, labels, self.config.vocab_size, **kwargs)
+
+        all_mtp_logits = None
+        if self.num_nextn_predict_layers > 0:
+            mtp_hidden_states = outputs.mtp_hidden_states
+            shift_labels_mtp = None
+            for i in range(self.num_nextn_predict_layers):
+                mtp_hidden_states = mtp_hidden_states[i]
+                mtp_logits = self.lm_head(mtp_hidden_states).float()
+                if all_mtp_logits is None:
+                    all_mtp_logits = []
+                all_mtp_logits.append(mtp_logits)
+                if labels is not None:
+                    if shift_labels_mtp is None:
+                        shift_labels_mtp = labels.clone()
+                    shift_labels_mtp, _ = roll_tensor(shift_labels_mtp, shifts=-1, dims=-1, fill_value=-100)
+                    mtp_logits_ = mtp_logits.view(-1, self.config.vocab_size)
+                    mtp_loss = self.loss_function(mtp_logits_, shift_labels_mtp.to(mtp_logits_.device).view(-1), self.config.vocab_size, **kwargs)
+                    if loss is not None:
+                        loss += self.mtp_loss_scaling_factor * mtp_loss
+                    else:
+                        loss = self.mtp_loss_scaling_factor * mtp_loss
+
+                    if all_mtp_loss is None:
+                        all_mtp_loss = []
+                    all_mtp_loss.append(mtp_loss)
 
         if not return_dict:
             output = (logits,) + outputs[1:]
@@ -1516,82 +1891,14 @@ class BailingMoeV2ForCausalLM(BailingMoeV2PreTrainedModel):
                 output = (aux_loss,) + output
             return (loss,) + output if loss is not None else output
 
-        return MoeCausalLMOutputWithPast(
+        return MoEV2CausalLMOutputWithPast(
             loss=loss,
+            mtp_loss=all_mtp_loss,
             aux_loss=aux_loss,
             logits=logits,
+            mtp_logits=all_mtp_logits,
             past_key_values=outputs.past_key_values,
             hidden_states=outputs.hidden_states,
             attentions=outputs.attentions,
             router_logits=outputs.router_logits,
         )
-
-    def prepare_inputs_for_generation(
-        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, token_type_ids=None, **kwargs
-    ):
-        if past_key_values is not None:
-            if isinstance(past_key_values, Cache):
-                cache_length = past_key_values.get_seq_length()
-                past_length = past_key_values.seen_tokens
-                max_cache_length = (
-                    past_key_values.get_max_length()
-                    if hasattr(past_key_values, "get_max_length")
-                    else past_key_values.get_max_cache_shape()
-                )
-            else:
-                cache_length = past_length = past_key_values[0][0].shape[2]
-                max_cache_length = None
-
-            # Keep only the unprocessed tokens:
-            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
-            # some of the inputs are exclusivelly passed as part of the cache (e.g. when passing input_embeds as input)
-            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
-                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
-            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
-            # input_ids based on the past_length.
-            elif past_length < input_ids.shape[1]:
-                input_ids = input_ids[:, past_length:]
-            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
-
-            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
-            if (
-                max_cache_length is not None
-                and attention_mask is not None
-                and cache_length + input_ids.shape[1] > max_cache_length
-            ):
-                attention_mask = attention_mask[:, -max_cache_length:]
-
-        position_ids = kwargs.get("position_ids", None)
-        if attention_mask is not None and position_ids is None:
-            # create position_ids on the fly for batch generation
-            position_ids = attention_mask.long().cumsum(-1) - 1
-            position_ids.masked_fill_(attention_mask == 0, 1)
-            if past_key_values:
-                position_ids = position_ids[:, -input_ids.shape[1] :]
-
-        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
-        if inputs_embeds is not None and past_key_values is None:
-            model_inputs = {"inputs_embeds": inputs_embeds}
-        else:
-            model_inputs = {"input_ids": input_ids}
-
-        model_inputs.update(
-            {
-                "position_ids": position_ids,
-                "past_key_values": past_key_values,
-                "use_cache": kwargs.get("use_cache"),
-                "attention_mask": attention_mask,
-            }
-        )
-        return model_inputs
-
-    @staticmethod
-    def _reorder_cache(past_key_values, beam_idx):
-        reordered_past = ()
-        for layer_past in past_key_values:
-            reordered_past += (
-                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
-            )
-        return reordered_past
-
-