seamless_m4t_v2

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2

MindSpore SeamlessM4Tv2 model.

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Attention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2Attention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    # Copied from transformers.models.bart.modeling_bart.BartAttention.__init__ with Bart->SeamlessM4Tv2
    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[SeamlessM4Tv2Config] = None,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    def _shape(self, projection: mindspore.Tensor) -> mindspore.Tensor:
        new_projection_shape = projection.shape[:-1] + (self.num_heads, self.head_dim)
        # move heads to 2nd position (B, T, H * D) -> (B, T, H, D) -> (B, H, T, D)
        new_projection = projection.view(new_projection_shape).permute(0, 2, 1, 3)
        return new_projection

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        """Input shape: Batch x Time x Channel"""

        is_cross_attention = encoder_hidden_states is not None
        batch_size, seq_length = hidden_states.shape[:2]

        # use encoder_hidden_states if cross attention
        current_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
        # checking that the `sequence_length` of the `past_key_value` is the same as the he provided
        # `encoder_hidden_states` to support prefix tuning
        if is_cross_attention and past_key_value and past_key_value[0].shape[2] == current_states.shape[1]:
            # reuse k,v, cross_attentions
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        else:
            key_states = self._shape(self.k_proj(current_states))
            value_states = self._shape(self.v_proj(current_states))
            if past_key_value is not None and not is_cross_attention:
                # reuse k, v, self_attention
                key_states = ops.cat([past_key_value[0], key_states], dim=2)
                value_states = ops.cat([past_key_value[1], value_states], dim=2)

        query_states = self._shape(self.q_proj(hidden_states) * self.scaling)
        attention_scores = ops.matmul(query_states, key_states.swapaxes(-1, -2))

        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_states, value_states)

        if attention_mask is not None:
            attention_scores = attention_scores + attention_mask

        # (batch_size, n_heads, seq_length, key_length)
        attn_weights = nn.functional.softmax(attention_scores.float(), dim=-1).type_as(attention_scores)
        attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        #  attn_output = ops.bmm(attn_probs, value_states) ?
        context_states = ops.matmul(attn_weights, value_states)
        # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) ?
        context_states = context_states.permute(0, 2, 1, 3).view(batch_size, seq_length, -1)
        attn_output = self.out_proj(context_states)

        if output_attentions:
            return attn_output, attn_weights, past_key_value
        else:
            return attn_output, None, past_key_value

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Attention.forward(hidden_states, encoder_hidden_states=None, past_key_value=None, attention_mask=None, output_attentions=False)

Input shape: Batch x Time x Channel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    output_attentions: bool = False,
) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
    """Input shape: Batch x Time x Channel"""

    is_cross_attention = encoder_hidden_states is not None
    batch_size, seq_length = hidden_states.shape[:2]

    # use encoder_hidden_states if cross attention
    current_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
    # checking that the `sequence_length` of the `past_key_value` is the same as the he provided
    # `encoder_hidden_states` to support prefix tuning
    if is_cross_attention and past_key_value and past_key_value[0].shape[2] == current_states.shape[1]:
        # reuse k,v, cross_attentions
        key_states = past_key_value[0]
        value_states = past_key_value[1]
    else:
        key_states = self._shape(self.k_proj(current_states))
        value_states = self._shape(self.v_proj(current_states))
        if past_key_value is not None and not is_cross_attention:
            # reuse k, v, self_attention
            key_states = ops.cat([past_key_value[0], key_states], dim=2)
            value_states = ops.cat([past_key_value[1], value_states], dim=2)

    query_states = self._shape(self.q_proj(hidden_states) * self.scaling)
    attention_scores = ops.matmul(query_states, key_states.swapaxes(-1, -2))

    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_states, value_states)

    if attention_mask is not None:
        attention_scores = attention_scores + attention_mask

    # (batch_size, n_heads, seq_length, key_length)
    attn_weights = nn.functional.softmax(attention_scores.float(), dim=-1).type_as(attention_scores)
    attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

    #  attn_output = ops.bmm(attn_probs, value_states) ?
    context_states = ops.matmul(attn_weights, value_states)
    # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) ?
    context_states = context_states.permute(0, 2, 1, 3).view(batch_size, seq_length, -1)
    attn_output = self.out_proj(context_states)

    if output_attentions:
        return attn_output, attn_weights, past_key_value
    else:
        return attn_output, None, past_key_value

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2CodeHifiGan

Bases: PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2CodeHifiGan(PreTrainedModel):
    config_class = SeamlessM4Tv2Config
    main_input_name = "input_embeds"
    _no_split_modules = []

    def __init__(self, config):
        super().__init__(config)

        self.pad_token_id = config.t2u_pad_token_id
        embed_dim = config.unit_embed_dim
        kernel_size = config.variance_predictor_kernel_size
        var_pred_dropout = config.var_pred_dropout
        self.dur_predictor = SeamlessM4Tv2VariancePredictor(embed_dim, embed_dim, kernel_size, var_pred_dropout)

        self.unit_embedding = nn.Embedding(config.unit_hifi_gan_vocab_size, config.unit_embed_dim)
        self.speaker_embedding = nn.Embedding(config.vocoder_num_spkrs, config.spkr_embed_dim)
        self.language_embedding = nn.Embedding(config.vocoder_num_langs, config.lang_embed_dim)

        self.hifi_gan = SeamlessM4Tv2HifiGan(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan._get_dur_output_lengths
    def _get_dur_output_lengths(self, input_ids, dur_out):
        """
        Computes the output length after the duration layer.
        """
        unit_lengths = (input_ids != self.pad_token_id).sum(1)

        # take care of edge cases where no padding or too many padding
        unit_lengths = ops.clamp(unit_lengths, 0, dur_out.shape[1] - 1)

        cumulative_dur_out = ops.cumsum(dur_out, dim=1)
        unit_lengths = ops.gather(cumulative_dur_out, dim=1, index=unit_lengths.unsqueeze(1)).squeeze()

        return unit_lengths

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan._get_output_hifigan_lengths
    def _get_output_hifigan_lengths(self, input_lengths: Union[mindspore.Tensor, int]):
        """
        Computes the output length of the hifigan convolutional layers
        """

        def _conv_out_length(input_length, kernel_size, stride, pad, dilation=1):
            # 1D convolutional layer output length formula taken
            return (
                ops.div(input_length + 2 * pad - dilation * (kernel_size - 1) - 1, stride, rounding_mode="floor") + 1
            )

        def _transpose_conv_out_length(input_length, kernel_size, stride, pad, dilation=1):
            return (input_length - 1) * stride - 2 * pad + dilation * (kernel_size - 1) + 1

        # conv_pre
        input_lengths = _conv_out_length(input_lengths, 7, 1, 3)

        # upsampler
        for i, (upsample_rate, kernel_size) in enumerate(
            zip(self.config.upsample_rates, self.config.upsample_kernel_sizes)
        ):
            input_lengths = _transpose_conv_out_length(
                input_lengths, kernel_size, upsample_rate, (kernel_size - upsample_rate) // 2
            )

        # resblock
        for i in range(len(self.config.upsample_rates)):
            for kernel_size, dilation in zip(self.config.resblock_kernel_sizes, self.config.resblock_dilation_sizes):
                for dil in dilation:
                    input_lengths = _conv_out_length(
                        input_lengths, kernel_size, 1, (kernel_size - 1) * dil // 2, dilation=dil
                    )

                for dil in dilation:
                    input_lengths = _conv_out_length(input_lengths, kernel_size, 1, (kernel_size - 1) // 2, dilation=1)

        # conv_post
        input_lengths = _conv_out_length(input_lengths, 7, 1, 3)

        return input_lengths

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan.forward with SeamlessM4T->SeamlessM4Tv2, spkr_id->speaker_id
    def forward(
        self, input_ids: mindspore.Tensor, speaker_id: mindspore.Tensor, lang_id: mindspore.Tensor
    ) -> Tuple[mindspore.Tensor]:
        """
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4Tv2TextToUnitForConditionalGeneration`]. [What are input
                IDs?](../glossary#input-ids)
            speaker_id (`int`, *optional*):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            tgt_lang (`str`, *optional*):
                The language id to use as target language for translation.
        """
        hidden_states = self.unit_embedding(input_ids).swapaxes(1, 2)
        spkr = self.speaker_embedding(speaker_id).swapaxes(1, 2)
        lang = self.language_embedding(lang_id).swapaxes(1, 2)

        log_dur_pred = self.dur_predictor(hidden_states.swapaxes(1, 2))
        dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
        # B x C x T
        if hidden_states.shape[0] == 1:
            hidden_states = ops.repeat_interleave(hidden_states, dur_out.view(-1), dim=2)
        else:
            # if batched sample, need to interleave per sample, and pad -> loss of parallelism
            if hidden_states.shape[0] > 1 and self.training:
                logger.warning(
                    """`self.training=True` and you use batching. You lose parallelism during the hifigan
                               forward pass because the samples are interleaved."""
                )
            hidden_states = [
                ops.repeat_interleave(hidden_state, duration.tolist(), dim=-1).swapaxes(0, 1)
                for (hidden_state, duration) in zip(hidden_states, dur_out)
            ]

            hidden_states = pad_sequence(hidden_states, batch_first=True).swapaxes(1, 2)

        spkr = spkr.tile((1, 1, hidden_states.shape[-1]))
        lang = lang.tile((1, 1, hidden_states.shape[-1]))
        hidden_states = ops.cat([lang, hidden_states, spkr], dim=1)

        hidden_states = self.hifi_gan(hidden_states)

        unit_lengths = self._get_dur_output_lengths(input_ids, dur_out)
        lengths = self._get_output_hifigan_lengths(unit_lengths)

        return hidden_states, lengths

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan._init_weights
    def _init_weights(self, module):
        """Initialize the weights."""
        if isinstance(module, (nn.Linear, nn.Conv1d, nn.ConvTranspose1d)):
            nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
            if module.padding_idx is not None:
                module.weight[module.padding_idx] = 0

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan.apply_weight_norm
    def apply_weight_norm(self):
        nn.utils.weight_norm(self.hifi_gan.conv_pre)
        for layer in self.hifi_gan.upsampler:
            nn.utils.weight_norm(layer)
        for layer in self.hifi_gan.resblocks:
            layer.apply_weight_norm()
        nn.utils.weight_norm(self.hifi_gan.conv_post)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan.remove_weight_norm
    def remove_weight_norm(self):
        nn.utils.remove_weight_norm(self.hifi_gan.conv_pre)
        for layer in self.hifi_gan.upsampler:
            nn.utils.remove_weight_norm(layer)
        for layer in self.hifi_gan.resblocks:
            layer.remove_weight_norm()
        nn.utils.remove_weight_norm(self.hifi_gan.conv_post)

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2CodeHifiGan.forward(input_ids, speaker_id, lang_id)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4Tv2TextToUnitForConditionalGeneration]. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`
speaker_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*
tgt_lang	The language id to use as target language for translation. TYPE: `str`, *optional*

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self, input_ids: mindspore.Tensor, speaker_id: mindspore.Tensor, lang_id: mindspore.Tensor
) -> Tuple[mindspore.Tensor]:
    """
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4Tv2TextToUnitForConditionalGeneration`]. [What are input
            IDs?](../glossary#input-ids)
        speaker_id (`int`, *optional*):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        tgt_lang (`str`, *optional*):
            The language id to use as target language for translation.
    """
    hidden_states = self.unit_embedding(input_ids).swapaxes(1, 2)
    spkr = self.speaker_embedding(speaker_id).swapaxes(1, 2)
    lang = self.language_embedding(lang_id).swapaxes(1, 2)

    log_dur_pred = self.dur_predictor(hidden_states.swapaxes(1, 2))
    dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
    # B x C x T
    if hidden_states.shape[0] == 1:
        hidden_states = ops.repeat_interleave(hidden_states, dur_out.view(-1), dim=2)
    else:
        # if batched sample, need to interleave per sample, and pad -> loss of parallelism
        if hidden_states.shape[0] > 1 and self.training:
            logger.warning(
                """`self.training=True` and you use batching. You lose parallelism during the hifigan
                           forward pass because the samples are interleaved."""
            )
        hidden_states = [
            ops.repeat_interleave(hidden_state, duration.tolist(), dim=-1).swapaxes(0, 1)
            for (hidden_state, duration) in zip(hidden_states, dur_out)
        ]

        hidden_states = pad_sequence(hidden_states, batch_first=True).swapaxes(1, 2)

    spkr = spkr.tile((1, 1, hidden_states.shape[-1]))
    lang = lang.tile((1, 1, hidden_states.shape[-1]))
    hidden_states = ops.cat([lang, hidden_states, spkr], dim=1)

    hidden_states = self.hifi_gan(hidden_states)

    unit_lengths = self._get_dur_output_lengths(input_ids, dur_out)
    lengths = self._get_output_hifigan_lengths(unit_lengths)

    return hidden_states, lengths

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ConformerConvolutionModule

Bases: Module

Convolution block used in the conformer block. Uses a causal depthwise convolution similar to that described in Section 2.1 of `https://doi.org/10.48550/arxiv.1609.03499

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ConformerConvolutionModule(nn.Module):
    """Convolution block used in the conformer block. Uses a causal depthwise convolution similar to that
    described in Section 2.1 of `https://doi.org/10.48550/arxiv.1609.03499"""

    def __init__(self, config):
        super().__init__()
        if (config.conv_depthwise_kernel_size - 1) % 2 == 1:
            raise ValueError("`config.conv_depthwise_kernel_size` should be a odd number for 'SAME' padding")
        self.layer_norm = nn.LayerNorm(config.hidden_size)
        self.pointwise_conv1 = nn.Conv1d(
            config.hidden_size,
            2 * config.hidden_size,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False,
        )
        self.glu = nn.GLU(dim=1)
        self.depthwise_conv = nn.Conv1d(
            config.hidden_size,
            config.hidden_size,
            config.conv_depthwise_kernel_size,
            stride=1,
            padding=0,
            groups=config.hidden_size,
            bias=False,
        )
        self.depthwise_layer_norm = nn.LayerNorm(config.hidden_size)
        self.activation = ACT2FN[config.speech_encoder_hidden_act]
        self.pointwise_conv2 = nn.Conv1d(
            config.hidden_size,
            config.hidden_size,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False,
        )
        self.dropout = nn.Dropout(config.speech_encoder_dropout)

    def forward(self, hidden_states, attention_mask=None):
        hidden_states = self.layer_norm(hidden_states)

        # Ensure that we do not leak padded positions in depthwise convolution.
        # Put 0 where necessary
        if attention_mask is not None:
            hidden_states = hidden_states.masked_fill(~attention_mask.bool().unsqueeze(-1), 0.0)

        # exchange the temporal dimension and the feature dimension
        hidden_states = hidden_states.swapaxes(1, 2)

        # GLU mechanism
        # => (batch, 2*channel, dim)
        hidden_states = self.pointwise_conv1(hidden_states)
        # => (batch, channel, dim)
        hidden_states = self.glu(hidden_states)

        # Pad the sequence entirely on the left because of causal convolution.
        hidden_states = nn.functional.pad(hidden_states, (self.depthwise_conv.kernel_size[0] - 1, 0))

        # 1D Depthwise Conv
        hidden_states = self.depthwise_conv(hidden_states)
        hidden_states = self.depthwise_layer_norm(hidden_states.swapaxes(1, 2)).swapaxes(1, 2)
        hidden_states = self.activation(hidden_states)

        hidden_states = self.pointwise_conv2(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = hidden_states.swapaxes(1, 2)
        return hidden_states

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ConformerEncoder

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ConformerEncoder(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config

        self.dropout = nn.Dropout(config.speech_encoder_dropout)
        self.layers = nn.ModuleList(
            [SeamlessM4Tv2ConformerEncoderLayer(config) for _ in range(config.speech_encoder_layers)]
        )

        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

        self.gradient_checkpointing = False

    def _apply_chunk_attention(self, attention_mask, hidden_states):
        """
        Creates a chunk attention mask. It creates a mask to prevent attention across chunks, ensuring that each
        position attends only to positions within its own chunk. If a left chunk overlap is specified
        (`speech_encoder_chunk_size` in the configuration), the attention mask is adjusted accordingly to allow each
        position to also attends the `speech_encoder_chunk_size - 1` previous chunks.
        """
        sequence_len = hidden_states.shape[1]

        chunk_indices = ops.arange(sequence_len)
        chunk_indices = ops.div(chunk_indices, self.config.speech_encoder_chunk_size).long()

        start_indices = ops.full_like(chunk_indices, 0)
        if self.config.speech_encoder_left_chunk_num >= 0:
            start_indices = (chunk_indices - self.config.speech_encoder_left_chunk_num).clamp(min=0)
            start_indices = start_indices * self.config.speech_encoder_chunk_size
        start_indices = start_indices.unsqueeze(1).broadcast_to((-1, sequence_len))

        end_indices = ((chunk_indices + 1) * self.config.speech_encoder_chunk_size).clamp(max=sequence_len)

        end_indices = end_indices.unsqueeze(1).broadcast_to((-1, sequence_len))

        indices = ops.arange(sequence_len).unsqueeze(0).broadcast_to((sequence_len, -1))

        chunk_mask = (indices < start_indices).int() | (indices >= end_indices).int()
        chunk_mask = chunk_mask.unsqueeze(0).unsqueeze(0)

        attention_mask = chunk_mask if attention_mask is None else (attention_mask.int() | chunk_mask)
        attention_mask = attention_mask.to(dtype=hidden_states.dtype)
        return attention_mask

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        output_attentions=False,
        output_hidden_states=False,
        return_dict=True,
    ):
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None

        conv_attention_mask = attention_mask
        if attention_mask is not None:
            # make sure padded tokens output 0
            hidden_states = hidden_states.masked_fill(~attention_mask.bool().unsqueeze(-1), 0.0)
            # extend attention_mask
            attention_mask = 1.0 - attention_mask[:, None, None, :].to(dtype=hidden_states.dtype)
            attention_mask = attention_mask.broadcast_to(
                (attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
            )

        if self.config.speech_encoder_chunk_size is not None:
            attention_mask = self._apply_chunk_attention(attention_mask, hidden_states)

        if attention_mask is not None:
            attention_mask = attention_mask * float(ops.finfo(hidden_states.dtype).min)

        hidden_states = self.dropout(hidden_states)

        for i, layer in enumerate(self.layers):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            dropout_probability = ops.rand([])

            skip_the_layer = self.training and (dropout_probability < self.config.speech_encoder_layerdrop)
            if not skip_the_layer:
                # under deepspeed zero3 all gpus must run in sync
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        layer.__call__,
                        hidden_states,
                        attention_mask,
                        output_attentions,
                        conv_attention_mask,
                    )
                else:
                    layer_outputs = layer(
                        hidden_states,
                        attention_mask=attention_mask,
                        output_attentions=output_attentions,
                        conv_attention_mask=conv_attention_mask,
                    )
                hidden_states = layer_outputs[0]

            if skip_the_layer:
                layer_outputs = (None, None)

            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
        )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ConformerEncoderLayer

Bases: Module

Conformer block based on https://arxiv.org/abs/2005.08100.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ConformerEncoderLayer(nn.Module):
    """Conformer block based on https://arxiv.org/abs/2005.08100."""

    # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerEncoderLayer.__init__ with Wav2Vec2->SeamlessM4Tv2, attention_dropout->speech_encoder_dropout, nn->nn
    def __init__(self, config):
        super().__init__()
        embed_dim = config.hidden_size
        dropout = config.speech_encoder_dropout

        # Feed-forward 1
        self.ffn1_layer_norm = nn.LayerNorm(embed_dim)
        self.ffn1 = SeamlessM4Tv2ConformerFeedForward(config)

        # Self-Attention
        self.self_attn_layer_norm = nn.LayerNorm(embed_dim)
        self.self_attn_dropout = nn.Dropout(dropout)
        self.self_attn = SeamlessM4Tv2ConformerSelfAttention(config)

        # Conformer Convolution
        self.conv_module = SeamlessM4Tv2ConformerConvolutionModule(config)

        # Feed-forward 2
        self.ffn2_layer_norm = nn.LayerNorm(embed_dim)
        self.ffn2 = SeamlessM4Tv2ConformerFeedForward(config)
        self.final_layer_norm = nn.LayerNorm(embed_dim)

    def forward(
        self,
        hidden_states,
        attention_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
        conv_attention_mask: Optional[mindspore.Tensor] = None,
    ):
        # 1. Feed-Forward 1 layer
        residual = hidden_states
        hidden_states = self.ffn1_layer_norm(hidden_states)
        hidden_states = self.ffn1(hidden_states)
        hidden_states = hidden_states * 0.5 + residual
        residual = hidden_states

        # 2. Self-Attention layer
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.self_attn_dropout(hidden_states)
        hidden_states = hidden_states + residual

        # 3. Convolutional Layer
        residual = hidden_states
        hidden_states = self.conv_module(hidden_states, attention_mask=conv_attention_mask)
        hidden_states = residual + hidden_states

        # 4. Feed-Forward 2 Layer
        residual = hidden_states
        hidden_states = self.ffn2_layer_norm(hidden_states)
        hidden_states = self.ffn2(hidden_states)
        hidden_states = hidden_states * 0.5 + residual
        hidden_states = self.final_layer_norm(hidden_states)

        return hidden_states, attn_weights

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ConformerSelfAttention

Bases: Module

Construct a SeamlessM4Tv2ConformerSelfAttention object. Can be enhanced with relative position embeddings.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ConformerSelfAttention(nn.Module):
    """Construct a SeamlessM4Tv2ConformerSelfAttention object.
    Can be enhanced with relative position embeddings.
    """

    def __init__(self, config, use_position_embeddings=True):
        super().__init__()

        self.head_size = config.hidden_size // config.speech_encoder_attention_heads
        self.num_heads = config.speech_encoder_attention_heads
        self.position_embeddings_type = config.position_embeddings_type if use_position_embeddings else None

        self.linear_q = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_k = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_v = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_out = nn.Linear(config.hidden_size, config.hidden_size)

        self.dropout = nn.Dropout(p=config.speech_encoder_dropout)

        if self.position_embeddings_type == "relative_key":
            self.left_max_position_embeddings = config.left_max_position_embeddings
            self.right_max_position_embeddings = config.right_max_position_embeddings
            num_positions = self.left_max_position_embeddings + self.right_max_position_embeddings + 1
            self.distance_embedding = nn.Embedding(num_positions, self.head_size)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        # self-attention mechanism
        batch_size, sequence_length, hidden_size = hidden_states.shape

        # make sure query/key states can be != value states
        query_key_states = hidden_states
        value_states = hidden_states

        # project query_key_states and value_states
        query = self.linear_q(query_key_states).view(batch_size, -1, self.num_heads, self.head_size)
        key = self.linear_k(query_key_states).view(batch_size, -1, self.num_heads, self.head_size)
        value = self.linear_v(value_states).view(batch_size, -1, self.num_heads, self.head_size)

        # => (batch, head, time1, d_k)
        query = query.swapaxes(1, 2)
        key = key.swapaxes(1, 2)
        value = value.swapaxes(1, 2)

        attn_weights = ops.matmul(query, key.swapaxes(-2, -1)) / math.sqrt(self.head_size)

        if self.position_embeddings_type == "relative_key":
            query_length, key_length = query.shape[2], key.shape[2]

            position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
            position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
            distance = position_ids_r - position_ids_l
            distance = ops.clamp(distance, -self.left_max_position_embeddings, self.right_max_position_embeddings)

            positional_embedding = self.distance_embedding(distance + self.left_max_position_embeddings)
            positional_embedding = positional_embedding.to(dtype=query.dtype)  # fp16 compatibility

            relative_position_attn_weights = ops.einsum("bhld,lrd->bhlr", query, positional_embedding)
            attn_weights = attn_weights + (relative_position_attn_weights / math.sqrt(self.head_size))

        # apply attention_mask if necessary
        if attention_mask is not None:
            attn_weights = attn_weights + attention_mask

        # => (batch, head, time1, time2)
        attn_weights = ops.softmax(attn_weights, dim=-1)
        attn_weights = self.dropout(attn_weights)

        # => (batch, head, time1, d_k)
        attn_output = ops.matmul(attn_weights, value)

        # => (batch, time1, hidden_size)
        attn_output = attn_output.swapaxes(1, 2).reshape(batch_size, -1, self.num_heads * self.head_size)
        attn_output = self.linear_out(attn_output)

        if not output_attentions:
            attn_weights = None

        return attn_output, attn_weights

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Decoder

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel):
    def __init__(
        self,
        config: SeamlessM4Tv2Config,
        embed_tokens: Optional[nn.Embedding] = None,
    ):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size
        self.max_target_positions = config.max_position_embeddings
        embed_scale = math.sqrt(config.hidden_size) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            # if embed_tokens defined, use its shape instead
            self.embed_tokens = SeamlessM4Tv2ScaledWordEmbedding(
                embed_tokens.num_embeddings, embed_tokens.embedding_dim, self.padding_idx, embed_scale=embed_scale
            )
            self.embed_tokens.weight = embed_tokens.weight
        else:
            self.embed_tokens = SeamlessM4Tv2ScaledWordEmbedding(
                self.vocab_size, config.hidden_size, self.padding_idx, embed_scale=embed_scale
            )

        self.embed_positions = SeamlessM4Tv2SinusoidalPositionalEmbedding(
            self.max_target_positions,
            config.hidden_size,
            padding_idx=self.padding_idx,
        )

        layers = []
        for _ in range(config.decoder_layers):
            layers.append(
                SeamlessM4Tv2DecoderLayer(
                    config,
                    decoder_attention_heads=config.decoder_attention_heads,
                    decoder_ffn_dim=config.decoder_ffn_dim,
                )
            )
        self.layers = nn.ModuleList(layers)
        self.layer_norm = nn.LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                of the decoder.
            encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
                Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
                selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
                that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
                all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input = input_ids
            input_shape = input.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
            input = inputs_embeds[:, :, -1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

        # expand encoder attention mask
        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
            encoder_attention_mask = _prepare_4d_attention_mask(
                encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
            )

        # embed positions
        positions = self.embed_positions(input, past_key_values_length=past_key_values_length)

        hidden_states = inputs_embeds + positions

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`..."
                )
                use_cache = False

        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = ops.rand([])
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    attention_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    None,
                    output_attentions,
                    use_cache,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=attention_mask,
                    encoder_hidden_states=encoder_hidden_states,
                    encoder_attention_mask=encoder_attention_mask,
                    past_key_value=past_key_value,
                    output_attentions=output_attentions,
                    use_cache=use_cache,
                )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[1],)

            if output_attentions:
                all_self_attns += (layer_outputs[2],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[3],)

        hidden_states = self.layer_norm(hidden_states)

        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Decoder.forward(input_ids=None, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional* DEFAULT: None
encoder_attention_mask	Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional* DEFAULT: None
past_key_values	Tuple of tuple(mindspore.Tensor) of length config.n_layers, with each tuple having 2 tensors of shape (batch_size, num_heads, sequence_length, embed_size_per_head)) and 2 additional tensors of shape (batch_size, num_heads, encoder_sequence_length, embed_size_per_head). Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see past_key_values input) to speed up sequential decoding. If past_key_values are used, the user can optionally input only the last decoder_input_ids (those that don't have their past key value states given to this model) of shape (batch_size, 1) instead of all decoder_input_ids of shape (batch_size, sequence_length). TYPE: `tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True` DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
            Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
            selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
            cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
            that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
            all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
    elif input_ids is not None:
        input = input_ids
        input_shape = input.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
        input = inputs_embeds[:, :, -1]
    else:
        raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    attention_mask = _prepare_4d_causal_attention_mask(
        attention_mask, input_shape, inputs_embeds, past_key_values_length
    )

    # expand encoder attention mask
    if encoder_hidden_states is not None and encoder_attention_mask is not None:
        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
        encoder_attention_mask = _prepare_4d_attention_mask(
            encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
        )

    # embed positions
    positions = self.embed_positions(input, past_key_values_length=past_key_values_length)

    hidden_states = inputs_embeds + positions

    hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`..."
            )
            use_cache = False

    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None
    all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
    next_decoder_cache = () if use_cache else None

    for idx, decoder_layer in enumerate(self.layers):
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        if output_hidden_states:
            all_hidden_states += (hidden_states,)
        if self.training:
            dropout_probability = ops.rand([])
            if dropout_probability < self.layerdrop:
                continue

        past_key_value = past_key_values[idx] if past_key_values is not None else None

        if self.gradient_checkpointing and self.training:
            layer_outputs = self._gradient_checkpointing_func(
                decoder_layer.__call__,
                hidden_states,
                attention_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                None,
                output_attentions,
                use_cache,
            )
        else:
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                encoder_attention_mask=encoder_attention_mask,
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )
        hidden_states = layer_outputs[0]

        if use_cache:
            next_decoder_cache += (layer_outputs[1],)

        if output_attentions:
            all_self_attns += (layer_outputs[2],)

            if encoder_hidden_states is not None:
                all_cross_attentions += (layer_outputs[3],)

    hidden_states = self.layer_norm(hidden_states)

    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    next_cache = next_decoder_cache if use_cache else None
    if not return_dict:
        return tuple(
            v
            for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2DecoderLayer

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2DecoderLayer(nn.Module):
    def __init__(self, config: SeamlessM4Tv2Config, decoder_ffn_dim=None, decoder_attention_heads=None):
        super().__init__()
        decoder_ffn_dim = config.decoder_ffn_dim if decoder_ffn_dim is None else decoder_ffn_dim
        decoder_attention_heads = (
            config.decoder_attention_heads if decoder_attention_heads is None else decoder_attention_heads
        )

        self.embed_dim = config.hidden_size
        self.self_attn = SeamlessM4Tv2Attention(
            embed_dim=self.embed_dim,
            num_heads=decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.attn_dropout = nn.Dropout(config.dropout)

        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.cross_attention = SeamlessM4Tv2Attention(
            self.embed_dim, decoder_attention_heads, config.attention_dropout, is_decoder=True
        )
        self.cross_attention_layer_norm = nn.LayerNorm(self.embed_dim)

        self.ffn = SeamlessM4Tv2FeedForwardNetwork(config, ffn_dim=decoder_ffn_dim)

        self.ffn_layer_norm = nn.LayerNorm(config.hidden_size)
        self.ffn_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`):
                attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
                large negative values.
            encoder_hidden_states (`mindspore.Tensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`mindspore.Tensor`):
                encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by
                very large negative values.
            past_key_value (`Tuple(mindspore.Tensor)`):
                cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        # Self Attention
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        # add present self-attn cache to positions 1,2 of present_key_value tuple
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        # Cross-Attention Block
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.cross_attention_layer_norm(hidden_states)

            # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None

            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention(
                hidden_states=hidden_states,
                encoder_hidden_states=encoder_hidden_states,
                past_key_value=cross_attn_past_key_value,
                attention_mask=encoder_attention_mask,
                output_attentions=output_attentions,
            )
            hidden_states = self.attn_dropout(hidden_states)
            hidden_states = residual + hidden_states

            # add cross-attn to positions 3,4 of present_key_value tuple
            present_key_value += cross_attn_present_key_value

        # Fully Connected
        residual = hidden_states

        hidden_states = self.ffn_layer_norm(hidden_states)

        hidden_states = self.ffn(hidden_states)
        hidden_states = self.ffn_dropout(hidden_states)

        hidden_states = residual + hidden_states

        outputs = (hidden_states, present_key_value)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2DecoderLayer.forward(hidden_states, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False, use_cache=True)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
encoder_hidden_states	cross attention input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor` DEFAULT: None
encoder_attention_mask	encoder attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
past_key_value	cached past key and value projection states TYPE: `Tuple(mindspore.Tensor)` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: Optional[bool] = False,
    use_cache: Optional[bool] = True,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`):
            input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`):
            attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
            large negative values.
        encoder_hidden_states (`mindspore.Tensor`):
            cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
        encoder_attention_mask (`mindspore.Tensor`):
            encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by
            very large negative values.
        past_key_value (`Tuple(mindspore.Tensor)`):
            cached past key and value projection states
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)

    # Self Attention
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    # add present self-attn cache to positions 1,2 of present_key_value tuple
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        past_key_value=self_attn_past_key_value,
        attention_mask=attention_mask,
        output_attentions=output_attentions,
    )
    hidden_states = self.attn_dropout(hidden_states)
    hidden_states = residual + hidden_states

    # Cross-Attention Block
    cross_attn_present_key_value = None
    cross_attn_weights = None
    if encoder_hidden_states is not None:
        residual = hidden_states
        hidden_states = self.cross_attention_layer_norm(hidden_states)

        # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None

        hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention(
            hidden_states=hidden_states,
            encoder_hidden_states=encoder_hidden_states,
            past_key_value=cross_attn_past_key_value,
            attention_mask=encoder_attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        # add cross-attn to positions 3,4 of present_key_value tuple
        present_key_value += cross_attn_present_key_value

    # Fully Connected
    residual = hidden_states

    hidden_states = self.ffn_layer_norm(hidden_states)

    hidden_states = self.ffn(hidden_states)
    hidden_states = self.ffn_dropout(hidden_states)

    hidden_states = residual + hidden_states

    outputs = (hidden_states, present_key_value)

    if output_attentions:
        outputs += (self_attn_weights, cross_attn_weights)

    return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Encoder

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2Encoder(SeamlessM4Tv2PreTrainedModel):
    def __init__(
        self,
        config: SeamlessM4Tv2Config,
        embed_tokens: Optional[nn.Embedding] = None,
        is_t2u_encoder: bool = False,
    ):
        super().__init__(config)

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop
        self.padding_idx = config.pad_token_id
        embed_dim = config.hidden_size

        self.is_t2u_encoder = is_t2u_encoder
        self.max_source_positions = config.max_position_embeddings

        if not self.is_t2u_encoder:
            embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

            self.embed_tokens = SeamlessM4Tv2ScaledWordEmbedding(
                config.vocab_size, embed_dim, self.padding_idx, embed_scale=embed_scale
            )

            if embed_tokens is not None:
                self.embed_tokens.weight = embed_tokens.weight

            self.embed_positions = SeamlessM4Tv2SinusoidalPositionalEmbedding(
                self.max_source_positions,
                embed_dim,
                self.padding_idx,
            )

        layers = []
        for _ in range(config.encoder_layers):
            layers.append(
                SeamlessM4Tv2EncoderLayer(
                    config,
                    encoder_attention_heads=config.encoder_attention_heads,
                    encoder_ffn_dim=config.encoder_ffn_dim,
                )
            )

        self.layers = nn.ModuleList(layers)

        self.layer_norm = nn.LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Tuple, BaseModelOutput]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is not None and self.is_t2u_encoder:
            raise ValueError(
                "You cannot pass input_ids to the encoder of the text_to_units model. Pass inputs_embeds instead."
            )

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            input = input_ids
            input_shape = input.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input = inputs_embeds[:, :, -1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        if not self.is_t2u_encoder:
            embed_pos = self.embed_positions(input)

            hidden_states = inputs_embeds + embed_pos
        else:
            hidden_states = inputs_embeds

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        # expand attention_mask
        if attention_mask is not None:
            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
            attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            to_drop = False
            if self.training:
                dropout_probability = ops.rand([])
                if dropout_probability < self.layerdrop:  # skip the layer
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.forward,
                        hidden_states,
                        attention_mask,
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        attention_mask,
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Encoder.forward(input_ids=None, attention_mask=None, inputs_embeds=None, output_attentions=None, output_hidden_states=None, return_dict=None, **kwargs)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
    **kwargs,
) -> Union[Tuple, BaseModelOutput]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if input_ids is not None and self.is_t2u_encoder:
        raise ValueError(
            "You cannot pass input_ids to the encoder of the text_to_units model. Pass inputs_embeds instead."
        )

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    elif input_ids is not None:
        input = input_ids
        input_shape = input.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input = inputs_embeds[:, :, -1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    if not self.is_t2u_encoder:
        embed_pos = self.embed_positions(input)

        hidden_states = inputs_embeds + embed_pos
    else:
        hidden_states = inputs_embeds

    hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

    # expand attention_mask
    if attention_mask is not None:
        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
        attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

    encoder_states = () if output_hidden_states else None
    all_attentions = () if output_attentions else None

    for idx, encoder_layer in enumerate(self.layers):
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        to_drop = False
        if self.training:
            dropout_probability = ops.rand([])
            if dropout_probability < self.layerdrop:  # skip the layer
                to_drop = True

        if to_drop:
            layer_outputs = (None, None)
        else:
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    encoder_layer.forward,
                    hidden_states,
                    attention_mask,
                    output_attentions,
                )
            else:
                layer_outputs = encoder_layer(
                    hidden_states,
                    attention_mask,
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

        if output_attentions:
            all_attentions = all_attentions + (layer_outputs[1],)

    hidden_states = self.layer_norm(hidden_states)

    if output_hidden_states:
        encoder_states = encoder_states + (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
    return BaseModelOutput(
        last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
    )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2EncoderLayer

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2EncoderLayer(nn.Module):
    def __init__(self, config: SeamlessM4Tv2Config, encoder_ffn_dim=None, encoder_attention_heads=None):
        super().__init__()
        encoder_ffn_dim = config.encoder_ffn_dim if encoder_ffn_dim is None else encoder_ffn_dim
        encoder_attention_heads = (
            config.encoder_attention_heads if encoder_attention_heads is None else encoder_attention_heads
        )

        self.embed_dim = config.hidden_size
        self.self_attn = SeamlessM4Tv2Attention(
            embed_dim=self.embed_dim,
            num_heads=encoder_attention_heads,
            dropout=config.attention_dropout,
        )
        self.attn_dropout = nn.Dropout(config.dropout)
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)

        self.ffn = SeamlessM4Tv2FeedForwardNetwork(config, ffn_dim=encoder_ffn_dim)

        self.ffn_layer_norm = nn.LayerNorm(config.hidden_size)
        self.ffn_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: mindspore.Tensor,
        output_attentions: bool = False,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`):
                attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
                large negative values.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        residual = hidden_states

        hidden_states = self.ffn_layer_norm(hidden_states)

        hidden_states = self.ffn(hidden_states)
        hidden_states = self.ffn_dropout(hidden_states)

        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2EncoderLayer.forward(hidden_states, attention_mask, output_attentions=False)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor`

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: mindspore.Tensor,
    output_attentions: bool = False,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`):
            input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`):
            attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
            large negative values.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)
    hidden_states, attn_weights, _ = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        output_attentions=output_attentions,
    )
    hidden_states = self.attn_dropout(hidden_states)
    hidden_states = residual + hidden_states

    residual = hidden_states

    hidden_states = self.ffn_layer_norm(hidden_states)

    hidden_states = self.ffn(hidden_states)
    hidden_states = self.ffn_dropout(hidden_states)

    hidden_states = residual + hidden_states

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (attn_weights,)

    return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForSpeechToSpeech

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ForSpeechToSpeech(SeamlessM4Tv2PreTrainedModel):
    _keys_to_ignore_on_load_missing = ["text_encoder"]
    main_input_name = "input_features"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_decoder.embed_tokens.weight",
    ]

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.__init__ with SeamlessM4T->SeamlessM4Tv2
    def __init__(self, config):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
        self.speech_encoder = SeamlessM4Tv2SpeechEncoder(config)
        self.text_decoder = SeamlessM4Tv2Decoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.t2u_model = SeamlessM4Tv2TextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4Tv2CodeHifiGan(config)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.get_encoder
    def get_encoder(self):
        return self.speech_encoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.get_decoder
    def get_decoder(self):
        return self.text_decoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.get_output_embeddings
    def get_output_embeddings(self):
        return self.lm_head

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.get_input_embeddings
    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.set_input_embeddings
    def set_input_embeddings(self, value):
        self.text_decoder.embed_tokens = value

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech._tie_weights
    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.forward with SeamlessM4T->SeamlessM4Tv2
    def forward(
        self,
        input_features: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This is the same forward method as `SeamlessM4Tv2ForSpeechToText`. It doesn't use `self.t2u_model`."
                "If you want to generate speech, use the `generate` method."
            )

            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        speaker_id: Optional[int] = 0,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
        """
        Generates translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_features, num_beams=4, speech_do_sample=True)` will successively perform
        beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Args:
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            speaker_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.

            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.


        Returns:
            `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]`:
            - If `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
            - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
              sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        """
        batch_size = len(input_features) if input_features is not None else len(kwargs.get("inputs_embeds"))

        if tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
        else:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                    more languages for text translation than for speech synthesis."""
                    )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        text_generation_output = super().generate(input_features, **kwargs_text)
        sequences = text_generation_output.sequences

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        # get last_hidden_state from encoder
        encoder_hidden_states = self.speech_encoder(input_features=input_features, attention_mask=attention_mask)[0]

        # input modality = speech so new attention mask for the decoder
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
            )

            # repeat attention mask alongside batch dimension
            attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)

        # repeat attention mask alongside batch dimension
        encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences[:, :-1],  # Manually trim the final EOS token
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # REMOVE EOS and lang_id
        t2u_input_ids = sequences[:, 2:-1]
        # replace every other EOS
        t2u_input_ids = ops.masked_fill(
            t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
        )

        # compute t2u_char_input_ids
        t2u_subwords = self._indices_to_subwords(t2u_input_ids)
        t2u_char_count_per_id = self._count_character_length_in_subword(
            t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
        )

        # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
        pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
        t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
        t2u_char_input_ids = self._get_char_input_ids(
            t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
        )

        # second pass
        t2u_output = self.t2u_model(
            inputs_embeds=t2u_input_embeds,
            char_input_ids=t2u_char_input_ids,
            char_count_per_id=t2u_char_count_per_id,
            **kwargs_speech,
        )

        t2u_logits = t2u_output[0]
        padding_mask = t2u_output[1].bool()

        # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
        temperature = kwargs_speech.get("temperature", None)
        if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
            unit_ids = ops.argmax(t2u_logits, dim=-1)
        else:
            t2u_logits = t2u_logits / temperature
            # apply softmax
            probs = nn.functional.softmax(t2u_logits, dim=-1)
            # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
            probs = probs.reshape((-1, probs.shape[2]))
            # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
            unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

        output_unit_ids = unit_ids.copy()

        replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
        # replace eos per pad
        unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(
            input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
        )

        if return_intermediate_token_ids:
            return SeamlessM4Tv2GenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    @staticmethod
    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech._reorder_cache
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForSpeechToSpeech.generate(input_features=None, return_intermediate_token_ids=None, tgt_lang=None, speaker_id=0, **kwargs)

Generates translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_features, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)` DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
speaker_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
kwargs	Remaining dictionary of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4Tv2GenerationOutput]	Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]:
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If return_intermediate_token_ids, returns [SeamlessM4Tv2GenerationOutput].
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@no_grad()
def generate(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    speaker_id: Optional[int] = 0,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
    """
    Generates translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_features, num_beams=4, speech_do_sample=True)` will successively perform
    beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Args:
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        speaker_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.

        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.


    Returns:
        `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]`:
        - If `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
        - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
          sequence_length)`and and `waveform_lengths` which gives the length of each sample.
    """
    batch_size = len(input_features) if input_features is not None else len(kwargs.get("inputs_embeds"))

    if tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
    else:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                more languages for text translation than for speech synthesis."""
                )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
    text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    text_generation_output = super().generate(input_features, **kwargs_text)
    sequences = text_generation_output.sequences

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    # get last_hidden_state from encoder
    encoder_hidden_states = self.speech_encoder(input_features=input_features, attention_mask=attention_mask)[0]

    # input modality = speech so new attention mask for the decoder
    if attention_mask is not None:
        sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
        attention_mask = _compute_new_attention_mask(
            hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
        )

        # repeat attention mask alongside batch dimension
        attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)

    # repeat attention mask alongside batch dimension
    encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences[:, :-1],  # Manually trim the final EOS token
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # REMOVE EOS and lang_id
    t2u_input_ids = sequences[:, 2:-1]
    # replace every other EOS
    t2u_input_ids = ops.masked_fill(
        t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
    )

    # compute t2u_char_input_ids
    t2u_subwords = self._indices_to_subwords(t2u_input_ids)
    t2u_char_count_per_id = self._count_character_length_in_subword(
        t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
    )

    # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
    pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
    t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
    t2u_char_input_ids = self._get_char_input_ids(
        t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
    )

    # second pass
    t2u_output = self.t2u_model(
        inputs_embeds=t2u_input_embeds,
        char_input_ids=t2u_char_input_ids,
        char_count_per_id=t2u_char_count_per_id,
        **kwargs_speech,
    )

    t2u_logits = t2u_output[0]
    padding_mask = t2u_output[1].bool()

    # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
    temperature = kwargs_speech.get("temperature", None)
    if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
        unit_ids = ops.argmax(t2u_logits, dim=-1)
    else:
        t2u_logits = t2u_logits / temperature
        # apply softmax
        probs = nn.functional.softmax(t2u_logits, dim=-1)
        # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
        probs = probs.reshape((-1, probs.shape[2]))
        # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
        unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

    output_unit_ids = unit_ids.copy()

    replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
    # replace eos per pad
    unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(
        input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
    )

    if return_intermediate_token_ids:
        return SeamlessM4Tv2GenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForSpeechToText

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ForSpeechToText(SeamlessM4Tv2PreTrainedModel):
    _keys_to_ignore_on_load_missing = ["text_decoder", "t2u_model", "vocoder"]
    main_input_name = "input_features"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_decoder.embed_tokens.weight",
    ]

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.__init__ with SeamlessM4T->SeamlessM4Tv2
    def __init__(self, config: SeamlessM4Tv2Config):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
        self.speech_encoder = SeamlessM4Tv2SpeechEncoder(config)
        self.text_decoder = SeamlessM4Tv2Decoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.get_encoder
    def get_encoder(self):
        return self.speech_encoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.get_decoder
    def get_decoder(self):
        return self.text_decoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.get_output_embeddings
    def get_output_embeddings(self):
        return self.lm_head

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.get_input_embeddings
    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.set_input_embeddings
    def set_input_embeddings(self, value):
        self.text_decoder.embed_tokens = value

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText._tie_weights
    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.forward
    def forward(
        self,
        input_features: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.generate
    def generate(
        self,
        input_features=None,
        tgt_lang=None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        **kwargs,
    ):
        """
        Generates sequences of token ids.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.

            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model.

        Return:
            [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
            [`~utils.ModelOutput`] types are:
                - [`~generation.GenerateEncoderDecoderOutput`],
                - [`~generation.GenerateBeamEncoderDecoderOutput`]
        """
        text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        input_features = input_features if input_features is not None else kwargs.pop("inputs")
        if tgt_lang is not None:
            inputs = kwargs.get("input_embeds") if input_features is None else input_features
            inputs = (
                inputs
                if inputs is not None
                else kwargs.get("encoder_outputs", {"last_hidden_state": None})["last_hidden_state"]
            )
            batch_size = len(inputs)

            if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
                # also accept __xxx__
                tgt_lang = tgt_lang.replace("__", "")
                if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                        {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                    )
                # tgt_lang gets priority over decoder input ids
                text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
                text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
            else:
                raise ValueError(
                    """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                    the target language to the right token id. Make sure to load the right generation config."""
                )
        else:
            # only a warning, otherwise errors appear in the tests
            logger.warning(
                """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
                a correct generation, otherwise the generation will probably make no sense."""
            )
        return super().generate(
            input_features,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            decoder_input_ids=text_decoder_input_ids,
            **kwargs,
        )

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText._reorder_cache
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForSpeechToText.generate(input_features=None, tgt_lang=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, **kwargs)

Generates sequences of token ids.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)` DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

Return

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. The possible [~utils.ModelOutput] types are: - [~generation.GenerateEncoderDecoderOutput], - [~generation.GenerateBeamEncoderDecoderOutput]

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def generate(
    self,
    input_features=None,
    tgt_lang=None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    **kwargs,
):
    """
    Generates sequences of token ids.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.

        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model.

    Return:
        [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
        or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
        [`~utils.ModelOutput`] types are:
            - [`~generation.GenerateEncoderDecoderOutput`],
            - [`~generation.GenerateBeamEncoderDecoderOutput`]
    """
    text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    input_features = input_features if input_features is not None else kwargs.pop("inputs")
    if tgt_lang is not None:
        inputs = kwargs.get("input_embeds") if input_features is None else input_features
        inputs = (
            inputs
            if inputs is not None
            else kwargs.get("encoder_outputs", {"last_hidden_state": None})["last_hidden_state"]
        )
        batch_size = len(inputs)

        if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                    {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                )
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
        else:
            raise ValueError(
                """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                the target language to the right token id. Make sure to load the right generation config."""
            )
    else:
        # only a warning, otherwise errors appear in the tests
        logger.warning(
            """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
            a correct generation, otherwise the generation will probably make no sense."""
        )
    return super().generate(
        input_features,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        decoder_input_ids=text_decoder_input_ids,
        **kwargs,
    )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForTextToSpeech

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel):
    _keys_to_ignore_on_load_missing = ["speech_encoder"]
    main_input_name = "input_ids"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.__init__ with SeamlessM4T->SeamlessM4Tv2
    def __init__(self, config: SeamlessM4Tv2Config):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4Tv2Encoder(config, self.shared)
        self.text_decoder = SeamlessM4Tv2Decoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.t2u_model = SeamlessM4Tv2TextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4Tv2CodeHifiGan(config)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.get_encoder
    def get_encoder(self):
        return self.text_encoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.get_decoder
    def get_decoder(self):
        return self.text_decoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.get_output_embeddings
    def get_output_embeddings(self):
        return self.lm_head

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.get_input_embeddings
    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.set_input_embeddings
    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech._tie_weights
    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.forward with SeamlessM4T->SeamlessM4Tv2
    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This is the same forward method as `SeamlessM4Tv2ForTextToText`."
                "It doesn't use the text-to-unit model `SeamlessM4Tv2TextToUnitForConditionalGeneration`."
                "If you want to generate speech, use the `.generate` method."
            )
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        speaker_id: Optional[int] = 0,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
        """
        Generates translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_ids, num_beams=4, speech_do_sample=True)` will successively perform
        beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            speaker_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.


        Returns:
            `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]`:
            - If `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
            - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
              sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        """
        batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

        if tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
        else:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                    more languages for text translation than for speech synthesis."""
                    )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")

        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        text_generation_output = super().generate(input_ids, **kwargs_text)
        sequences = text_generation_output.sequences

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        if attention_mask is not None:
            # repeat attention mask alongside batch dimension
            attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)
        encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

        # repeat attention mask alongside batch dimension
        encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences[:, :-1],  # Manually trim the final EOS token
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # REMOVE EOS and lang_id
        t2u_input_ids = sequences[:, 2:-1]
        # replace every other EOS
        t2u_input_ids = ops.masked_fill(
            t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
        )

        # compute t2u_char_input_ids
        t2u_subwords = self._indices_to_subwords(t2u_input_ids)
        t2u_char_count_per_id = self._count_character_length_in_subword(
            t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
        )

        # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
        pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
        t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
        t2u_char_input_ids = self._get_char_input_ids(
            t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
        )

        # second pass
        t2u_output = self.t2u_model(
            inputs_embeds=t2u_input_embeds,
            char_input_ids=t2u_char_input_ids,
            char_count_per_id=t2u_char_count_per_id,
            **kwargs_speech,
        )

        t2u_logits = t2u_output[0]
        padding_mask = t2u_output[1].bool()

        # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
        temperature = kwargs_speech.get("temperature", None)
        if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
            unit_ids = ops.argmax(t2u_logits, dim=-1)
        else:
            t2u_logits = t2u_logits / temperature
            # apply softmax
            probs = nn.functional.softmax(t2u_logits, dim=-1)
            # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
            probs = probs.reshape((-1, probs.shape[2]))
            # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
            unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

        output_unit_ids = unit_ids.copy()

        replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
        # replace eos per pad
        unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(
            input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
        )

        if return_intermediate_token_ids:
            return SeamlessM4Tv2GenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech._reorder_cache
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForTextToSpeech.generate(input_ids=None, return_intermediate_token_ids=None, tgt_lang=None, speaker_id=0, **kwargs)

Generates translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_ids, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
speaker_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
kwargs	Remaining dictionary of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4Tv2GenerationOutput]	Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]:
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If return_intermediate_token_ids, returns [SeamlessM4Tv2GenerationOutput].
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@no_grad()
def generate(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    speaker_id: Optional[int] = 0,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
    """
    Generates translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_ids, num_beams=4, speech_do_sample=True)` will successively perform
    beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        speaker_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.


    Returns:
        `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor]]`:
        - If `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
        - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
          sequence_length)`and and `waveform_lengths` which gives the length of each sample.
    """
    batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

    if tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
    else:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                more languages for text translation than for speech synthesis."""
                )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")

    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
    text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    text_generation_output = super().generate(input_ids, **kwargs_text)
    sequences = text_generation_output.sequences

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    if attention_mask is not None:
        # repeat attention mask alongside batch dimension
        attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)
    encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

    # repeat attention mask alongside batch dimension
    encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences[:, :-1],  # Manually trim the final EOS token
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # REMOVE EOS and lang_id
    t2u_input_ids = sequences[:, 2:-1]
    # replace every other EOS
    t2u_input_ids = ops.masked_fill(
        t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
    )

    # compute t2u_char_input_ids
    t2u_subwords = self._indices_to_subwords(t2u_input_ids)
    t2u_char_count_per_id = self._count_character_length_in_subword(
        t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
    )

    # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
    pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
    t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
    t2u_char_input_ids = self._get_char_input_ids(
        t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
    )

    # second pass
    t2u_output = self.t2u_model(
        inputs_embeds=t2u_input_embeds,
        char_input_ids=t2u_char_input_ids,
        char_count_per_id=t2u_char_count_per_id,
        **kwargs_speech,
    )

    t2u_logits = t2u_output[0]
    padding_mask = t2u_output[1].bool()

    # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
    temperature = kwargs_speech.get("temperature", None)
    if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
        unit_ids = ops.argmax(t2u_logits, dim=-1)
    else:
        t2u_logits = t2u_logits / temperature
        # apply softmax
        probs = nn.functional.softmax(t2u_logits, dim=-1)
        # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
        probs = probs.reshape((-1, probs.shape[2]))
        # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
        unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

    output_unit_ids = unit_ids.copy()

    replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
    # replace eos per pad
    unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(
        input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
    )

    if return_intermediate_token_ids:
        return SeamlessM4Tv2GenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForTextToText

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ForTextToText(SeamlessM4Tv2PreTrainedModel):
    _keys_to_ignore_on_load_missing = ["speech_encoder", "t2u_model", "vocoder"]
    main_input_name = "input_ids"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config: SeamlessM4Tv2Config):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4Tv2Encoder(config, self.shared)
        self.text_decoder = SeamlessM4Tv2Decoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_encoder(self):
        return self.text_encoder

    def get_decoder(self):
        return self.text_decoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    def generate(
        self,
        input_ids=None,
        tgt_lang=None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        **kwargs,
    ):
        """
        Generates sequences of token ids.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            input_ids (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model.

        Return:
            [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
            [`~utils.ModelOutput`] types are:
                - [`~generation.GenerateEncoderDecoderOutput`],
                - [`~generation.GenerateBeamEncoderDecoderOutput`]
        """
        # prepare text_decoder_input_ids
        text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        if tgt_lang is not None:
            batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

            if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
                # also accept __xxx__
                tgt_lang = tgt_lang.replace("__", "")
                if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                        {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                    )
                # tgt_lang gets priority over decoder input ids
                text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
                text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
            else:
                raise ValueError(
                    """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                    the target language to the right token id. Make sure to load the right generation config."""
                )
        else:
            # only a warning, otherwise errors appear in the tests
            logger.warning(
                """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
                a correct generation, otherwise the generation will probably make no sense."""
            )

        return super().generate(
            input_ids,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            decoder_input_ids=text_decoder_input_ids,
            **kwargs,
        )

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ForTextToText.generate(input_ids=None, tgt_lang=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, **kwargs)

Generates sequences of token ids.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of varying shape depending on the modality, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

Return

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. The possible [~utils.ModelOutput] types are: - [~generation.GenerateEncoderDecoderOutput], - [~generation.GenerateBeamEncoderDecoderOutput]

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def generate(
    self,
    input_ids=None,
    tgt_lang=None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    **kwargs,
):
    """
    Generates sequences of token ids.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        input_ids (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model.

    Return:
        [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
        or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
        [`~utils.ModelOutput`] types are:
            - [`~generation.GenerateEncoderDecoderOutput`],
            - [`~generation.GenerateBeamEncoderDecoderOutput`]
    """
    # prepare text_decoder_input_ids
    text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    if tgt_lang is not None:
        batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

        if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                    {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                )
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
        else:
            raise ValueError(
                """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                the target language to the right token id. Make sure to load the right generation config."""
            )
    else:
        # only a warning, otherwise errors appear in the tests
        logger.warning(
            """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
            a correct generation, otherwise the generation will probably make no sense."""
        )

    return super().generate(
        input_ids,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        decoder_input_ids=text_decoder_input_ids,
        **kwargs,
    )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2GenerationOutput dataclass

Bases: ModelOutput

Class defining the generated outputs from [SeamlessM4Tv2Model], [SeamlessM4Tv2ForTextToText], [SeamlessM4Tv2ForTextToSpeech], [SeamlessM4Tv2ForSpeechToSpeech] and [SeamlessM4Tv2ForTextToSpeech].

PARAMETER	DESCRIPTION
waveform	The final audio waveform predicted by the model. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
waveform_lengths	The length in samples of each element in the waveform batch. TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None
sequences	The generated translated sequences. This is the output of the text-to-text or the speech-to-text models. The second dimension (sequence_length) is either equal to max_length or shorter if all batches finished early due to the eos_token_id. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
unit_sequences	The generated translated unit sequences. This is the output of the text-to-units model. The second dimension (unit_sequence_length) is either equal to t2u_max_length or shorter if all batches finished early due to the t2u_eos_token_id. TYPE: `mindspore.Tensor` of shape `(batch_size, unit_sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@dataclass
# Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TGenerationOutput with SeamlessM4T->SeamlessM4Tv2
class SeamlessM4Tv2GenerationOutput(ModelOutput):
    """
    Class defining the generated outputs from [`SeamlessM4Tv2Model`], [`SeamlessM4Tv2ForTextToText`],
    [`SeamlessM4Tv2ForTextToSpeech`], [`SeamlessM4Tv2ForSpeechToSpeech`] and [`SeamlessM4Tv2ForTextToSpeech`].

    Args:
        waveform (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            The final audio waveform predicted by the model.
        waveform_lengths (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            The length in samples of each element in the `waveform` batch.
        sequences (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            The generated translated sequences. This is the output of the text-to-text or the speech-to-text models.
            The second dimension (sequence_length) is either equal to `max_length` or shorter if all batches finished
            early due to the `eos_token_id`.
        unit_sequences (`mindspore.Tensor` of shape `(batch_size, unit_sequence_length)`, *optional*):
            The generated translated unit sequences. This is the output of the text-to-units model. The second
            dimension (unit_sequence_length) is either equal to `t2u_max_length` or shorter if all batches finished
            early due to the `t2u_eos_token_id`.
    """

    waveform: Optional[mindspore.Tensor] = None
    waveform_lengths: Optional[mindspore.Tensor] = None
    sequences: Optional[Tuple[mindspore.Tensor]] = None
    unit_sequences: Optional[Tuple[mindspore.Tensor]] = None

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2HifiGan

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2HifiGan(nn.Module):
    def __init__(self, config: SeamlessM4Tv2Config):
        super().__init__()
        model_in_dim = config.unit_embed_dim + config.lang_embed_dim + config.spkr_embed_dim
        self.leaky_relu_slope = config.leaky_relu_slope
        self.num_kernels = len(config.resblock_kernel_sizes)
        self.num_upsamples = len(config.upsample_rates)
        self.conv_pre = nn.Conv1d(
            model_in_dim,
            config.upsample_initial_channel,
            kernel_size=7,
            stride=1,
            padding=3,
        )

        self.upsampler = nn.ModuleList()
        for i, (upsample_rate, kernel_size) in enumerate(zip(config.upsample_rates, config.upsample_kernel_sizes)):
            self.upsampler.append(
                nn.ConvTranspose1d(
                    config.upsample_initial_channel // (2**i),
                    config.upsample_initial_channel // (2 ** (i + 1)),
                    kernel_size=kernel_size,
                    stride=upsample_rate,
                    padding=(kernel_size - upsample_rate) // 2,
                )
            )

        self.resblocks = nn.ModuleList()
        for i in range(len(self.upsampler)):
            channels = config.upsample_initial_channel // (2 ** (i + 1))
            for kernel_size, dilation in zip(config.resblock_kernel_sizes, config.resblock_dilation_sizes):
                self.resblocks.append(HifiGanResidualBlock(channels, kernel_size, dilation, config.leaky_relu_slope))

        self.conv_post = nn.Conv1d(channels, 1, kernel_size=7, stride=1, padding=3)

    def forward(self, input_embeds: mindspore.Tensor) -> mindspore.Tensor:
        r"""
        Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch
        of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech
        waveform.

        Args:
            spectrogram (`mindspore.Tensor`):
                Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length,
                model_in_dim)`, or un-batched and of shape `(sequence_length, model_in_dim)`. Note that `model_in_dim`
                is the sum of `config.unit_embed_dim`, `config.lang_embed_dim` and `config.spkr_embed_dim`.

        Returns:
            `mindspore.Tensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
            shape `(batch_size, num_frames,)`. If un-batched, will be of shape `(num_frames,)`.
        """

        hidden_states = self.conv_pre(input_embeds)
        for i in range(self.num_upsamples):
            hidden_states = nn.functional.leaky_relu(hidden_states, self.leaky_relu_slope)
            hidden_states = self.upsampler[i](hidden_states)

            res_state = self.resblocks[i * self.num_kernels](hidden_states)
            for j in range(1, self.num_kernels):
                res_state += self.resblocks[i * self.num_kernels + j](hidden_states)
            hidden_states = res_state / self.num_kernels

        hidden_states = nn.functional.leaky_relu(hidden_states)
        hidden_states = self.conv_post(hidden_states)
        hidden_states = ops.tanh(hidden_states)

        # remove seq-len dim since this collapses to 1
        waveform = hidden_states.squeeze(1)

        return waveform

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2HifiGan.forward(input_embeds)

Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech waveform.

PARAMETER	DESCRIPTION
spectrogram	Tensor containing the log-mel spectrograms. Can be batched and of shape (batch_size, sequence_length, model_in_dim), or un-batched and of shape (sequence_length, model_in_dim). Note that model_in_dim is the sum of config.unit_embed_dim, config.lang_embed_dim and config.spkr_embed_dim. TYPE: `mindspore.Tensor`

RETURNS	DESCRIPTION
Tensor	mindspore.Tensor: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
Tensor	shape (batch_size, num_frames,). If un-batched, will be of shape (num_frames,).

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(self, input_embeds: mindspore.Tensor) -> mindspore.Tensor:
    r"""
    Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch
    of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech
    waveform.

    Args:
        spectrogram (`mindspore.Tensor`):
            Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length,
            model_in_dim)`, or un-batched and of shape `(sequence_length, model_in_dim)`. Note that `model_in_dim`
            is the sum of `config.unit_embed_dim`, `config.lang_embed_dim` and `config.spkr_embed_dim`.

    Returns:
        `mindspore.Tensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
        shape `(batch_size, num_frames,)`. If un-batched, will be of shape `(num_frames,)`.
    """

    hidden_states = self.conv_pre(input_embeds)
    for i in range(self.num_upsamples):
        hidden_states = nn.functional.leaky_relu(hidden_states, self.leaky_relu_slope)
        hidden_states = self.upsampler[i](hidden_states)

        res_state = self.resblocks[i * self.num_kernels](hidden_states)
        for j in range(1, self.num_kernels):
            res_state += self.resblocks[i * self.num_kernels + j](hidden_states)
        hidden_states = res_state / self.num_kernels

    hidden_states = nn.functional.leaky_relu(hidden_states)
    hidden_states = self.conv_post(hidden_states)
    hidden_states = ops.tanh(hidden_states)

    # remove seq-len dim since this collapses to 1
    waveform = hidden_states.squeeze(1)

    return waveform

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Model

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2Model(SeamlessM4Tv2PreTrainedModel):
    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.__init__ with SeamlessM4T->SeamlessM4Tv2
    def __init__(self, config, current_modality="text"):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4Tv2Encoder(config, self.shared)
        self.speech_encoder = SeamlessM4Tv2SpeechEncoder(config)
        self.text_decoder = SeamlessM4Tv2Decoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.current_modality = current_modality
        if current_modality == "speech":
            self.main_input_name = "input_features"

        # these models already call post_init in their initialization
        self.t2u_model = SeamlessM4Tv2TextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4Tv2CodeHifiGan(config)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.set_modality
    def set_modality(self, modality="text"):
        if modality == "text":
            self.main_input_name = "input_ids"
            self.current_modality = "text"
        elif modality == "speech":
            self.main_input_name = "input_features"
            self.current_modality = "speech"
        else:
            raise ValueError(f"`modality={modality}` is not a valid modality. It must be `text` or `speech`.")

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.get_encoder
    def get_encoder(self):
        if self.current_modality == "text":
            return self.text_encoder
        else:
            return self.speech_encoder

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.get_output_embeddings
    def get_output_embeddings(self):
        return self.lm_head

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.get_input_embeddings
    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.set_input_embeddings
    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel._tie_weights
    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.forward with SeamlessM4T->SeamlessM4Tv2
    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        input_features: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        if input_ids is None and input_features is None and inputs_embeds is None and encoder_outputs is None:
            raise ValueError(
                "`input_ids`,`input_features`, `inputs_embeds` and `encoder_outputs` are all empty. Make sure at least one of them is not."
            )
        elif input_features is not None:
            if input_ids is not None:
                logger.warning(
                    "`input_ids` is not `None` but `input_features` has been given."
                    "`input_features` will be used in priority through the `speech_encoder`. "
                    "Make sure that `input_features` and `input_ids` are mutually exclusive."
                )

            if inputs_embeds is not None:
                logger.warning(
                    "`inputs_embeds` is not `None` but `input_features` has been given."
                    "`input_features` will be used in priority through `speech_encoder`. "
                    "`inputs_embeds` will be ignored."
                )

            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This calls the same method `forward` as `SeamlessM4Tv2ForTextToText` and `SeamlessM4Tv2ForSpeechToText`"
                "depending on the input modality. If you want to generate speech, use the `generate` method."
            )

            self.set_modality("speech")

            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        elif input_ids is not None or inputs_embeds is not None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This calls the same method `forward` as `SeamlessM4Tv2ForTextToText` and `SeamlessM4Tv2ForSpeechToText`"
                "depending on the input modality. If you want to generate speech, use the `generate` method."
            )
            self.set_modality("text")
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        # input modality = speech so new attention mask
        if self.current_modality == "speech" and attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        input_features: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        speaker_id: Optional[int] = 0,
        generate_speech: Optional[bool] = True,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
        """
        Generates translated token ids and/or translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_ids=input_ids, num_beams=4, speech_do_sample=True)` will successively
        perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>


        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional*):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio. Note that if `generate_speech=True`, this parameter will be
                ignored.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            speaker_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            generate_speech (`bool`, *optional*, defaults to `True`):
                If `False`, will only returns the text tokens and won't generate speech.

            kwargs (*optional*):
                Remaining dictioy of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.

        Returns:
            `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor], ModelOutput]`:
            - If `generate_speech` and `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
            - If `generate_speech` and not `return_intermediate_token_ids`, returns a tuple composed of waveforms of
              shape `(batch_size, sequence_length)`and and `waveform_lengths` which gives the length of each sample.
            - If `generate_speech=False`, it will returns `ModelOutput`.
        """
        if input_ids is None and input_features is None and kwargs.get("inputs_embeds", None) is None:
            raise ValueError(
                "`input_ids`,`input_features` and `inputs_embeds` are all empty. Make sure at least one of them is not."
            )

        if generate_speech and tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")

        if tgt_lang is not None:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            if generate_speech:
                keys_to_check = ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]
            else:
                keys_to_check = ["text_decoder_lang_to_code_id"]
            for key in keys_to_check:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                    more languages for text translation than for speech synthesis."""
                    )

        batch_size = (
            len(input_features)
            if input_features is not None
            else (len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds")))
        )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        if tgt_lang is not None:
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        if input_features is not None:
            self.set_modality("speech")
            if input_ids is not None:
                logger.warning(
                    "`input_features` and `input_ids` are both non empty. `input_features` will be used in priority "
                    "through the speech encoder. Make sure `input_features=None` if you want to use the text encoder."
                )
            text_generation_output = super().generate(input_features=input_features, **kwargs_text)
        else:
            self.set_modality("text")
            text_generation_output = super().generate(input_ids=input_ids, input_features=None, **kwargs_text)
        sequences = text_generation_output.sequences

        if not generate_speech:
            return text_generation_output

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        # get encoder last hidden states
        if self.current_modality == "speech":
            # get last_hidden_state from encoder - must do a pass through the speech encoder
            encoder_hidden_states = self.speech_encoder(
                input_features=input_features, attention_mask=attention_mask
            ).last_hidden_state

            # input modality = speech so new attention mask for the decoder
            if attention_mask is not None:
                sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
                attention_mask = _compute_new_attention_mask(
                    hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
                )
        else:
            encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

        if attention_mask is not None:
            # repeat attention mask alongside batch dimension
            attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)

        # repeat attention mask alongside batch dimension
        encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences[:, :-1],  # Manually trim the final EOS token
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # REMOVE EOS and lang_id
        t2u_input_ids = sequences[:, 2:-1]
        # replace every other EOS
        t2u_input_ids = ops.masked_fill(
            t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
        )

        # compute t2u_char_input_ids
        t2u_subwords = self._indices_to_subwords(t2u_input_ids)
        t2u_char_count_per_id = self._count_character_length_in_subword(
            t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
        )

        # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
        pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
        t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
        t2u_char_input_ids = self._get_char_input_ids(
            t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
        )

        # second pass
        t2u_output = self.t2u_model(
            inputs_embeds=t2u_input_embeds,
            char_input_ids=t2u_char_input_ids,
            char_count_per_id=t2u_char_count_per_id,
            **kwargs_speech,
        )

        t2u_logits = t2u_output[0]
        padding_mask = t2u_output[1].bool()

        # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
        temperature = kwargs_speech.get("temperature", None)
        if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
            unit_ids = ops.argmax(t2u_logits, dim=-1)
        else:
            t2u_logits = t2u_logits / temperature
            # apply softmax
            probs = nn.functional.softmax(t2u_logits, dim=-1)
            # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
            probs = probs.reshape((-1, probs.shape[2]))
            # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
            unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

        output_unit_ids = unit_ids.copy()

        replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
        # replace eos per pad
        unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(
            input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
        )

        if return_intermediate_token_ids:
            return SeamlessM4Tv2GenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel._reorder_cache
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2Model.generate(input_ids=None, input_features=None, return_intermediate_token_ids=None, tgt_lang=None, speaker_id=0, generate_speech=True, **kwargs)

Generates translated token ids and/or translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_ids=input_ids, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional* DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. Note that if generate_speech=True, this parameter will be ignored. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
speaker_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
generate_speech	If False, will only returns the text tokens and won't generate speech. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
kwargs	Remaining dictioy of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4Tv2GenerationOutput]	Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor], ModelOutput]:
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If generate_speech and return_intermediate_token_ids, returns [SeamlessM4Tv2GenerationOutput].
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If generate_speech and not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.
Union[Tensor, SeamlessM4Tv2GenerationOutput]	If generate_speech=False, it will returns ModelOutput.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@no_grad()
def generate(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    input_features: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    speaker_id: Optional[int] = 0,
    generate_speech: Optional[bool] = True,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4Tv2GenerationOutput]:
    """
    Generates translated token ids and/or translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_ids=input_ids, num_beams=4, speech_do_sample=True)` will successively
    perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>


    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional*):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio. Note that if `generate_speech=True`, this parameter will be
            ignored.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        speaker_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        generate_speech (`bool`, *optional*, defaults to `True`):
            If `False`, will only returns the text tokens and won't generate speech.

        kwargs (*optional*):
            Remaining dictioy of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.

    Returns:
        `Union[SeamlessM4Tv2GenerationOutput, Tuple[Tensor], ModelOutput]`:
        - If `generate_speech` and `return_intermediate_token_ids`, returns [`SeamlessM4Tv2GenerationOutput`].
        - If `generate_speech` and not `return_intermediate_token_ids`, returns a tuple composed of waveforms of
          shape `(batch_size, sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        - If `generate_speech=False`, it will returns `ModelOutput`.
    """
    if input_ids is None and input_features is None and kwargs.get("inputs_embeds", None) is None:
        raise ValueError(
            "`input_ids`,`input_features` and `inputs_embeds` are all empty. Make sure at least one of them is not."
        )

    if generate_speech and tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")

    if tgt_lang is not None:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        if generate_speech:
            keys_to_check = ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]
        else:
            keys_to_check = ["text_decoder_lang_to_code_id"]
        for key in keys_to_check:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4Tv2 supports
                more languages for text translation than for speech synthesis."""
                )

    batch_size = (
        len(input_features)
        if input_features is not None
        else (len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds")))
    )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    if tgt_lang is not None:
        # tgt_lang gets priority over decoder input ids
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    if input_features is not None:
        self.set_modality("speech")
        if input_ids is not None:
            logger.warning(
                "`input_features` and `input_ids` are both non empty. `input_features` will be used in priority "
                "through the speech encoder. Make sure `input_features=None` if you want to use the text encoder."
            )
        text_generation_output = super().generate(input_features=input_features, **kwargs_text)
    else:
        self.set_modality("text")
        text_generation_output = super().generate(input_ids=input_ids, input_features=None, **kwargs_text)
    sequences = text_generation_output.sequences

    if not generate_speech:
        return text_generation_output

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    # get encoder last hidden states
    if self.current_modality == "speech":
        # get last_hidden_state from encoder - must do a pass through the speech encoder
        encoder_hidden_states = self.speech_encoder(
            input_features=input_features, attention_mask=attention_mask
        ).last_hidden_state

        # input modality = speech so new attention mask for the decoder
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
            )
    else:
        encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

    if attention_mask is not None:
        # repeat attention mask alongside batch dimension
        attention_mask = ops.repeat_interleave(attention_mask, num_return_sequences, dim=0)

    # repeat attention mask alongside batch dimension
    encoder_hidden_states = ops.repeat_interleave(encoder_hidden_states, num_return_sequences, dim=0)

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences[:, :-1],  # Manually trim the final EOS token
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences[:, :-1] != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # REMOVE EOS and lang_id
    t2u_input_ids = sequences[:, 2:-1]
    # replace every other EOS
    t2u_input_ids = ops.masked_fill(
        t2u_input_ids, t2u_input_ids == self.generation_config.eos_token_id, pad_token_id
    )

    # compute t2u_char_input_ids
    t2u_subwords = self._indices_to_subwords(t2u_input_ids)
    t2u_char_count_per_id = self._count_character_length_in_subword(
        t2u_input_ids, t2u_subwords, pad_token_id=pad_token_id
    )

    # Add pads for lang, EOS tokens as per NLLB "source" tokenizer mode.
    pad_zero = ops.zeros((t2u_char_count_per_id.shape[0], 1), dtype=t2u_char_count_per_id.dtype)
    t2u_char_count_per_id = ops.cat([pad_zero, t2u_char_count_per_id, pad_zero], dim=1)
    t2u_char_input_ids = self._get_char_input_ids(
        t2u_input_ids, t2u_subwords, t2u_char_count_per_id, pad_token_id=pad_token_id
    )

    # second pass
    t2u_output = self.t2u_model(
        inputs_embeds=t2u_input_embeds,
        char_input_ids=t2u_char_input_ids,
        char_count_per_id=t2u_char_count_per_id,
        **kwargs_speech,
    )

    t2u_logits = t2u_output[0]
    padding_mask = t2u_output[1].bool()

    # The text-to-unit model is non auto-regressive. We keep the ability to use sampling with temperature
    temperature = kwargs_speech.get("temperature", None)
    if (temperature is None or temperature == 1.0) or not kwargs_speech.get("do_sample", False):
        unit_ids = ops.argmax(t2u_logits, dim=-1)
    else:
        t2u_logits = t2u_logits / temperature
        # apply softmax
        probs = nn.functional.softmax(t2u_logits, dim=-1)
        # reshape to 2D: (batch_size, seq_len, t2u_vocab_size) -> (batch_size*seq_len, t2u_vocab_size)
        probs = probs.reshape((-1, probs.shape[2]))
        # multinomial then reshape : (batch_size*seq_len)-> (batch_size,seq_len)
        unit_ids = ops.multinomial(probs, num_samples=1).view(t2u_logits.shape[0], -1)

    output_unit_ids = unit_ids.copy()

    replace_mask = (unit_ids == self.config.t2u_eos_token_id).int() | (~padding_mask).int()
    # replace eos per pad
    unit_ids = unit_ids.masked_fill(replace_mask.bool(), self.config.t2u_pad_token_id)

    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    speaker_id = mindspore.tensor([[speaker_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(
        input_ids=unit_ids, speaker_id=speaker_id, lang_id=vocoder_tgt_lang_id
    )

    if return_intermediate_token_ids:
        return SeamlessM4Tv2GenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2PreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2PreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """

    config_class = SeamlessM4Tv2Config
    base_model_prefix = "seamless_m4t_v2"
    supports_gradient_checkpointing = True
    _no_split_modules = [
        "SeamlessM4Tv2EncoderLayer",
        "SeamlessM4Tv2DecoderLayer",
        "SeamlessM4Tv2ConformerEncoderLayer",
        "SeamlessM4Tv2TextToUnitDecoderLayer",
    ]

    def _init_weights(self, module):
        """Initialize the weights"""
        std = self.config.initializer_range
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, mean=0.0, std=std)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=std)
            if module.padding_idx is not None:
                module.weight[module.padding_idx] = 0
        elif isinstance(module, SeamlessM4Tv2ConformerSelfAttention):
            if hasattr(module, "pos_bias_u"):
                nn.init.xavier_uniform_(module.pos_bias_u)
            if hasattr(module, "pos_bias_v"):
                nn.init.xavier_uniform_(module.pos_bias_v)
        elif isinstance(module, SeamlessM4Tv2ConformerFeatureProjection):
            k = math.sqrt(1 / module.projection.in_features)
            nn.init.uniform_(module.projection.weight, a=-k, b=k)
            nn.init.uniform_(module.projection.bias, a=-k, b=k)
        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
            nn.init.zeros_(module.bias)
            nn.init.ones_(module.weight)
        elif isinstance(module, (nn.Conv1d, nn.ConvTranspose1d)):
            nn.init.kaiming_normal_(module.weight)
            if module.bias is not None:
                k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
                nn.init.uniform_(module.bias, a=-k, b=k)

    # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TPreTrainedModel._compute_sub_sample_lengths_from_attention_mask
    def _compute_sub_sample_lengths_from_attention_mask(self, attention_mask):
        kernel_size, stride = self.config.adaptor_kernel_size, self.config.adaptor_stride
        pad = kernel_size // 2
        seq_lens = attention_mask.shape[1] - (1 - attention_mask.int()).sum(1)

        seq_lens = ((seq_lens + 2 * pad - kernel_size) / stride) + 1

        return seq_lens.floor()

    def _indices_to_subwords(self, input_ids):
        """
        Returns the corresponding text string for each input id.
        """
        if not hasattr(self.generation_config, "id_to_text"):
            raise ValueError(
                """This model generation config doesn't have a `id_to_text` key which maps
                token ids to subwords. Make sure to load the right generation config."""
            )
        batch_size, sequence_len = input_ids.shape

        subwords_batch = []
        for batch_id in range(batch_size):
            subwords = []
            for i in range(sequence_len):
                subword = self.generation_config.id_to_text.get(str(input_ids[batch_id, i].item()))
                subwords.append(str(subword))
            subwords_batch.append(subwords)
        return subwords_batch

    def _count_character_length_in_subword(
        self,
        input_ids,
        subwords_batch,
        merge_space_with_prev_subword=False,
        pad_token_id=0,
        unk_token_id=1,
        space="▁",
    ):
        """
        Counts the number of characters per text string associated with the input token id.

        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.
            subwords_batch (`List[List[str]]` of shape `(batch_size, sequence_length)`):
                Corresponding text string for each input id.
            merge_space_with_prev_subword (`bool`, *optional*, defaults to `False`):
                Indicates if the space character is merged with the previous subword. If `False`, it will be merged
                with the next subword.
            pad_token_id (`int`, *optional*, defaults to 0):
                The id of the _padding_ text token. If it is encountered when calculating the length of a subword
                sample, the lengths of subsequent subwords will be set to 0.
            unk_token_id (`int`, *optional*, defaults to 1):
                The id of the _unknown_ text token. Associated to a subword of length 1.
            space (`str`, *optional*, defaults to `"▁"`):
                The space character.
        """
        batch_size, _ = input_ids.shape

        char_count_per_id = ops.zeros(input_ids.shape, dtype=input_ids.dtype)

        subword_lens = input_ids.ne(pad_token_id).sum(1)

        for batch_id in range(batch_size):
            # We slice out the tensor till the padding index.
            subword_indices = input_ids[batch_id, : subword_lens[batch_id]]
            subwords = subwords_batch[batch_id][: subword_lens[batch_id]]

            is_next_start_with_space = [
                len(subwords[i + 1]) > 1 and subwords[i + 1][0] == space if i < len(subwords) - 1 else False
                for i in range(len(subwords))
            ]
            is_punc = [
                len(subwords[i]) == 1
                and not subwords[i].isalpha()
                and not subwords[i].isnumeric()
                and subwords[i] != space
                for i in range(len(subwords))
            ]
            for i, (subword_idx, subword) in enumerate(zip(subword_indices, subwords)):
                if subword_idx == pad_token_id:
                    break

                if subword_idx == unk_token_id:
                    # We set char_len to 1 for an unk token.
                    char_len = 1

                    if merge_space_with_prev_subword and is_next_start_with_space[i]:
                        char_len += 1
                else:
                    # By default, spaces are merged with the next subword.
                    # char_len includes the space.
                    char_len = len(subword)

                    if merge_space_with_prev_subword:
                        # Add the space for the next subword.
                        if is_next_start_with_space[i]:
                            char_len += 1
                        # Subtract the space for the current subword.
                        if i > 0 and is_next_start_with_space[i - 1]:
                            char_len -= 1
                    else:
                        # Merge space with punctuation mark by default.
                        if is_punc[i] and is_next_start_with_space[i]:
                            char_len += 1
                        # Subtract the space for the subword succeeding the punctuation mark.
                        elif i > 0 and is_punc[i - 1] and is_next_start_with_space[i - 1]:
                            char_len -= 1

                char_count_per_id[batch_id, i] = char_len

        return char_count_per_id

    def _get_char_input_ids(self, input_ids, subwords_batch, char_count_per_id, pad_token_id=0, unk_token_id=1):
        """
        Returns the corresponding character input id for each character of `subwords_batch`.

        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.
            subwords_batch (`List[List[str]]` of shape `(batch_size, sequence_length)`):
                Corresponding text string for each input id.
            char_count_per_id (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Number of characters per input id.
            pad_token_id (`int`, *optional*, defaults to 0):
                The id of the _padding_ text token. If it is encountered when calculating the length of a subword
                sample, the lengths of subsequent subwords will be set to 0.
            unk_token_id (`int`, *optional*, defaults to 1):
                The id of the _unknown_ text token. Associated to a subword of length 1.
        Returns:
            `mindspore.Tensor`: Tensor of shape `(batch_size, char_sequence_length)` containing the id of each character.
        """
        if not hasattr(self.generation_config, "char_to_id"):
            raise ValueError(
                """This model generation config doesn't have a `char_to_id` key which maps
                characters to character ids. Make sure to load the right generation config."""
            )

        batch_size = input_ids.shape[0]
        max_len = int(char_count_per_id.sum(1).max().item())

        char_seqs = ops.full((batch_size, max_len), pad_token_id, dtype=input_ids.dtype)

        subword_lens = input_ids.ne(pad_token_id).sum(1)

        for batch_id in range(batch_size):
            total = 0
            subword_indices = input_ids[batch_id, : subword_lens[batch_id]]
            subwords = subwords_batch[batch_id][: subword_lens[batch_id]]
            for subword_idx, subword in zip(subword_indices, subwords):
                if subword_idx == unk_token_id:
                    char_ids = [unk_token_id]
                else:
                    # Get char token indices corresponding to the subwords.
                    char_ids = [self.generation_config.char_to_id.get(ch, unk_token_id) for ch in list(subword)]
                char_seq_len = len(char_ids)
                char_seqs[batch_id, total : total + char_seq_len] = mindspore.tensor(char_ids).to(char_seqs.dtype)
                total += char_seq_len
        return char_seqs

    def _hard_upsample(self, hidden_states, durations):
        """
        Repeats the time dimension of each sample in the batch based on the corresponding duration.

        Args:
            hidden_states (`mindspore.Tensor` of shape `(batch_size, sequence_length, *)`, *optional*):
                The sequence to repeat, where `*` is any number of sequence-specific dimensions including none.
            durations (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indicates how many times to repeat time segments.
        """
        if hidden_states.shape[0] == 1:
            hidden_states = ops.repeat_interleave(hidden_states, durations.view(-1), dim=1)
        else:
            # if batched sample, need to interleave per sample, and pad -> loss of parallelism
            if hidden_states.shape[0] > 1 and self.training:
                logger.warning_once(
                    """`self.training=True` and you use batching. You lose parallelism during the hifigan
                               forward pass because the samples are interleaved."""
                )
            hidden_states = [
                ops.repeat_interleave(hidden_state, duration.tolist(), dim=0)
                for (hidden_state, duration) in zip(hidden_states, durations)
            ]

            hidden_states = pad_sequence(hidden_states, batch_first=True)

        return hidden_states

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2ScaledWordEmbedding

Bases: Embedding

This module overrides nn.Embeddings' forward by multiplying with embeddings scale.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2ScaledWordEmbedding(nn.Embedding):
    """
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    """

    def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, embed_scale: Optional[float] = 1.0):
        super().__init__(num_embeddings, embedding_dim, padding_idx)
        self.embed_scale = embed_scale

    def forward(self, input_ids: mindspore.Tensor):
        return super().forward(input_ids) * self.embed_scale

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2SinusoidalPositionalEmbedding

Bases: Module

This module produces sinusoidal positional embeddings of any length.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2SinusoidalPositionalEmbedding(nn.Module):
    """This module produces sinusoidal positional embeddings of any length."""

    def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None):
        super().__init__()
        self.offset = 2
        self.embedding_dim = embedding_dim
        self.padding_idx = padding_idx
        self.make_weights(num_positions + self.offset, embedding_dim, padding_idx)

    def make_weights(self, num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        emb_weights = self.get_embedding(num_embeddings, embedding_dim, padding_idx)
        if hasattr(self, "weights"):
            # in forward put the weights on the correct dtype and device of the param
            emb_weights = emb_weights.to(dtype=self.weights.dtype)

        self.register_buffer("weights", emb_weights, persistent=False)

    @staticmethod
    def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        """
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        """
        half_dim = embedding_dim // 2
        emb = math.log(10000) / (half_dim - 1)
        emb = ops.exp(ops.arange(half_dim, dtype=mindspore.int64).float() * -emb)
        emb = ops.arange(num_embeddings, dtype=mindspore.int64).float().unsqueeze(1) * emb.unsqueeze(0)
        emb = ops.cat([ops.sin(emb), ops.cos(emb)], dim=1).view(num_embeddings, -1)
        if embedding_dim % 2 == 1:
            # zero pad
            emb = ops.cat([emb, ops.zeros(num_embeddings, 1)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0

        return emb.to(get_default_dtype())

    @no_grad()
    def forward(
        self, input_ids: mindspore.Tensor = None, inputs_embeds: mindspore.Tensor = None, past_key_values_length: int = 0
    ):
        if input_ids is not None:
            bsz, seq_len = input_ids.shape
            # Create the position ids from the input token ids. Any padded tokens remain padded.
            position_ids = create_position_ids_from_input_ids(input_ids, self.padding_idx, past_key_values_length)
        else:
            bsz, seq_len = inputs_embeds.shape[:-1]
            position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length)

        # expand embeddings if needed
        max_pos = self.padding_idx + 1 + seq_len + past_key_values_length
        if max_pos > self.weights.shape[0]:
            self.make_weights(max_pos + self.offset, self.embedding_dim, self.padding_idx)

        return self.weights.index_select(0, position_ids.view(-1)).view(bsz, seq_len, self.weights.shape[-1])

    def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length):
        """
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: mindspore.Tensor

        Returns: mindspore.Tensor
        """
        input_shape = inputs_embeds.shape[:-1]
        sequence_length = input_shape[1]

        position_ids = ops.arange(
            self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=mindspore.int64
        )
        return position_ids.unsqueeze(0).broadcast_to(input_shape) + past_key_values_length

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2SinusoidalPositionalEmbedding.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length)

We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

PARAMETER	DESCRIPTION
inputs_embeds	mindspore.Tensor

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length):
    """
    We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

    Args:
        inputs_embeds: mindspore.Tensor

    Returns: mindspore.Tensor
    """
    input_shape = inputs_embeds.shape[:-1]
    sequence_length = input_shape[1]

    position_ids = ops.arange(
        self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=mindspore.int64
    )
    return position_ids.unsqueeze(0).broadcast_to(input_shape) + past_key_values_length

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2SinusoidalPositionalEmbedding.get_embedding(num_embeddings, embedding_dim, padding_idx=None) staticmethod

Build sinusoidal embeddings.

This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of "Attention Is All You Need".

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@staticmethod
def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
    """
    Build sinusoidal embeddings.

    This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
    "Attention Is All You Need".
    """
    half_dim = embedding_dim // 2
    emb = math.log(10000) / (half_dim - 1)
    emb = ops.exp(ops.arange(half_dim, dtype=mindspore.int64).float() * -emb)
    emb = ops.arange(num_embeddings, dtype=mindspore.int64).float().unsqueeze(1) * emb.unsqueeze(0)
    emb = ops.cat([ops.sin(emb), ops.cos(emb)], dim=1).view(num_embeddings, -1)
    if embedding_dim % 2 == 1:
        # zero pad
        emb = ops.cat([emb, ops.zeros(num_embeddings, 1)], dim=1)
    if padding_idx is not None:
        emb[padding_idx, :] = 0

    return emb.to(get_default_dtype())

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitDecoder

Bases: SeamlessM4Tv2PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2TextToUnitDecoder(SeamlessM4Tv2PreTrainedModel):
    def __init__(
        self,
        config: SeamlessM4Tv2Config,
        embed_tokens: Optional[nn.Embedding] = None,
    ):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size
        self.max_target_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(config.hidden_size) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            # if embed_tokens defined, use its shape instead
            self.embed_tokens = nn.Embedding(embed_tokens.num_embeddings, embed_tokens.embedding_dim, self.padding_idx)
            self.embed_tokens.weight = embed_tokens.weight
        else:
            self.embed_tokens = nn.Embedding(self.vocab_size, config.hidden_size, self.padding_idx)

        self.embed_char = nn.Embedding(config.char_vocab_size, config.hidden_size)
        self.embed_char_positions = SeamlessM4Tv2SinusoidalPositionalEmbedding(
            self.max_target_positions,
            config.hidden_size,
            padding_idx=self.padding_idx,
        )

        self.pos_emb_alpha_char = nn.Parameter(ops.ones(1))
        self.pos_emb_alpha = nn.Parameter(ops.ones(1))
        self.duration_predictor = SeamlessM4Tv2VariancePredictor(
            config.variance_predictor_embed_dim,
            config.variance_predictor_hidden_dim,
            config.variance_predictor_kernel_size,
            config.variance_pred_dropout,
        )

        self.embed_positions = SeamlessM4Tv2SinusoidalPositionalEmbedding(
            self.max_target_positions,
            config.hidden_size,
            padding_idx=self.padding_idx,
        )

        layers = []
        for _ in range(config.decoder_layers):
            layers.append(
                SeamlessM4Tv2TextToUnitDecoderLayer(
                    config,
                    decoder_attention_heads=config.decoder_attention_heads,
                    decoder_ffn_dim=config.decoder_ffn_dim,
                )
            )
        self.layers = nn.ModuleList(layers)
        self.layer_norm = nn.LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def forward(
        self,
        char_input_ids: mindspore.Tensor = None,
        char_count_per_id: mindspore.Tensor = None,
        encoder_hidden_states: mindspore.Tensor = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, SeamlessM4Tv2TextToUnitDecoderOutput]:
        r"""
        Args:
            char_input_ids (`mindspore.Tensor` of shape `(batch_size, char_sequence_length)`):
                Character indices. The correspondence between characters and indices can be found in `char_to_id`, a
                dictionary in the generation configuration.
            char_count_per_id (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`):
                Number of characters per text input id.
            encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                of the decoder.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # create padding mask for character lengths
        char_padding_mask = _compute_new_attention_mask(char_input_ids, char_count_per_id.sum(1))

        # upsample hidden states according to characters sequence lengths
        char_hidden_states = self._hard_upsample(encoder_hidden_states, char_count_per_id)
        # embed char positions
        char_positions = self.pos_emb_alpha_char * self.embed_char_positions(inputs_embeds=char_hidden_states)
        # update char hidden states with positions and char embeddings
        char_hidden_states = self.embed_char(char_input_ids) * self.embed_scale + char_positions + char_hidden_states

        # predict duration
        log_dur_pred = self.duration_predictor(char_hidden_states, padding_mask=char_padding_mask)
        dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
        dur_out = dur_out.masked_fill(~char_padding_mask.bool(), 0.0)

        # upsample char hidden states according to predicted duration
        char_hidden_states = self._hard_upsample(char_hidden_states, dur_out)

        positions = self.pos_emb_alpha * self.embed_positions(inputs_embeds=char_hidden_states)
        hidden_states = char_hidden_states + positions

        padding_mask = _compute_new_attention_mask(hidden_states, dur_out.sum(1))
        attention_mask = _prepare_4d_attention_mask(padding_mask, hidden_states.dtype)

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None

        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = ops.rand([])
                if dropout_probability < self.layerdrop:
                    continue

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    attention_mask,
                    padding_mask,
                    output_attentions,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=attention_mask,
                    padding_mask=padding_mask,
                    output_attentions=output_attentions,
                )
            hidden_states = layer_outputs[0]

            if output_attentions:
                all_self_attns += (layer_outputs[2],)

        hidden_states = self.layer_norm(hidden_states)

        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attns, padding_mask] if v is not None)
        return SeamlessM4Tv2TextToUnitDecoderOutput(
            last_hidden_state=hidden_states,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            padding_mask=padding_mask,
        )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitDecoder.forward(char_input_ids=None, char_count_per_id=None, encoder_hidden_states=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
char_input_ids	Character indices. The correspondence between characters and indices can be found in char_to_id, a dictionary in the generation configuration. TYPE: `mindspore.Tensor` of shape `(batch_size, char_sequence_length)` DEFAULT: None
char_count_per_id	Number of characters per text input id. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)` DEFAULT: None
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    char_input_ids: mindspore.Tensor = None,
    char_count_per_id: mindspore.Tensor = None,
    encoder_hidden_states: mindspore.Tensor = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, SeamlessM4Tv2TextToUnitDecoderOutput]:
    r"""
    Args:
        char_input_ids (`mindspore.Tensor` of shape `(batch_size, char_sequence_length)`):
            Character indices. The correspondence between characters and indices can be found in `char_to_id`, a
            dictionary in the generation configuration.
        char_count_per_id (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`):
            Number of characters per text input id.
        encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # create padding mask for character lengths
    char_padding_mask = _compute_new_attention_mask(char_input_ids, char_count_per_id.sum(1))

    # upsample hidden states according to characters sequence lengths
    char_hidden_states = self._hard_upsample(encoder_hidden_states, char_count_per_id)
    # embed char positions
    char_positions = self.pos_emb_alpha_char * self.embed_char_positions(inputs_embeds=char_hidden_states)
    # update char hidden states with positions and char embeddings
    char_hidden_states = self.embed_char(char_input_ids) * self.embed_scale + char_positions + char_hidden_states

    # predict duration
    log_dur_pred = self.duration_predictor(char_hidden_states, padding_mask=char_padding_mask)
    dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
    dur_out = dur_out.masked_fill(~char_padding_mask.bool(), 0.0)

    # upsample char hidden states according to predicted duration
    char_hidden_states = self._hard_upsample(char_hidden_states, dur_out)

    positions = self.pos_emb_alpha * self.embed_positions(inputs_embeds=char_hidden_states)
    hidden_states = char_hidden_states + positions

    padding_mask = _compute_new_attention_mask(hidden_states, dur_out.sum(1))
    attention_mask = _prepare_4d_attention_mask(padding_mask, hidden_states.dtype)

    hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None

    for idx, decoder_layer in enumerate(self.layers):
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        if output_hidden_states:
            all_hidden_states += (hidden_states,)
        if self.training:
            dropout_probability = ops.rand([])
            if dropout_probability < self.layerdrop:
                continue

        if self.gradient_checkpointing and self.training:
            layer_outputs = self._gradient_checkpointing_func(
                decoder_layer.__call__,
                hidden_states,
                attention_mask,
                padding_mask,
                output_attentions,
            )
        else:
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                padding_mask=padding_mask,
                output_attentions=output_attentions,
            )
        hidden_states = layer_outputs[0]

        if output_attentions:
            all_self_attns += (layer_outputs[2],)

    hidden_states = self.layer_norm(hidden_states)

    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, all_hidden_states, all_self_attns, padding_mask] if v is not None)
    return SeamlessM4Tv2TextToUnitDecoderOutput(
        last_hidden_state=hidden_states,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
        padding_mask=padding_mask,
    )

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitDecoderLayer

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

class SeamlessM4Tv2TextToUnitDecoderLayer(nn.Module):
    def __init__(self, config: SeamlessM4Tv2Config, decoder_ffn_dim=None, decoder_attention_heads=None):
        super().__init__()
        decoder_ffn_dim = config.decoder_ffn_dim if decoder_ffn_dim is None else decoder_ffn_dim
        decoder_attention_heads = (
            config.decoder_attention_heads if decoder_attention_heads is None else decoder_attention_heads
        )
        self.dropout = config.dropout
        self.embed_dim = config.hidden_size

        self.self_attn = SeamlessM4Tv2Attention(
            embed_dim=self.embed_dim,
            num_heads=decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
        )
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)

        self.conv1 = nn.Conv1d(self.embed_dim, self.embed_dim, kernel_size=7, stride=1, padding="same")
        self.activation_fn = ACT2FN[config.activation_function]
        self.conv2 = nn.Conv1d(self.embed_dim, self.embed_dim, kernel_size=7, stride=1, padding="same")

        self.conv_layer_norm = nn.LayerNorm(config.hidden_size)
        self.conv_dropout = nn.Dropout(self.dropout)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        padding_mask: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = False,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`):
                attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
                large negative values.
            padding_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked*
                or 0 for *masked*
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states

        # Self Attention
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = residual + hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        # Conv
        residual = hidden_states

        # Apply padding mask to avoid leaking padded positions in the convolution layer
        if padding_mask is not None:
            hidden_states = hidden_states.masked_fill(~padding_mask.bool().unsqueeze(-1), 0.0)
        hidden_states = self.conv1(hidden_states.swapaxes(1, 2)).swapaxes(1, 2)

        if padding_mask is not None:
            hidden_states = hidden_states.masked_fill(~padding_mask.bool().unsqueeze(-1), 0.0)

        hidden_states = self.activation_fn(hidden_states)
        hidden_states = self.conv2(hidden_states.swapaxes(1, 2)).swapaxes(1, 2)

        hidden_states = self.conv_dropout(hidden_states)
        hidden_states = residual + hidden_states
        hidden_states = self.conv_layer_norm(hidden_states)

        outputs = (hidden_states, present_key_value)

        if output_attentions:
            outputs += self_attn_weights

        return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitDecoderLayer.forward(hidden_states, attention_mask=None, padding_mask=None, output_attentions=False)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
padding_mask	Indicates which inputs are to be ignored due to padding, where elements are either 1 for not masked or 0 for masked TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    padding_mask: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = False,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`):
            input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`):
            attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
            large negative values.
        padding_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked*
            or 0 for *masked*
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states

    # Self Attention
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        output_attentions=output_attentions,
    )
    hidden_states = residual + hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)

    # Conv
    residual = hidden_states

    # Apply padding mask to avoid leaking padded positions in the convolution layer
    if padding_mask is not None:
        hidden_states = hidden_states.masked_fill(~padding_mask.bool().unsqueeze(-1), 0.0)
    hidden_states = self.conv1(hidden_states.swapaxes(1, 2)).swapaxes(1, 2)

    if padding_mask is not None:
        hidden_states = hidden_states.masked_fill(~padding_mask.bool().unsqueeze(-1), 0.0)

    hidden_states = self.activation_fn(hidden_states)
    hidden_states = self.conv2(hidden_states.swapaxes(1, 2)).swapaxes(1, 2)

    hidden_states = self.conv_dropout(hidden_states)
    hidden_states = residual + hidden_states
    hidden_states = self.conv_layer_norm(hidden_states)

    outputs = (hidden_states, present_key_value)

    if output_attentions:
        outputs += self_attn_weights

    return outputs

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitDecoderOutput dataclass

Bases: ModelOutput

Class defining the outputs from [SeamlessM4Tv2TextToUnitDecoder].

PARAMETER	DESCRIPTION
last_hidden_state	Sequence of hidden-states at the output of the last layer of the model. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)` DEFAULT: None
hidden_states	Tuple of mindspore.Tensor (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size). Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None
attentions	Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length). Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None
padding_mask	Indicates which inputs are to be ignored due to padding, where elements are either 1 for not masked or 0 for masked TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@dataclass
class SeamlessM4Tv2TextToUnitDecoderOutput(ModelOutput):
    """
    Class defining the outputs from [`SeamlessM4Tv2TextToUnitDecoder`].

    Args:
        last_hidden_state (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        padding_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0
            for *masked*
    """

    last_hidden_state: mindspore.Tensor = None
    hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    attentions: Optional[Tuple[mindspore.Tensor]] = None
    padding_mask: Optional[mindspore.Tensor] = None

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.SeamlessM4Tv2TextToUnitOutput dataclass

Bases: ModelOutput

Class defining the outputs from [`SeamlessM4Tv2TextToUnitForConditionalGeneration`] and
[`SeamlessM4Tv2TextToUnitModel`].

PARAMETER	DESCRIPTION
last_hidden_state	Sequence of hidden-states at the output of the last layer of the decoder of the model. If past_key_values is used only the last hidden-state of the sequences of shape (batch_size, 1, hidden_size) is output. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)` DEFAULT: None
padding_mask	Indicates which inputs are to be ignored due to padding, where elements are either 1 for not masked or 0 for masked TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
decoder_hidden_states	Tuple of mindspore.Tensor (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size). Hidden-states of the decoder at the output of each layer plus the optional initial embedding outputs. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None
decoder_attentions	Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length). Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the self-attention heads. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None
encoder_last_hidden_state	Sequence of hidden-states at the output of the last layer of the encoder of the model. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
encoder_hidden_states	Tuple of mindspore.Tensor (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size). Hidden-states of the encoder at the output of each layer plus the optional initial embedding outputs. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None
encoder_attentions	Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length). Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the self-attention heads. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None
loss	Language modeling loss. TYPE: `mindspore.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

@dataclass
class SeamlessM4Tv2TextToUnitOutput(ModelOutput):
    """
        Class defining the outputs from [`SeamlessM4Tv2TextToUnitForConditionalGeneration`] and
        [`SeamlessM4Tv2TextToUnitModel`].

    Args:
        last_hidden_state (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the decoder of the model.

            If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
            hidden_size)` is output.
        padding_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0
            for *masked*
        decoder_hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the decoder at the output of each layer plus the optional initial embedding outputs.
        decoder_attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
            self-attention heads.
        encoder_last_hidden_state (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder of the model.
        encoder_hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the encoder at the output of each layer plus the optional initial embedding outputs.
        encoder_attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
            self-attention heads.
        loss (`mindspore.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss.
    """

    last_hidden_state: mindspore.Tensor = None
    padding_mask: Optional[mindspore.Tensor] = None
    decoder_hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    decoder_attentions: Optional[Tuple[mindspore.Tensor]] = None
    encoder_last_hidden_state: Optional[mindspore.Tensor] = None
    encoder_hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    encoder_attentions: Optional[Tuple[mindspore.Tensor]] = None
    loss: Optional[mindspore.Tensor] = None

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0)

Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's utils.make_positions.

PARAMETER	DESCRIPTION
x	mindspore.Tensor x:

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0):
    """
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: mindspore.Tensor x:

    Returns: mindspore.Tensor
    """
    # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
    mask = input_ids.ne(padding_idx).int()
    incremental_indices = (ops.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask
    return incremental_indices.long() + padding_idx

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.format_speech_generation_kwargs(kwargs)

Format kwargs for SeamlessM4Tv2 models that generate speech, attribute kwargs to either the text generation or the speech generation models.

PARAMETER

DESCRIPTION

kwargs

Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: `dict`)`

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def format_speech_generation_kwargs(kwargs):
    """
    Format kwargs for SeamlessM4Tv2 models that generate speech, attribute kwargs to either the text generation or the
    speech generation models.

    Args:
        kwargs (`dict`)`:
             Keyword arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.
    """
    # attribute kwargs to models
    kwargs_text = {}
    kwargs_speech = {}
    for key, value in kwargs.items():
        if key.startswith("text_"):
            key = key[len("text_") :]
            kwargs_text[key] = value
        elif key.startswith("speech_"):
            key = key[len("speech_") :]
            kwargs_speech[key] = value
        else:
            # If the key is already in a specific config, then it's been set with a
            # submodules specific value and we don't override
            if key not in kwargs_text:
                kwargs_text[key] = value
            if key not in kwargs_speech:
                kwargs_speech[key] = value
    return kwargs_text, kwargs_speech

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.pad_sequence(sequences, batch_first=False, padding_value=0.0)

Pad a list of sequences to the same length.

PARAMETER	DESCRIPTION
sequences	The list of sequences to be padded. TYPE: List[List[float]]
batch_first	If True, the output tensor will have shape (batch_size, max_len, features). If False, the shape will be (max_len, batch_size, features). Default is False. TYPE: bool DEFAULT: False
padding_value	The value used for padding. Default is 0.0. TYPE: float DEFAULT: 0.0

RETURNS	DESCRIPTION
	mindspore.Tensor: A tensor containing the padded sequences.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def pad_sequence(sequences, batch_first=False, padding_value=0.0):
    """
    Pad a list of sequences to the same length.

    Args:
        sequences (List[List[float]]): The list of sequences to be padded.
        batch_first (bool, optional): If True, the output tensor will have shape (batch_size, max_len, features).
            If False, the shape will be (max_len, batch_size, features). Default is False.
        padding_value (float, optional): The value used for padding. Default is 0.0.

    Returns:
        mindspore.Tensor: A tensor containing the padded sequences.

    Raises:
        None.
    """
    # Determine the maximum sequence length
    max_len = max(len(seq) for seq in sequences)

    # Pad each sequence using cp.pad
    padded_sequences = [ops.pad(seq, (0, 0, 0, max_len - len(seq)), mode='constant', value=padding_value) for seq in sequences]
    # Stack the padded sequences along the appropriate axis
    if batch_first:
        padded_sequence = ops.stack(padded_sequences)
    else:
        padded_sequence = ops.stack(padded_sequences, 1)
    return padded_sequence

mindnlp.transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2.shift_tokens_right(input_ids, pad_token_id, decoder_start_token_id)

Shift input ids one token to the right.

Source code in mindnlp\transformers\models\seamless_m4t_v2\modeling_seamless_m4t_v2.py

def shift_tokens_right(input_ids: mindspore.Tensor, pad_token_id: int, decoder_start_token_id: int):
    """
    Shift input ids one token to the right.
    """
    shifted_input_ids = ops.zeros(input_ids.shape, dtype=input_ids.dtype)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].copy()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    # replace possible -100 values in labels by `pad_token_id`
    shifted_input_ids = shifted_input_ids.masked_fill(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids

mindnlp.transformers.models.seamless_m4t_v2.configuration_seamless_m4t_v2

SeamlessM4Tv2 model configuration

mindnlp.transformers.models.seamless_m4t_v2.configuration_seamless_m4t_v2.SeamlessM4Tv2Config

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [~SeamlessM4Tv2Model]. It is used to instantiate an SeamlessM4Tv2 model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the SeamlessM4Tv2 "" architecture.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the text modality of the SeamlessM4Tv2 model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [~SeamlessM4Tv2Model], [~SeamlessM4Tv2ForTextToSpeech] or [~SeamlessM4Tv2ForTextToText]. TYPE: `int`, *optional*, defaults to 256102 DEFAULT: 256102
t2u_vocab_size	Unit vocabulary size of the SeamlessM4Tv2 model. Defines the number of different "unit tokens" that can be represented by the inputs_ids passed when calling the Text-To-Units sub-model of [~SeamlessM4Tv2Model], [~SeamlessM4Tv2ForSpeechToSpeech] or [~SeamlessM4Tv2ForTextToSpeech]. TYPE: `int`, *optional*, defaults to 10082 DEFAULT: 10082
char_vocab_size	Character vocabulary size of the SeamlessM4Tv2 model. Defines the number of different character tokens that can be represented by the char_inputs_ids passed when calling the Text-To-Units sub-model of [~SeamlessM4Tv2Model], [~SeamlessM4Tv2ForSpeechToSpeech] or [~SeamlessM4Tv2ForTextToSpeech]. TYPE: `int`, *optional*, defaults to 10943 DEFAULT: 10943
Parameters	param below are Parameters shared across sub-models TYPE: shared across sub-models
hidden_size	Dimensionality of the "intermediate" layers in the architecture. TYPE: `int`, *optional*, defaults to 1024 DEFAULT: 1024
initializer_range	The standard deviation of the truncated_normal_initializer for initializing all weight matrices. TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02
layer_norm_eps	The epsilon used by the layer normalization layers. TYPE: `float`, *optional*, defaults to 1e-05 DEFAULT: 1e-05
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
max_position_embeddings	The maximum sequence length that this model text encoder and decoder might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
is_encoder_decoder	Whether the model is used as an encoder/decoder or not. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
encoder_layerdrop	The LayerDrop probability for the encoders. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.05 DEFAULT: 0.05
decoder_layerdrop	The LayerDrop probability for the decoders. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.05 DEFAULT: 0.05
activation_function	The non-linear activation function (function or string) in the decoder and feed-forward layers. If string, "gelu", "relu", "selu", "swish" and "gelu_new" are supported. TYPE: `str` or `function`, *optional*, defaults to `"relu"` DEFAULT: 'relu'
dropout	The dropout probability for all fully connected layers in the embeddings, encoder, decoder, and pooler. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
attention_dropout	The dropout probability for all attention layers. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
activation_dropout	The dropout probability for all activation layers in the model. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
scale_embedding	Scale embeddings by diving by sqrt(d_model). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
Text	param below are Text encoder and text decoder specific parameters TYPE: encoder and text decoder specific parameters
encoder_layers	Number of hidden layers in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
encoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
encoder_attention_heads	Number of attention heads for each attention layer in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
decoder_layers	Number of hidden layers in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
decoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
decoder_attention_heads	Number of attention heads for each attention layer in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
decoder_start_token_id	If an encoder-decoder model starts decoding with a different token than bos, the id of that token. Only applied in the text decoder. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
max_new_tokens	The maximum numbers of text tokens to generate, ignoring the number of tokens in the prompt. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
pad_token_id	The id of the padding text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
bos_token_id	The id of the beginning-of-stream text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
eos_token_id	The id of the end-of-stream text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
Speech	param below are Speech encoder specific parameters TYPE: encoder specific parameters
speech_encoder_layers	Number of hidden layers in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
speech_encoder_attention_heads	Number of attention heads for each attention layer in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
speech_encoder_intermediate_size	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
speech_encoder_hidden_act	The non-linear activation function (function or string) in the speech encoder. If string, "gelu", "relu", "selu", "swish" and "gelu_new" are supported. TYPE: `str` or `function`, *optional*, defaults to `"swish"` DEFAULT: 'swish'
speech_encoder_dropout	The dropout probability for all layers in the speech encoder. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
add_adapter	Add an adapter layer on top of the speech encoder. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
speech_encoder_layerdrop	The LayerDrop probability for the speech encoder. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
feature_projection_input_dim	Input dimension of the input feature projection of the speech encoder, i.e the dimension after processing input audios with [SeamlessM4TFeatureExtractor]. TYPE: `int`, *optional*, defaults to 160 DEFAULT: 160
adaptor_kernel_size	Kernel size of the convolutional layers in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
adaptor_stride	Stride of the convolutional layers in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
adaptor_dropout	The dropout probability for all layers in the speech adapter. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
num_adapter_layers	Number of convolutional layers that should be used in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
position_embeddings_type	Can be specified to relative_key. If left to None, no relative position embedding is applied. Only applied to the speech encoder. For more information on "relative_key", please refer to Self-Attention with Relative Position Representations (Shaw et al.). TYPE: `str`, *optional*, defaults to `"relative_key"` DEFAULT: 'relative_key'
conv_depthwise_kernel_size	Kernel size of convolutional depthwise 1D layer in Conformer blocks. Only applied to the speech encoder. TYPE: `int`, *optional*, defaults to 31 DEFAULT: 31
left_max_position_embeddings	The left clipping value for relative positions. TYPE: `int`, *optional*, defaults to 64 DEFAULT: 64
right_max_position_embeddings	The right clipping value for relative positions. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
speech_encoder_chunk_size	The size of each attention chunk. TYPE: `int`, *optional*, defaults to 20000 DEFAULT: 20000
speech_encoder_left_chunk_num	Number of chunks on the left up to which lookahead is allowed. TYPE: `int`, *optional*, defaults to 128 DEFAULT: 128
Text-To-Unit	param below are Text-To-Unit (t2u) model specific parameters TYPE: t2u) model specific parameters
t2u_bos_token_id	The id of the beginning-of-stream unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
t2u_pad_token_id	The id of the padding unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
t2u_eos_token_id	The id of the end-of-stream unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
t2u_encoder_layers	Number of hidden layers in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
t2u_encoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
t2u_encoder_attention_heads	Number of attention heads for each attention layer in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
t2u_decoder_layers	Number of hidden layers in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
t2u_decoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
t2u_decoder_attention_heads	Number of attention heads for each attention layer in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
t2u_max_position_embeddings	The maximum sequence length that this model text-to-unit component might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
t2u_variance_predictor_embed_dim	The projection dimension of the text-to-unit's duration predictor. TYPE: `int`, *optional*, defaults to 1024 DEFAULT: 1024
t2u_variance_predictor_hidden_dim	Internal dimension of the text-to-unit's duration predictor. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
t2u_variance_predictor_kernel_size	Kernel size of the convolutional layers of the text-to-unit's duration predictor. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
t2u_variance_pred_dropout	The dropout probabilitiy of the text-to-unit's duration predictor. Hifi-Gan Vocoder specific parameters: param below are Hifi-Gan Vocoder specific parameters TYPE: `float`, *optional*, defaults to 0.5 DEFAULT: 0.5
sampling_rate	The sampling rate at which the output audio will be generated, expressed in hertz (Hz). TYPE: `int`, *optional*, defaults to 16000 DEFAULT: 16000
upsample_initial_channel	The number of input channels into the hifi-gan upsampling network. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 512 DEFAULT: 512
upsample_rates	A tuple of integers defining the stride of each 1D convolutional layer in the vocoder upsampling network. The length of upsample_rates defines the number of convolutional layers and has to match the length of upsample_kernel_sizes. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[5, 4, 4, 2, 2]` DEFAULT: [5, 4, 4, 2, 2]
upsample_kernel_sizes	A tuple of integers defining the kernel size of each 1D convolutional layer in the vocoder upsampling network. The length of upsample_kernel_sizes defines the number of convolutional layers and has to match the length of upsample_rates. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[11, 8, 8, 4, 4]` DEFAULT: [11, 8, 8, 4, 4]
resblock_kernel_sizes	A tuple of integers defining the kernel sizes of the vocoder 1D convolutional layers in the multi-receptive field fusion (MRF) module. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[3, 7, 11]` DEFAULT: [3, 7, 11]
resblock_dilation_sizes	A nested tuple of integers defining the dilation rates of the vocoder dilated 1D convolutional layers in the multi-receptive field fusion (MRF) module. Applies to the vocoder only. TYPE: `Tuple[Tuple[int]]` or `List[List[int]]`, *optional*, defaults to `[[1, 3, 5], [1, 3, 5], [1, 3, 5]]` DEFAULT: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
leaky_relu_slope	The angle of the negative slope used by the leaky ReLU activation in the vocoder. Applies to the vocoder only. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
unit_hifi_gan_vocab_size	Vocabulary size of the SeamlessM4Tv2 vocoder. Defines the number of different unit tokens that can be represented by the inputs_ids passed when calling the vocoder of [~SeamlessM4Tv2Model], [~SeamlessM4Tv2ForSpeechToSpeech] or [~SeamlessM4Tv2ForTextToSpeech]. TYPE: `int`, *optional*, defaults to 10000 DEFAULT: 10000
unit_embed_dim	The projection dimension of the input ids given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 1280 DEFAULT: 1280
lang_embed_dim	The projection dimension of the target language given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
spkr_embed_dim	The projection dimension of the speaker id given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
vocoder_num_langs	Number of langs supported by the vocoder. Might be different from t2u_num_langs. TYPE: `int`, *optional*, defaults to 36 DEFAULT: 36
vocoder_num_spkrs	Number of speakers supported by the vocoder. TYPE: `int`, *optional*, defaults to 200 DEFAULT: 200
variance_predictor_kernel_size	Kernel size of the duration predictor. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
var_pred_dropout	The dropout probabilitiy of the duration predictor. Applies to the vocoder only. TYPE: `float`, *optional*, defaults to 0.5 DEFAULT: 0.5
vocoder_offset	Offset the unit token ids by this number to account for symbol tokens. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 4 DEFAULT: 4

Example

>>> from transformers import SeamlessM4Tv2Model, SeamlessM4Tv2Config
...
>>> # Initializing a SeamlessM4Tv2 "" style configuration
>>> configuration = SeamlessM4Tv2Config()
...
>>> # Initializing a model from the "" style configuration
>>> model = SeamlessM4Tv2Model(configuration)
...
>>> # Accessing the model configuration
>>> configuration = model.config

Source code in mindnlp\transformers\models\seamless_m4t_v2\configuration_seamless_m4t_v2.py

class SeamlessM4Tv2Config(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`~SeamlessM4Tv2Model`]. It is used to instantiate
    an SeamlessM4Tv2 model according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the SeamlessM4Tv2
    [""](https://hf-mirror.com/"") architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 256102):
            Vocabulary size of the text modality of the SeamlessM4Tv2 model. Defines the number of different tokens
            that can be represented by the `inputs_ids` passed when calling [`~SeamlessM4Tv2Model`],
            [`~SeamlessM4Tv2ForTextToSpeech`] or [`~SeamlessM4Tv2ForTextToText`].
        t2u_vocab_size (`int`, *optional*, defaults to 10082):
            Unit vocabulary size of the SeamlessM4Tv2 model. Defines the number of different "unit tokens" that can be
            represented by the `inputs_ids` passed when calling the Text-To-Units sub-model of [`~SeamlessM4Tv2Model`],
            [`~SeamlessM4Tv2ForSpeechToSpeech`] or [`~SeamlessM4Tv2ForTextToSpeech`].
        char_vocab_size (`int`, *optional*, defaults to 10943):
            Character vocabulary size of the SeamlessM4Tv2 model. Defines the number of different character tokens that
            can be represented by the `char_inputs_ids` passed when calling the Text-To-Units sub-model of
            [`~SeamlessM4Tv2Model`], [`~SeamlessM4Tv2ForSpeechToSpeech`] or [`~SeamlessM4Tv2ForTextToSpeech`].

        Parameters shared across sub-models: param below are Parameters shared across sub-models

        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the "intermediate" layers in the architecture.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
        max_position_embeddings (`int`, *optional*, defaults to 4096):
            The maximum sequence length that this model text encoder and decoder might ever be used with. Typically set
            this to something large just in case (e.g., 512 or 1024 or 2048).
        is_encoder_decoder (`bool`, *optional*, defaults to `True`):
            Whether the model is used as an encoder/decoder or not.
        encoder_layerdrop (`float`, *optional*, defaults to 0.05):
            The LayerDrop probability for the encoders. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        decoder_layerdrop (`float`, *optional*, defaults to 0.05):
            The LayerDrop probability for the decoders. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        activation_function (`str` or `function`, *optional*, defaults to `"relu"`):
            The non-linear activation function (function or string) in the decoder and feed-forward layers. If string,
            `"gelu"`, `"relu"`, `"selu"`, `"swish"` and `"gelu_new"` are supported.
        dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, decoder, and pooler.
        attention_dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all attention layers.
        activation_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for all activation layers in the model.
        scale_embedding (`bool`, *optional*, defaults to `True`):
            Scale embeddings by diving by sqrt(d_model).

        Text encoder and text decoder specific parameters: param below are Text encoder and text decoder specific parameters

        encoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer text encoder.
        encoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text encoder.
        encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text encoder.
        decoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer text decoder.
        decoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text decoder.
        decoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text decoder.
        decoder_start_token_id (`int`, *optional*, defaults to 3):
            If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token. Only
            applied in the text decoder.
        max_new_tokens (`int`, *optional*, defaults to 256):
            The maximum numbers of text tokens to generate, ignoring the number of tokens in the prompt.
        pad_token_id (`int`, *optional*, defaults to 0):
            The id of the _padding_ text token. Only applied to the text-decoder model.
        bos_token_id (`int`, *optional*, defaults to 2):
            The id of the _beginning-of-stream_ text token. Only applied to the text-decoder model.
        eos_token_id (`int`, *optional*, defaults to 3):
            The id of the _end-of-stream_ text token. Only applied to the text-decoder model.

        Speech encoder specific parameters: param below are Speech encoder specific parameters

        speech_encoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer speech encoder.
        speech_encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer speech encoder.
        speech_encoder_intermediate_size (`int`, *optional*, defaults to 4096):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer speech encoder.
        speech_encoder_hidden_act (`str` or `function`, *optional*, defaults to `"swish"`):
            The non-linear activation function (function or string) in the speech encoder. If string, `"gelu"`,
            `"relu"`, `"selu"`, `"swish"` and `"gelu_new"` are supported.
        speech_encoder_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for all layers in the speech encoder.
        add_adapter (`bool`, *optional*, defaults to `True`):
            Add an adapter layer on top of the speech encoder.
        speech_encoder_layerdrop (`float`, *optional*, defaults to 0.1):
            The LayerDrop probability for the speech encoder. See the [LayerDrop paper](see
            https://arxiv.org/abs/1909.11556) for more details.
        feature_projection_input_dim (`int`, *optional*, defaults to 160):
            Input dimension of the input feature projection of the speech encoder, i.e the dimension after processing
            input audios with [`SeamlessM4TFeatureExtractor`].
        adaptor_kernel_size (`int`, *optional*, defaults to 8):
            Kernel size of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
        adaptor_stride (`int`, *optional*, defaults to 8):
            Stride of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
        adaptor_dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all layers in the speech adapter.
        num_adapter_layers (`int`, *optional*, defaults to 1):
            Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is
            True`.
        position_embeddings_type (`str`, *optional*, defaults to `"relative_key"`):
            Can be specified to `relative_key`. If left to `None`, no relative position embedding is applied. Only
            applied to the speech encoder. For more information on `"relative_key"`, please refer to [Self-Attention
            with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
        conv_depthwise_kernel_size (`int`, *optional*, defaults to 31):
            Kernel size of convolutional depthwise 1D layer in Conformer blocks. Only applied to the speech encoder.
        left_max_position_embeddings (`int`, *optional*, defaults to 64):
            The left clipping value for relative positions.
        right_max_position_embeddings (`int`, *optional*, defaults to 8):
            The right clipping value for relative positions.
        speech_encoder_chunk_size (`int`, *optional*, defaults to 20000): The size of each attention chunk.
        speech_encoder_left_chunk_num (`int`, *optional*, defaults to 128):
            Number of chunks on the left up to which lookahead is allowed.

        Text-To-Unit (t2u) model specific parameters:  param below are Text-To-Unit (t2u) model specific parameters

        t2u_bos_token_id (`int`, *optional*, defaults to 0):
            The id of the _beginning-of-stream_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_pad_token_id (`int`, *optional*, defaults to 1):
            The id of the _padding_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_eos_token_id (`int`, *optional*, defaults to 2):
            The id of the _end-of-stream_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_encoder_layers (`int`, *optional*, defaults to 6):
            Number of hidden layers in the Transformer text-to-unit encoder.
        t2u_encoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit encoder.
        t2u_encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text-to-unit encoder.
        t2u_decoder_layers (`int`, *optional*, defaults to 6):
            Number of hidden layers in the Transformer text-to-unit decoder.
        t2u_decoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit decoder.
        t2u_decoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text-to-unit decoder.
        t2u_max_position_embeddings (`int`, *optional*, defaults to 4096):
            The maximum sequence length that this model text-to-unit component might ever be used with. Typically set
            this to something large just in case (e.g., 512 or 1024 or 2048).
        t2u_variance_predictor_embed_dim (`int`, *optional*, defaults to 1024):
            The projection dimension of the text-to-unit's duration predictor.
        t2u_variance_predictor_hidden_dim (`int`, *optional*, defaults to 256):
            Internal dimension of the text-to-unit's duration predictor.
        t2u_variance_predictor_kernel_size (`int`, *optional*, defaults to 3):
            Kernel size of the convolutional layers of the text-to-unit's duration predictor.
        t2u_variance_pred_dropout (`float`, *optional*, defaults to 0.5):
            The dropout probabilitiy of the text-to-unit's duration predictor.

         Hifi-Gan Vocoder specific parameters: param below are Hifi-Gan Vocoder specific parameters

        sampling_rate (`int`, *optional*, defaults to 16000):
            The sampling rate at which the output audio will be generated, expressed in hertz (Hz).
        upsample_initial_channel (`int`, *optional*, defaults to 512):
            The number of input channels into the hifi-gan upsampling network. Applies to the vocoder only.
        upsample_rates (`Tuple[int]` or `List[int]`, *optional*, defaults to `[5, 4, 4, 2, 2]`):
            A tuple of integers defining the stride of each 1D convolutional layer in the vocoder upsampling network.
            The length of *upsample_rates* defines the number of convolutional layers and has to match the length of
            *upsample_kernel_sizes*. Applies to the vocoder only.
        upsample_kernel_sizes (`Tuple[int]` or `List[int]`, *optional*, defaults to `[11, 8, 8, 4, 4]`):
            A tuple of integers defining the kernel size of each 1D convolutional layer in the vocoder upsampling
            network. The length of *upsample_kernel_sizes* defines the number of convolutional layers and has to match
            the length of *upsample_rates*. Applies to the vocoder only.
        resblock_kernel_sizes (`Tuple[int]` or `List[int]`, *optional*, defaults to `[3, 7, 11]`):
            A tuple of integers defining the kernel sizes of the vocoder 1D convolutional layers in the multi-receptive
            field fusion (MRF) module. Applies to the vocoder only.
        resblock_dilation_sizes (`Tuple[Tuple[int]]` or `List[List[int]]`, *optional*, defaults to `[[1, 3, 5], [1, 3, 5], [1, 3, 5]]`):
            A nested tuple of integers defining the dilation rates of the vocoder dilated 1D convolutional layers in
            the multi-receptive field fusion (MRF) module. Applies to the vocoder only.
        leaky_relu_slope (`float`, *optional*, defaults to 0.1):
            The angle of the negative slope used by the leaky ReLU activation in the vocoder. Applies to the vocoder
            only.
        unit_hifi_gan_vocab_size (`int`, *optional*, defaults to 10000):
            Vocabulary size of the SeamlessM4Tv2 vocoder. Defines the number of different unit tokens that can be
            represented by the `inputs_ids` passed when calling the vocoder of [`~SeamlessM4Tv2Model`],
            [`~SeamlessM4Tv2ForSpeechToSpeech`] or [`~SeamlessM4Tv2ForTextToSpeech`].
        unit_embed_dim (`int`, *optional*, defaults to 1280):
            The projection dimension of the input ids given to the hifi-gan vocoder. Applies to the vocoder only.
        lang_embed_dim (`int`, *optional*, defaults to 256):
            The projection dimension of the target language given to the hifi-gan vocoder. Applies to the vocoder only.
        spkr_embed_dim (`int`, *optional*, defaults to 256):
            The projection dimension of the speaker id given to the hifi-gan vocoder. Applies to the vocoder only.
        vocoder_num_langs (`int`, *optional*, defaults to 36):
            Number of langs supported by the vocoder. Might be different from `t2u_num_langs`.
        vocoder_num_spkrs (`int`, *optional*, defaults to 200):
            Number of speakers supported by the vocoder.
        variance_predictor_kernel_size (`int`, *optional*, defaults to 3):
            Kernel size of the duration predictor. Applies to the vocoder only.
        var_pred_dropout (`float`, *optional*, defaults to 0.5):
            The dropout probabilitiy of the duration predictor. Applies to the vocoder only.
        vocoder_offset (`int`, *optional*, defaults to 4):
            Offset the unit token ids by this number to account for symbol tokens. Applies to the vocoder only.

    Example:
        ```python
        >>> from transformers import SeamlessM4Tv2Model, SeamlessM4Tv2Config
        ...
        >>> # Initializing a SeamlessM4Tv2 "" style configuration
        >>> configuration = SeamlessM4Tv2Config()
        ...
        >>> # Initializing a model from the "" style configuration
        >>> model = SeamlessM4Tv2Model(configuration)
        ...
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "seamless_m4t_v2"

    def __init__(
        self,
        vocab_size=256102,
        t2u_vocab_size=10082,
        char_vocab_size=10943,
        # shared config
        hidden_size=1024,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        use_cache=True,
        max_position_embeddings=4096,
        is_encoder_decoder=True,
        encoder_layerdrop=0.05,
        decoder_layerdrop=0.05,
        activation_function="relu",
        dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        scale_embedding=True,
        # text encoder|decoder
        encoder_layers=24,
        encoder_ffn_dim=8192,
        encoder_attention_heads=16,
        decoder_layers=24,
        decoder_ffn_dim=8192,
        decoder_attention_heads=16,
        decoder_start_token_id=3,
        max_new_tokens=256,
        pad_token_id=0,
        bos_token_id=2,
        eos_token_id=3,
        # speech_encoder
        speech_encoder_layers=24,
        speech_encoder_attention_heads=16,
        speech_encoder_intermediate_size=4096,
        speech_encoder_hidden_act="swish",
        speech_encoder_dropout=0.0,
        add_adapter=True,
        speech_encoder_layerdrop=0.1,
        feature_projection_input_dim=160,
        adaptor_kernel_size=8,
        adaptor_stride=8,
        adaptor_dropout=0.1,
        num_adapter_layers=1,
        position_embeddings_type="relative_key",
        conv_depthwise_kernel_size=31,
        left_max_position_embeddings=64,
        right_max_position_embeddings=8,
        speech_encoder_chunk_size=20000,
        speech_encoder_left_chunk_num=128,
        # t2u config
        t2u_bos_token_id=0,
        t2u_pad_token_id=1,
        t2u_eos_token_id=2,
        t2u_encoder_layers=6,
        t2u_encoder_ffn_dim=8192,
        t2u_encoder_attention_heads=16,
        t2u_decoder_layers=6,
        t2u_decoder_ffn_dim=8192,
        t2u_decoder_attention_heads=16,
        t2u_max_position_embeddings=4096,
        t2u_variance_predictor_embed_dim=1024,
        t2u_variance_predictor_hidden_dim=256,
        t2u_variance_predictor_kernel_size=3,
        t2u_variance_pred_dropout=0.5,
        # hifi-gan vocoder config
        sampling_rate=16000,
        upsample_initial_channel=512,
        upsample_rates=[5, 4, 4, 2, 2],
        upsample_kernel_sizes=[11, 8, 8, 4, 4],
        resblock_kernel_sizes=[3, 7, 11],
        resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
        leaky_relu_slope=0.1,
        # specific to Code Hifi-Gan
        unit_hifi_gan_vocab_size=10000,
        unit_embed_dim=1280,
        lang_embed_dim=256,
        spkr_embed_dim=256,
        vocoder_num_langs=36,
        vocoder_num_spkrs=200,
        variance_predictor_kernel_size=3,
        var_pred_dropout=0.5,
        vocoder_offset=4,
        **kwargs,
    ):
        '''
        Initializes a new instance of the SeamlessM4Tv2Config class.

        Args:
            self: The instance of the class.
            vocab_size (int, optional): The size of the vocabulary. Defaults to 256102.
            t2u_vocab_size (int, optional): The size of the text-to-unit vocabulary. Defaults to 10082.
            char_vocab_size (int, optional): The size of the character vocabulary. Defaults to 10943.
            hidden_size (int, optional): The size of the hidden layers. Defaults to 1024.
            initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
            layer_norm_eps (float, optional): The epsilon value for layer normalization. Defaults to 1e-05.
            use_cache (bool, optional): Whether to use cache. Defaults to True.
            max_position_embeddings (int, optional): The maximum number of position embeddings. Defaults to 4096.
            is_encoder_decoder (bool, optional): Whether it is an encoder-decoder model. Defaults to True.
            encoder_layerdrop (float, optional): The layerdrop probability for the encoder. Defaults to 0.05.
            decoder_layerdrop (float, optional): The layerdrop probability for the decoder. Defaults to 0.05.
            activation_function (str, optional): The activation function to use. Defaults to 'relu'.
            dropout (float, optional): The dropout probability. Defaults to 0.1.
            attention_dropout (float, optional): The dropout probability for attention layers. Defaults to 0.1.
            activation_dropout (float, optional): The dropout probability for activation layers. Defaults to 0.0.
            scale_embedding (bool, optional): Whether to scale the embeddings. Defaults to True.
            encoder_layers (int, optional): The number of encoder layers. Defaults to 24.
            encoder_ffn_dim (int, optional): The dimension of the encoder feed-forward network. Defaults to 8192.
            encoder_attention_heads (int, optional): The number of attention heads in the encoder. Defaults to 16.
            decoder_layers (int, optional): The number of decoder layers. Defaults to 24.
            decoder_ffn_dim (int, optional): The dimension of the decoder feed-forward network. Defaults to 8192.
            decoder_attention_heads (int, optional): The number of attention heads in the decoder. Defaults to 16.
            decoder_start_token_id (int, optional): The token ID for the start of decoding. Defaults to 3.
            max_new_tokens (int, optional): The maximum number of new tokens. Defaults to 256.
            pad_token_id (int, optional): The token ID for padding. Defaults to 0.
            bos_token_id (int, optional): The token ID for the beginning of sequence. Defaults to 2.
            eos_token_id (int, optional): The token ID for the end of sequence. Defaults to 3.
            speech_encoder_layers (int, optional): The number of speech encoder layers. Defaults to 24.
            speech_encoder_attention_heads (int, optional): The number of attention heads in the speech encoder. Defaults to 16.
            speech_encoder_intermediate_size (int, optional): The intermediate size of the speech encoder. Defaults to 4096.
            speech_encoder_hidden_act (str, optional): The activation function for the speech encoder. Defaults to 'swish'.
            speech_encoder_dropout (float, optional): The dropout probability for the speech encoder. Defaults to 0.0.
            add_adapter (bool, optional): Whether to add an adapter. Defaults to True.
            speech_encoder_layerdrop (float, optional): The layerdrop probability for the speech encoder. Defaults to 0.1.
            feature_projection_input_dim (int, optional): The input dimension for feature projection. Defaults to 160.
            adaptor_kernel_size (int, optional): The kernel size for the adaptor. Defaults to 8.
            adaptor_stride (int, optional): The stride for the adaptor. Defaults to 8.
            adaptor_dropout (float, optional): The dropout probability for the adaptor. Defaults to 0.1.
            num_adapter_layers (int, optional): The number of adapter layers. Defaults to 1.
            position_embeddings_type (str, optional): The type of position embeddings. Defaults to 'relative_key'.
            conv_depthwise_kernel_size (int, optional): The kernel size for depthwise convolution. Defaults to 31.
            left_max_position_embeddings (int, optional): The maximum number of left position embeddings. Defaults to 64.
            right_max_position_embeddings (int, optional): The maximum number of right position embeddings. Defaults to 8.
            speech_encoder_chunk_size (int, optional): The chunk size for the speech encoder. Defaults to 20000.
            speech_encoder_left_chunk_num (int, optional): The number of left chunks for the speech encoder. Defaults to 128.
            t2u_bos_token_id (int, optional): The token ID for the beginning of text-to-unit conversion. Defaults to 0.
            t2u_pad_token_id (int, optional): The token ID for padding in text-to-unit conversion. Defaults to 1.
            t2u_eos_token_id (int, optional): The token ID for the end of text-to-unit conversion. Defaults to 2.
            t2u_encoder_layers (int, optional): The number of text-to-unit encoder layers. Defaults to 6.
            t2u_encoder_ffn_dim (int, optional): The dimension of the text-to-unit encoder feed-forward network. Defaults to 8192.
            t2u_encoder_attention_heads (int, optional): The number of attention heads in the text-to-unit encoder. Defaults to 16.
            t2u_decoder_layers (int, optional): The number of text-to-unit decoder layers. Defaults to 6.
            t2u_decoder_ffn_dim (int, optional): The dimension of the text-to-unit decoder feed-forward network. Defaults to 8192.
            t2u_decoder_attention_heads (int, optional): The number of attention heads in the text-to-unit decoder. Defaults to 16.
            t2u_max_position_embeddings (int, optional): The maximum number of position embeddings for text-to-unit conversion. Defaults to 4096.
            t2u_variance_predictor_embed_dim (int, optional): The embedding dimension for the variance predictor in text-to-unit conversion. Defaults to 1024.
            t2u_variance_predictor_hidden_dim (int, optional): The hidden dimension for the variance predictor in text-to-unit conversion. Defaults to 256.
            t2u_variance_predictor_kernel_size (int, optional): The kernel size for the variance predictor in text-to-unit conversion. Defaults to 3.
            t2u_variance_pred_dropout (float, optional): The dropout probability for the variance predictor in text-to-unit conversion. Defaults to 0.5.
            sampling_rate (int, optional): The sampling rate of audio data. Defaults to 16000.
            upsample_initial_channel (int, optional): The initial number of channels for upsampling. Defaults to 512.
            upsample_rates (List[int], optional): The rates for upsampling. Defaults to [5, 4, 4, 2, 2].
            upsample_kernel_sizes (List[int], optional): The kernel sizes for upsampling. Defaults to [11, 8, 8, 4, 4].
            resblock_kernel_sizes (List[int], optional): The kernel sizes for residual blocks. Defaults to [3, 7, 11].
            resblock_dilation_sizes (List[List[int]], optional): The dilation sizes for residual blocks. Defaults to [[1, 3, 5], [1, 3, 5], [1, 3, 5]].
            leaky_relu_slope (float, optional): The slope for LeakyReLU activation. Defaults to 0.1.
            unit_hifi_gan_vocab_size (int, optional): The vocabulary size for the unit HiFi-GAN. Defaults to 10000.
            unit_embed_dim (int, optional): The embedding dimension for the unit HiFi-GAN. Defaults to 1280.
            lang_embed_dim (int, optional): The embedding dimension for language. Defaults to 256.
            spkr_embed_dim (int, optional): The embedding dimension for speaker. Defaults to 256.
            vocoder_num_langs (int, optional): The number of languages for the vocoder. Defaults to 36.
            vocoder_num_spkrs (int, optional): The number of speakers for the vocoder. Defaults to 200.
            variance_predictor_kernel_size (int, optional): The kernel size for the variance predictor. Defaults to 3.
            var_pred_dropout (float, optional): The dropout probability for the variance predictor. Defaults to 0.5.
            vocoder_offset (int, optional): The offset for the vocoder. Defaults to 4.

        Returns:
            None

        Raises:
            None
        '''
        # overall_config
        self.vocab_size = vocab_size
        self.t2u_vocab_size = t2u_vocab_size
        self.char_vocab_size = char_vocab_size
        self.hidden_size = hidden_size
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.max_position_embeddings = max_position_embeddings
        self.use_cache = use_cache
        self.max_new_tokens = max_new_tokens
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.activation_function = activation_function
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.scale_embedding = scale_embedding
        # for proper config init
        self.num_attention_heads = decoder_attention_heads
        self.num_hidden_layers = decoder_layers

        # text|unit encoder|decoder
        self.encoder_layers = encoder_layers
        self.encoder_ffn_dim = encoder_ffn_dim
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_layers = decoder_layers
        self.decoder_ffn_dim = decoder_ffn_dim
        self.decoder_attention_heads = decoder_attention_heads

        # speech_encoder
        self.speech_encoder_layers = speech_encoder_layers
        self.speech_encoder_hidden_act = speech_encoder_hidden_act
        self.speech_encoder_dropout = speech_encoder_dropout
        self.speech_encoder_attention_heads = speech_encoder_attention_heads
        self.speech_encoder_layerdrop = speech_encoder_layerdrop
        self.speech_encoder_intermediate_size = speech_encoder_intermediate_size
        self.feature_projection_input_dim = feature_projection_input_dim
        self.adaptor_kernel_size = adaptor_kernel_size
        self.adaptor_stride = adaptor_stride
        self.adaptor_dropout = adaptor_dropout
        self.num_adapter_layers = num_adapter_layers
        self.position_embeddings_type = position_embeddings_type
        self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
        self.add_adapter = add_adapter
        self.left_max_position_embeddings = left_max_position_embeddings
        self.right_max_position_embeddings = right_max_position_embeddings
        self.speech_encoder_chunk_size = speech_encoder_chunk_size
        self.speech_encoder_left_chunk_num = speech_encoder_left_chunk_num

        # t2u config
        self.t2u_bos_token_id = t2u_bos_token_id
        self.t2u_pad_token_id = t2u_pad_token_id
        self.t2u_eos_token_id = t2u_eos_token_id
        self.t2u_encoder_layers = t2u_encoder_layers
        self.t2u_encoder_ffn_dim = t2u_encoder_ffn_dim
        self.t2u_encoder_attention_heads = t2u_encoder_attention_heads
        self.t2u_decoder_layers = t2u_decoder_layers
        self.t2u_decoder_ffn_dim = t2u_decoder_ffn_dim
        self.t2u_decoder_attention_heads = t2u_decoder_attention_heads
        self.t2u_max_position_embeddings = t2u_max_position_embeddings
        self.t2u_variance_predictor_embed_dim = t2u_variance_predictor_embed_dim  # TODO: add to docstrings
        self.t2u_variance_predictor_hidden_dim = t2u_variance_predictor_hidden_dim  # TODO: add to docstrings
        self.t2u_variance_predictor_kernel_size = t2u_variance_predictor_kernel_size  # TODO: add to docstrings
        self.t2u_variance_pred_dropout = t2u_variance_pred_dropout  # TODO: add to docstrings

        # hifi-gan vocoder config
        # original parameters specific to Hifi-Gan
        self.sampling_rate = sampling_rate
        self.upsample_initial_channel = upsample_initial_channel
        self.upsample_rates = upsample_rates
        self.upsample_kernel_sizes = upsample_kernel_sizes
        self.resblock_kernel_sizes = resblock_kernel_sizes
        self.resblock_dilation_sizes = resblock_dilation_sizes
        self.leaky_relu_slope = leaky_relu_slope

        # specific to Code Hifi-Gan
        self.unit_hifi_gan_vocab_size = unit_hifi_gan_vocab_size
        self.unit_embed_dim = unit_embed_dim
        self.lang_embed_dim = lang_embed_dim
        self.spkr_embed_dim = spkr_embed_dim
        self.vocoder_num_langs = vocoder_num_langs
        self.vocoder_num_spkrs = vocoder_num_spkrs
        self.variance_predictor_kernel_size = variance_predictor_kernel_size
        self.var_pred_dropout = var_pred_dropout
        self.vocoder_offset = vocoder_offset

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            is_encoder_decoder=is_encoder_decoder,
            max_position_embeddings=max_position_embeddings,
            **kwargs,
        )

mindnlp.transformers.models.seamless_m4t_v2.configuration_seamless_m4t_v2.SeamlessM4Tv2Config.__init__(vocab_size=256102, t2u_vocab_size=10082, char_vocab_size=10943, hidden_size=1024, initializer_range=0.02, layer_norm_eps=1e-05, use_cache=True, max_position_embeddings=4096, is_encoder_decoder=True, encoder_layerdrop=0.05, decoder_layerdrop=0.05, activation_function='relu', dropout=0.1, attention_dropout=0.1, activation_dropout=0.0, scale_embedding=True, encoder_layers=24, encoder_ffn_dim=8192, encoder_attention_heads=16, decoder_layers=24, decoder_ffn_dim=8192, decoder_attention_heads=16, decoder_start_token_id=3, max_new_tokens=256, pad_token_id=0, bos_token_id=2, eos_token_id=3, speech_encoder_layers=24, speech_encoder_attention_heads=16, speech_encoder_intermediate_size=4096, speech_encoder_hidden_act='swish', speech_encoder_dropout=0.0, add_adapter=True, speech_encoder_layerdrop=0.1, feature_projection_input_dim=160, adaptor_kernel_size=8, adaptor_stride=8, adaptor_dropout=0.1, num_adapter_layers=1, position_embeddings_type='relative_key', conv_depthwise_kernel_size=31, left_max_position_embeddings=64, right_max_position_embeddings=8, speech_encoder_chunk_size=20000, speech_encoder_left_chunk_num=128, t2u_bos_token_id=0, t2u_pad_token_id=1, t2u_eos_token_id=2, t2u_encoder_layers=6, t2u_encoder_ffn_dim=8192, t2u_encoder_attention_heads=16, t2u_decoder_layers=6, t2u_decoder_ffn_dim=8192, t2u_decoder_attention_heads=16, t2u_max_position_embeddings=4096, t2u_variance_predictor_embed_dim=1024, t2u_variance_predictor_hidden_dim=256, t2u_variance_predictor_kernel_size=3, t2u_variance_pred_dropout=0.5, sampling_rate=16000, upsample_initial_channel=512, upsample_rates=[5, 4, 4, 2, 2], upsample_kernel_sizes=[11, 8, 8, 4, 4], resblock_kernel_sizes=[3, 7, 11], resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]], leaky_relu_slope=0.1, unit_hifi_gan_vocab_size=10000, unit_embed_dim=1280, lang_embed_dim=256, spkr_embed_dim=256, vocoder_num_langs=36, vocoder_num_spkrs=200, variance_predictor_kernel_size=3, var_pred_dropout=0.5, vocoder_offset=4, **kwargs)

Initializes a new instance of the SeamlessM4Tv2Config class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_size	The size of the vocabulary. Defaults to 256102. TYPE: int DEFAULT: 256102
t2u_vocab_size	The size of the text-to-unit vocabulary. Defaults to 10082. TYPE: int DEFAULT: 10082
char_vocab_size	The size of the character vocabulary. Defaults to 10943. TYPE: int DEFAULT: 10943
hidden_size	The size of the hidden layers. Defaults to 1024. TYPE: int DEFAULT: 1024
initializer_range	The range for weight initialization. Defaults to 0.02. TYPE: float DEFAULT: 0.02
layer_norm_eps	The epsilon value for layer normalization. Defaults to 1e-05. TYPE: float DEFAULT: 1e-05
use_cache	Whether to use cache. Defaults to True. TYPE: bool DEFAULT: True
max_position_embeddings	The maximum number of position embeddings. Defaults to 4096. TYPE: int DEFAULT: 4096
is_encoder_decoder	Whether it is an encoder-decoder model. Defaults to True. TYPE: bool DEFAULT: True
encoder_layerdrop	The layerdrop probability for the encoder. Defaults to 0.05. TYPE: float DEFAULT: 0.05
decoder_layerdrop	The layerdrop probability for the decoder. Defaults to 0.05. TYPE: float DEFAULT: 0.05
activation_function	The activation function to use. Defaults to 'relu'. TYPE: str DEFAULT: 'relu'
dropout	The dropout probability. Defaults to 0.1. TYPE: float DEFAULT: 0.1
attention_dropout	The dropout probability for attention layers. Defaults to 0.1. TYPE: float DEFAULT: 0.1
activation_dropout	The dropout probability for activation layers. Defaults to 0.0. TYPE: float DEFAULT: 0.0
scale_embedding	Whether to scale the embeddings. Defaults to True. TYPE: bool DEFAULT: True
encoder_layers	The number of encoder layers. Defaults to 24. TYPE: int DEFAULT: 24
encoder_ffn_dim	The dimension of the encoder feed-forward network. Defaults to 8192. TYPE: int DEFAULT: 8192
encoder_attention_heads	The number of attention heads in the encoder. Defaults to 16. TYPE: int DEFAULT: 16
decoder_layers	The number of decoder layers. Defaults to 24. TYPE: int DEFAULT: 24
decoder_ffn_dim	The dimension of the decoder feed-forward network. Defaults to 8192. TYPE: int DEFAULT: 8192
decoder_attention_heads	The number of attention heads in the decoder. Defaults to 16. TYPE: int DEFAULT: 16
decoder_start_token_id	The token ID for the start of decoding. Defaults to 3. TYPE: int DEFAULT: 3
max_new_tokens	The maximum number of new tokens. Defaults to 256. TYPE: int DEFAULT: 256
pad_token_id	The token ID for padding. Defaults to 0. TYPE: int DEFAULT: 0
bos_token_id	The token ID for the beginning of sequence. Defaults to 2. TYPE: int DEFAULT: 2
eos_token_id	The token ID for the end of sequence. Defaults to 3. TYPE: int DEFAULT: 3
speech_encoder_layers	The number of speech encoder layers. Defaults to 24. TYPE: int DEFAULT: 24
speech_encoder_attention_heads	The number of attention heads in the speech encoder. Defaults to 16. TYPE: int DEFAULT: 16
speech_encoder_intermediate_size	The intermediate size of the speech encoder. Defaults to 4096. TYPE: int DEFAULT: 4096
speech_encoder_hidden_act	The activation function for the speech encoder. Defaults to 'swish'. TYPE: str DEFAULT: 'swish'
speech_encoder_dropout	The dropout probability for the speech encoder. Defaults to 0.0. TYPE: float DEFAULT: 0.0
add_adapter	Whether to add an adapter. Defaults to True. TYPE: bool DEFAULT: True
speech_encoder_layerdrop	The layerdrop probability for the speech encoder. Defaults to 0.1. TYPE: float DEFAULT: 0.1
feature_projection_input_dim	The input dimension for feature projection. Defaults to 160. TYPE: int DEFAULT: 160
adaptor_kernel_size	The kernel size for the adaptor. Defaults to 8. TYPE: int DEFAULT: 8
adaptor_stride	The stride for the adaptor. Defaults to 8. TYPE: int DEFAULT: 8
adaptor_dropout	The dropout probability for the adaptor. Defaults to 0.1. TYPE: float DEFAULT: 0.1
num_adapter_layers	The number of adapter layers. Defaults to 1. TYPE: int DEFAULT: 1
position_embeddings_type	The type of position embeddings. Defaults to 'relative_key'. TYPE: str DEFAULT: 'relative_key'
conv_depthwise_kernel_size	The kernel size for depthwise convolution. Defaults to 31. TYPE: int DEFAULT: 31
left_max_position_embeddings	The maximum number of left position embeddings. Defaults to 64. TYPE: int DEFAULT: 64
right_max_position_embeddings	The maximum number of right position embeddings. Defaults to 8. TYPE: int DEFAULT: 8
speech_encoder_chunk_size	The chunk size for the speech encoder. Defaults to 20000. TYPE: int DEFAULT: 20000
speech_encoder_left_chunk_num	The number of left chunks for the speech encoder. Defaults to 128. TYPE: int DEFAULT: 128
t2u_bos_token_id	The token ID for the beginning of text-to-unit conversion. Defaults to 0. TYPE: int DEFAULT: 0
t2u_pad_token_id	The token ID for padding in text-to-unit conversion. Defaults to 1. TYPE: int DEFAULT: 1
t2u_eos_token_id	The token ID for the end of text-to-unit conversion. Defaults to 2. TYPE: int DEFAULT: 2
t2u_encoder_layers	The number of text-to-unit encoder layers. Defaults to 6. TYPE: int DEFAULT: 6
t2u_encoder_ffn_dim	The dimension of the text-to-unit encoder feed-forward network. Defaults to 8192. TYPE: int DEFAULT: 8192
t2u_encoder_attention_heads	The number of attention heads in the text-to-unit encoder. Defaults to 16. TYPE: int DEFAULT: 16
t2u_decoder_layers	The number of text-to-unit decoder layers. Defaults to 6. TYPE: int DEFAULT: 6
t2u_decoder_ffn_dim	The dimension of the text-to-unit decoder feed-forward network. Defaults to 8192. TYPE: int DEFAULT: 8192
t2u_decoder_attention_heads	The number of attention heads in the text-to-unit decoder. Defaults to 16. TYPE: int DEFAULT: 16
t2u_max_position_embeddings	The maximum number of position embeddings for text-to-unit conversion. Defaults to 4096. TYPE: int DEFAULT: 4096
t2u_variance_predictor_embed_dim	The embedding dimension for the variance predictor in text-to-unit conversion. Defaults to 1024. TYPE: int DEFAULT: 1024
t2u_variance_predictor_hidden_dim	The hidden dimension for the variance predictor in text-to-unit conversion. Defaults to 256. TYPE: int DEFAULT: 256
t2u_variance_predictor_kernel_size	The kernel size for the variance predictor in text-to-unit conversion. Defaults to 3. TYPE: int DEFAULT: 3
t2u_variance_pred_dropout	The dropout probability for the variance predictor in text-to-unit conversion. Defaults to 0.5. TYPE: float DEFAULT: 0.5
sampling_rate	The sampling rate of audio data. Defaults to 16000. TYPE: int DEFAULT: 16000
upsample_initial_channel	The initial number of channels for upsampling. Defaults to 512. TYPE: int DEFAULT: 512
upsample_rates	The rates for upsampling. Defaults to [5, 4, 4, 2, 2]. TYPE: List[int] DEFAULT: [5, 4, 4, 2, 2]
upsample_kernel_sizes	The kernel sizes for upsampling. Defaults to [11, 8, 8, 4, 4]. TYPE: List[int] DEFAULT: [11, 8, 8, 4, 4]
resblock_kernel_sizes	The kernel sizes for residual blocks. Defaults to [3, 7, 11]. TYPE: List[int] DEFAULT: [3, 7, 11]
resblock_dilation_sizes	The dilation sizes for residual blocks. Defaults to [[1, 3, 5], [1, 3, 5], [1, 3, 5]]. TYPE: List[List[int]] DEFAULT: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
leaky_relu_slope	The slope for LeakyReLU activation. Defaults to 0.1. TYPE: float DEFAULT: 0.1
unit_hifi_gan_vocab_size	The vocabulary size for the unit HiFi-GAN. Defaults to 10000. TYPE: int DEFAULT: 10000
unit_embed_dim	The embedding dimension for the unit HiFi-GAN. Defaults to 1280. TYPE: int DEFAULT: 1280
lang_embed_dim	The embedding dimension for language. Defaults to 256. TYPE: int DEFAULT: 256
spkr_embed_dim	The embedding dimension for speaker. Defaults to 256. TYPE: int DEFAULT: 256
vocoder_num_langs	The number of languages for the vocoder. Defaults to 36. TYPE: int DEFAULT: 36
vocoder_num_spkrs	The number of speakers for the vocoder. Defaults to 200. TYPE: int DEFAULT: 200
variance_predictor_kernel_size	The kernel size for the variance predictor. Defaults to 3. TYPE: int DEFAULT: 3
var_pred_dropout	The dropout probability for the variance predictor. Defaults to 0.5. TYPE: float DEFAULT: 0.5
vocoder_offset	The offset for the vocoder. Defaults to 4. TYPE: int DEFAULT: 4

RETURNS	DESCRIPTION
	None

Source code in mindnlp\transformers\models\seamless_m4t_v2\configuration_seamless_m4t_v2.py

def __init__(
    self,
    vocab_size=256102,
    t2u_vocab_size=10082,
    char_vocab_size=10943,
    # shared config
    hidden_size=1024,
    initializer_range=0.02,
    layer_norm_eps=1e-5,
    use_cache=True,
    max_position_embeddings=4096,
    is_encoder_decoder=True,
    encoder_layerdrop=0.05,
    decoder_layerdrop=0.05,
    activation_function="relu",
    dropout=0.1,
    attention_dropout=0.1,
    activation_dropout=0.0,
    scale_embedding=True,
    # text encoder|decoder
    encoder_layers=24,
    encoder_ffn_dim=8192,
    encoder_attention_heads=16,
    decoder_layers=24,
    decoder_ffn_dim=8192,
    decoder_attention_heads=16,
    decoder_start_token_id=3,
    max_new_tokens=256,
    pad_token_id=0,
    bos_token_id=2,
    eos_token_id=3,
    # speech_encoder
    speech_encoder_layers=24,
    speech_encoder_attention_heads=16,
    speech_encoder_intermediate_size=4096,
    speech_encoder_hidden_act="swish",
    speech_encoder_dropout=0.0,
    add_adapter=True,
    speech_encoder_layerdrop=0.1,
    feature_projection_input_dim=160,
    adaptor_kernel_size=8,
    adaptor_stride=8,
    adaptor_dropout=0.1,
    num_adapter_layers=1,
    position_embeddings_type="relative_key",
    conv_depthwise_kernel_size=31,
    left_max_position_embeddings=64,
    right_max_position_embeddings=8,
    speech_encoder_chunk_size=20000,
    speech_encoder_left_chunk_num=128,
    # t2u config
    t2u_bos_token_id=0,
    t2u_pad_token_id=1,
    t2u_eos_token_id=2,
    t2u_encoder_layers=6,
    t2u_encoder_ffn_dim=8192,
    t2u_encoder_attention_heads=16,
    t2u_decoder_layers=6,
    t2u_decoder_ffn_dim=8192,
    t2u_decoder_attention_heads=16,
    t2u_max_position_embeddings=4096,
    t2u_variance_predictor_embed_dim=1024,
    t2u_variance_predictor_hidden_dim=256,
    t2u_variance_predictor_kernel_size=3,
    t2u_variance_pred_dropout=0.5,
    # hifi-gan vocoder config
    sampling_rate=16000,
    upsample_initial_channel=512,
    upsample_rates=[5, 4, 4, 2, 2],
    upsample_kernel_sizes=[11, 8, 8, 4, 4],
    resblock_kernel_sizes=[3, 7, 11],
    resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
    leaky_relu_slope=0.1,
    # specific to Code Hifi-Gan
    unit_hifi_gan_vocab_size=10000,
    unit_embed_dim=1280,
    lang_embed_dim=256,
    spkr_embed_dim=256,
    vocoder_num_langs=36,
    vocoder_num_spkrs=200,
    variance_predictor_kernel_size=3,
    var_pred_dropout=0.5,
    vocoder_offset=4,
    **kwargs,
):
    '''
    Initializes a new instance of the SeamlessM4Tv2Config class.

    Args:
        self: The instance of the class.
        vocab_size (int, optional): The size of the vocabulary. Defaults to 256102.
        t2u_vocab_size (int, optional): The size of the text-to-unit vocabulary. Defaults to 10082.
        char_vocab_size (int, optional): The size of the character vocabulary. Defaults to 10943.
        hidden_size (int, optional): The size of the hidden layers. Defaults to 1024.
        initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
        layer_norm_eps (float, optional): The epsilon value for layer normalization. Defaults to 1e-05.
        use_cache (bool, optional): Whether to use cache. Defaults to True.
        max_position_embeddings (int, optional): The maximum number of position embeddings. Defaults to 4096.
        is_encoder_decoder (bool, optional): Whether it is an encoder-decoder model. Defaults to True.
        encoder_layerdrop (float, optional): The layerdrop probability for the encoder. Defaults to 0.05.
        decoder_layerdrop (float, optional): The layerdrop probability for the decoder. Defaults to 0.05.
        activation_function (str, optional): The activation function to use. Defaults to 'relu'.
        dropout (float, optional): The dropout probability. Defaults to 0.1.
        attention_dropout (float, optional): The dropout probability for attention layers. Defaults to 0.1.
        activation_dropout (float, optional): The dropout probability for activation layers. Defaults to 0.0.
        scale_embedding (bool, optional): Whether to scale the embeddings. Defaults to True.
        encoder_layers (int, optional): The number of encoder layers. Defaults to 24.
        encoder_ffn_dim (int, optional): The dimension of the encoder feed-forward network. Defaults to 8192.
        encoder_attention_heads (int, optional): The number of attention heads in the encoder. Defaults to 16.
        decoder_layers (int, optional): The number of decoder layers. Defaults to 24.
        decoder_ffn_dim (int, optional): The dimension of the decoder feed-forward network. Defaults to 8192.
        decoder_attention_heads (int, optional): The number of attention heads in the decoder. Defaults to 16.
        decoder_start_token_id (int, optional): The token ID for the start of decoding. Defaults to 3.
        max_new_tokens (int, optional): The maximum number of new tokens. Defaults to 256.
        pad_token_id (int, optional): The token ID for padding. Defaults to 0.
        bos_token_id (int, optional): The token ID for the beginning of sequence. Defaults to 2.
        eos_token_id (int, optional): The token ID for the end of sequence. Defaults to 3.
        speech_encoder_layers (int, optional): The number of speech encoder layers. Defaults to 24.
        speech_encoder_attention_heads (int, optional): The number of attention heads in the speech encoder. Defaults to 16.
        speech_encoder_intermediate_size (int, optional): The intermediate size of the speech encoder. Defaults to 4096.
        speech_encoder_hidden_act (str, optional): The activation function for the speech encoder. Defaults to 'swish'.
        speech_encoder_dropout (float, optional): The dropout probability for the speech encoder. Defaults to 0.0.
        add_adapter (bool, optional): Whether to add an adapter. Defaults to True.
        speech_encoder_layerdrop (float, optional): The layerdrop probability for the speech encoder. Defaults to 0.1.
        feature_projection_input_dim (int, optional): The input dimension for feature projection. Defaults to 160.
        adaptor_kernel_size (int, optional): The kernel size for the adaptor. Defaults to 8.
        adaptor_stride (int, optional): The stride for the adaptor. Defaults to 8.
        adaptor_dropout (float, optional): The dropout probability for the adaptor. Defaults to 0.1.
        num_adapter_layers (int, optional): The number of adapter layers. Defaults to 1.
        position_embeddings_type (str, optional): The type of position embeddings. Defaults to 'relative_key'.
        conv_depthwise_kernel_size (int, optional): The kernel size for depthwise convolution. Defaults to 31.
        left_max_position_embeddings (int, optional): The maximum number of left position embeddings. Defaults to 64.
        right_max_position_embeddings (int, optional): The maximum number of right position embeddings. Defaults to 8.
        speech_encoder_chunk_size (int, optional): The chunk size for the speech encoder. Defaults to 20000.
        speech_encoder_left_chunk_num (int, optional): The number of left chunks for the speech encoder. Defaults to 128.
        t2u_bos_token_id (int, optional): The token ID for the beginning of text-to-unit conversion. Defaults to 0.
        t2u_pad_token_id (int, optional): The token ID for padding in text-to-unit conversion. Defaults to 1.
        t2u_eos_token_id (int, optional): The token ID for the end of text-to-unit conversion. Defaults to 2.
        t2u_encoder_layers (int, optional): The number of text-to-unit encoder layers. Defaults to 6.
        t2u_encoder_ffn_dim (int, optional): The dimension of the text-to-unit encoder feed-forward network. Defaults to 8192.
        t2u_encoder_attention_heads (int, optional): The number of attention heads in the text-to-unit encoder. Defaults to 16.
        t2u_decoder_layers (int, optional): The number of text-to-unit decoder layers. Defaults to 6.
        t2u_decoder_ffn_dim (int, optional): The dimension of the text-to-unit decoder feed-forward network. Defaults to 8192.
        t2u_decoder_attention_heads (int, optional): The number of attention heads in the text-to-unit decoder. Defaults to 16.
        t2u_max_position_embeddings (int, optional): The maximum number of position embeddings for text-to-unit conversion. Defaults to 4096.
        t2u_variance_predictor_embed_dim (int, optional): The embedding dimension for the variance predictor in text-to-unit conversion. Defaults to 1024.
        t2u_variance_predictor_hidden_dim (int, optional): The hidden dimension for the variance predictor in text-to-unit conversion. Defaults to 256.
        t2u_variance_predictor_kernel_size (int, optional): The kernel size for the variance predictor in text-to-unit conversion. Defaults to 3.
        t2u_variance_pred_dropout (float, optional): The dropout probability for the variance predictor in text-to-unit conversion. Defaults to 0.5.
        sampling_rate (int, optional): The sampling rate of audio data. Defaults to 16000.
        upsample_initial_channel (int, optional): The initial number of channels for upsampling. Defaults to 512.
        upsample_rates (List[int], optional): The rates for upsampling. Defaults to [5, 4, 4, 2, 2].
        upsample_kernel_sizes (List[int], optional): The kernel sizes for upsampling. Defaults to [11, 8, 8, 4, 4].
        resblock_kernel_sizes (List[int], optional): The kernel sizes for residual blocks. Defaults to [3, 7, 11].
        resblock_dilation_sizes (List[List[int]], optional): The dilation sizes for residual blocks. Defaults to [[1, 3, 5], [1, 3, 5], [1, 3, 5]].
        leaky_relu_slope (float, optional): The slope for LeakyReLU activation. Defaults to 0.1.
        unit_hifi_gan_vocab_size (int, optional): The vocabulary size for the unit HiFi-GAN. Defaults to 10000.
        unit_embed_dim (int, optional): The embedding dimension for the unit HiFi-GAN. Defaults to 1280.
        lang_embed_dim (int, optional): The embedding dimension for language. Defaults to 256.
        spkr_embed_dim (int, optional): The embedding dimension for speaker. Defaults to 256.
        vocoder_num_langs (int, optional): The number of languages for the vocoder. Defaults to 36.
        vocoder_num_spkrs (int, optional): The number of speakers for the vocoder. Defaults to 200.
        variance_predictor_kernel_size (int, optional): The kernel size for the variance predictor. Defaults to 3.
        var_pred_dropout (float, optional): The dropout probability for the variance predictor. Defaults to 0.5.
        vocoder_offset (int, optional): The offset for the vocoder. Defaults to 4.

    Returns:
        None

    Raises:
        None
    '''
    # overall_config
    self.vocab_size = vocab_size
    self.t2u_vocab_size = t2u_vocab_size
    self.char_vocab_size = char_vocab_size
    self.hidden_size = hidden_size
    self.initializer_range = initializer_range
    self.layer_norm_eps = layer_norm_eps
    self.max_position_embeddings = max_position_embeddings
    self.use_cache = use_cache
    self.max_new_tokens = max_new_tokens
    self.encoder_layerdrop = encoder_layerdrop
    self.decoder_layerdrop = decoder_layerdrop
    self.activation_function = activation_function
    self.dropout = dropout
    self.attention_dropout = attention_dropout
    self.activation_dropout = activation_dropout
    self.scale_embedding = scale_embedding
    # for proper config init
    self.num_attention_heads = decoder_attention_heads
    self.num_hidden_layers = decoder_layers

    # text|unit encoder|decoder
    self.encoder_layers = encoder_layers
    self.encoder_ffn_dim = encoder_ffn_dim
    self.encoder_attention_heads = encoder_attention_heads
    self.decoder_layers = decoder_layers
    self.decoder_ffn_dim = decoder_ffn_dim
    self.decoder_attention_heads = decoder_attention_heads

    # speech_encoder
    self.speech_encoder_layers = speech_encoder_layers
    self.speech_encoder_hidden_act = speech_encoder_hidden_act
    self.speech_encoder_dropout = speech_encoder_dropout
    self.speech_encoder_attention_heads = speech_encoder_attention_heads
    self.speech_encoder_layerdrop = speech_encoder_layerdrop
    self.speech_encoder_intermediate_size = speech_encoder_intermediate_size
    self.feature_projection_input_dim = feature_projection_input_dim
    self.adaptor_kernel_size = adaptor_kernel_size
    self.adaptor_stride = adaptor_stride
    self.adaptor_dropout = adaptor_dropout
    self.num_adapter_layers = num_adapter_layers
    self.position_embeddings_type = position_embeddings_type
    self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
    self.add_adapter = add_adapter
    self.left_max_position_embeddings = left_max_position_embeddings
    self.right_max_position_embeddings = right_max_position_embeddings
    self.speech_encoder_chunk_size = speech_encoder_chunk_size
    self.speech_encoder_left_chunk_num = speech_encoder_left_chunk_num

    # t2u config
    self.t2u_bos_token_id = t2u_bos_token_id
    self.t2u_pad_token_id = t2u_pad_token_id
    self.t2u_eos_token_id = t2u_eos_token_id
    self.t2u_encoder_layers = t2u_encoder_layers
    self.t2u_encoder_ffn_dim = t2u_encoder_ffn_dim
    self.t2u_encoder_attention_heads = t2u_encoder_attention_heads
    self.t2u_decoder_layers = t2u_decoder_layers
    self.t2u_decoder_ffn_dim = t2u_decoder_ffn_dim
    self.t2u_decoder_attention_heads = t2u_decoder_attention_heads
    self.t2u_max_position_embeddings = t2u_max_position_embeddings
    self.t2u_variance_predictor_embed_dim = t2u_variance_predictor_embed_dim  # TODO: add to docstrings
    self.t2u_variance_predictor_hidden_dim = t2u_variance_predictor_hidden_dim  # TODO: add to docstrings
    self.t2u_variance_predictor_kernel_size = t2u_variance_predictor_kernel_size  # TODO: add to docstrings
    self.t2u_variance_pred_dropout = t2u_variance_pred_dropout  # TODO: add to docstrings

    # hifi-gan vocoder config
    # original parameters specific to Hifi-Gan
    self.sampling_rate = sampling_rate
    self.upsample_initial_channel = upsample_initial_channel
    self.upsample_rates = upsample_rates
    self.upsample_kernel_sizes = upsample_kernel_sizes
    self.resblock_kernel_sizes = resblock_kernel_sizes
    self.resblock_dilation_sizes = resblock_dilation_sizes
    self.leaky_relu_slope = leaky_relu_slope

    # specific to Code Hifi-Gan
    self.unit_hifi_gan_vocab_size = unit_hifi_gan_vocab_size
    self.unit_embed_dim = unit_embed_dim
    self.lang_embed_dim = lang_embed_dim
    self.spkr_embed_dim = spkr_embed_dim
    self.vocoder_num_langs = vocoder_num_langs
    self.vocoder_num_spkrs = vocoder_num_spkrs
    self.variance_predictor_kernel_size = variance_predictor_kernel_size
    self.var_pred_dropout = var_pred_dropout
    self.vocoder_offset = vocoder_offset

    super().__init__(
        pad_token_id=pad_token_id,
        bos_token_id=bos_token_id,
        eos_token_id=eos_token_id,
        decoder_start_token_id=decoder_start_token_id,
        is_encoder_decoder=is_encoder_decoder,
        max_position_embeddings=max_position_embeddings,
        **kwargs,
    )