seamless_m4t

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t

MindSpore SeamlessM4T model.

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TAttention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    # Copied from transformers.models.bart.modeling_bart.BartAttention.__init__ with Bart->SeamlessM4T
    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[SeamlessM4TConfig] = None,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    def _shape(self, tensor: mindspore.Tensor, seq_len: int, bsz: int):
        return ops.transpose(tensor.view(bsz, seq_len, self.num_heads, self.head_dim), 1, 2)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        """Input shape: Batch x Time x Channel"""

        # if encoder_hidden_states are provided this layer is used as a cross-attention layer
        # for the decoder
        is_cross_attention = encoder_hidden_states is not None

        bsz, tgt_len, _ = hidden_states.shape

        # get query proj
        query_states = self.q_proj(hidden_states) * self.scaling
        # get key, value proj
        # `past_key_value[0].shape[2] == encoder_hidden_states.shape[1]`
        # is checking that the `sequence_length` of the `past_key_value` is the same as
        # the provided `encoder_hidden_states` to support prefix tuning
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == encoder_hidden_states.shape[1]
        ):
            # reuse k,v, cross_attentions
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            # cross_attentions
            key_states = self._shape(self.k_proj(encoder_hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(encoder_hidden_states), -1, bsz)
        elif past_key_value is not None:
            # reuse k, v, self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = ops.cat([past_key_value[0], key_states], dim=2)
            value_states = ops.cat([past_key_value[1], value_states], dim=2)
        else:
            # self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.shape[1]
        attn_weights = ops.bmm(query_states, ops.transpose(key_states, 1, 2))

        if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.shape}"
            )

        if attention_mask is not None:
            if attention_mask.shape != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)

        if output_attentions:
            # this operation is a bit awkward, but it's required to
            # make sure that attn_weights keeps its gradient.
            # In order to do so, attn_weights have to be reshaped
            # twice and have to be reused in the following
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = ops.bmm(attn_probs, value_states)

        if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.shape}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = ops.transpose(attn_output, 1, 2)

        # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
        # partitioned across GPUs when using tensor-parallelism.
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)

        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TAttention.forward(hidden_states, encoder_hidden_states=None, past_key_value=None, attention_mask=None, output_attentions=False)

Input shape: Batch x Time x Channel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    output_attentions: bool = False,
) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
    """Input shape: Batch x Time x Channel"""

    # if encoder_hidden_states are provided this layer is used as a cross-attention layer
    # for the decoder
    is_cross_attention = encoder_hidden_states is not None

    bsz, tgt_len, _ = hidden_states.shape

    # get query proj
    query_states = self.q_proj(hidden_states) * self.scaling
    # get key, value proj
    # `past_key_value[0].shape[2] == encoder_hidden_states.shape[1]`
    # is checking that the `sequence_length` of the `past_key_value` is the same as
    # the provided `encoder_hidden_states` to support prefix tuning
    if (
        is_cross_attention
        and past_key_value is not None
        and past_key_value[0].shape[2] == encoder_hidden_states.shape[1]
    ):
        # reuse k,v, cross_attentions
        key_states = past_key_value[0]
        value_states = past_key_value[1]
    elif is_cross_attention:
        # cross_attentions
        key_states = self._shape(self.k_proj(encoder_hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(encoder_hidden_states), -1, bsz)
    elif past_key_value is not None:
        # reuse k, v, self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
        key_states = ops.cat([past_key_value[0], key_states], dim=2)
        value_states = ops.cat([past_key_value[1], value_states], dim=2)
    else:
        # self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_states, value_states)

    proj_shape = (bsz * self.num_heads, -1, self.head_dim)
    query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
    key_states = key_states.reshape(*proj_shape)
    value_states = value_states.reshape(*proj_shape)

    src_len = key_states.shape[1]
    attn_weights = ops.bmm(query_states, ops.transpose(key_states, 1, 2))

    if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
        raise ValueError(
            f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
            f" {attn_weights.shape}"
        )

    if attention_mask is not None:
        if attention_mask.shape != (bsz, 1, tgt_len, src_len):
            raise ValueError(
                f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
            )
        attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
        attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

    attn_weights = nn.functional.softmax(attn_weights, dim=-1)

    if output_attentions:
        # this operation is a bit awkward, but it's required to
        # make sure that attn_weights keeps its gradient.
        # In order to do so, attn_weights have to be reshaped
        # twice and have to be reused in the following
        attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
        attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
    else:
        attn_weights_reshaped = None

    attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

    attn_output = ops.bmm(attn_probs, value_states)

    if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
        raise ValueError(
            f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
            f" {attn_output.shape}"
        )

    attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
    attn_output = ops.transpose(attn_output, 1, 2)

    # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
    # partitioned across GPUs when using tensor-parallelism.
    attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)

    attn_output = self.out_proj(attn_output)

    return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan

Bases: PreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TCodeHifiGan(PreTrainedModel):
    config_class = SeamlessM4TConfig
    main_input_name = "input_embeds"
    _no_split_modules = []

    def __init__(self, config):
        super().__init__(config)

        self.pad_token_id = config.t2u_pad_token_id
        self.dur_predictor = SeamlessM4TVariancePredictor(config)

        self.unit_embedding = nn.Embedding(config.unit_hifi_gan_vocab_size, config.unit_embed_dim)
        self.speaker_embedding = nn.Embedding(config.vocoder_num_spkrs, config.spkr_embed_dim)
        self.language_embedding = nn.Embedding(config.vocoder_num_langs, config.lang_embed_dim)

        self.hifi_gan = SeamlessM4THifiGan(config)

        # Initialize weights and apply final processing
        self.post_init()

    def _get_dur_output_lengths(self, input_ids, dur_out):
        """
        Computes the output length after the duration layer.
        """
        unit_lengths = (input_ids != self.pad_token_id).sum(1)

        # take care of edge cases where no padding or too many padding
        unit_lengths = ops.clamp(unit_lengths, 0, dur_out.shape[1] - 1)

        cumulative_dur_out = ops.cumsum(dur_out, dim=1)
        unit_lengths = ops.gather(cumulative_dur_out, dim=1, index=unit_lengths.unsqueeze(1)).squeeze()

        return unit_lengths

    def _get_output_hifigan_lengths(self, input_lengths: Union[mindspore.Tensor, int]):
        """
        Computes the output length of the hifigan convolutional layers
        """

        def _conv_out_length(input_length, kernel_size, stride, pad, dilation=1):
            # 1D convolutional layer output length formula taken
            return (
                ops.div(input_length + 2 * pad - dilation * (kernel_size - 1) - 1, stride, rounding_mode="floor") + 1
            )

        def _transpose_conv_out_length(input_length, kernel_size, stride, pad, dilation=1):
            return (input_length - 1) * stride - 2 * pad + dilation * (kernel_size - 1) + 1

        # conv_pre
        input_lengths = _conv_out_length(input_lengths, 7, 1, 3)

        # upsampler
        for i, (upsample_rate, kernel_size) in enumerate(
            zip(self.config.upsample_rates, self.config.upsample_kernel_sizes)
        ):
            input_lengths = _transpose_conv_out_length(
                input_lengths, kernel_size, upsample_rate, (kernel_size - upsample_rate) // 2
            )

        # resblock
        for i in range(len(self.config.upsample_rates)):
            for kernel_size, dilation in zip(self.config.resblock_kernel_sizes, self.config.resblock_dilation_sizes):
                for dil in dilation:
                    input_lengths = _conv_out_length(
                        input_lengths, kernel_size, 1, (kernel_size - 1) * dil // 2, dilation=dil
                    )

                for dil in dilation:
                    input_lengths = _conv_out_length(input_lengths, kernel_size, 1, (kernel_size - 1) // 2, dilation=1)

        # conv_post
        input_lengths = _conv_out_length(input_lengths, 7, 1, 3)

        return input_lengths

    def forward(
        self, input_ids: mindspore.Tensor, spkr_id: mindspore.Tensor, lang_id: mindspore.Tensor
    ) -> Tuple[mindspore.Tensor]:
        """
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTextToUnitForConditionalGeneration`]. [What are input
                IDs?](../glossary#input-ids)
            spkr_id (`int`, *optional*):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            tgt_lang (`str`, *optional*):
                The language id to use as target language for translation.
        """
        hidden_states = ops.transpose(self.unit_embedding(input_ids), 1, 2)
        spkr = ops.transpose(self.speaker_embedding(spkr_id), 1, 2)
        lang = ops.transpose(self.language_embedding(lang_id), 1, 2)

        log_dur_pred = self.dur_predictor(ops.transpose(hidden_states, 1, 2))
        dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
        # B x C x T
        if hidden_states.shape[0] == 1:
            hidden_states = ops.repeat_interleave(hidden_states, dur_out.view(-1), dim=2)
        else:
            # if batched sample, need to interleave per sample, and pad -> loss of parallelism
            if hidden_states.shape[0] > 1 and self.training:
                logger.warning(
                    """`self.training=True` and you use batching. You lose parallelism during the hifigan
                               forward pass because the samples are interleaved."""
                )
            hidden_states = [
                ops.transpose(ops.repeat_interleave(hidden_state, duration.tolist(), dim=-1), 0, 1)
                for (hidden_state, duration) in zip(hidden_states, dur_out)
            ]

            # hidden_states = nn.utils.rnn.pad_sequence(hidden_states, batch_first=True).transpose(1, 2)
            hidden_states = ops.stack(hidden_states).swapaxes(1, 2)

        spkr = spkr.tile((1, 1, hidden_states.shape[-1]))
        lang = lang.tile((1, 1, hidden_states.shape[-1]))
        hidden_states = ops.cat([lang, hidden_states, spkr], dim=1)

        hidden_states = self.hifi_gan(hidden_states)

        unit_lengths = self._get_dur_output_lengths(input_ids, dur_out)
        lengths = self._get_output_hifigan_lengths(unit_lengths)

        return hidden_states, lengths

    def _init_weights(self, module):
        """Initialize the weights."""
        if isinstance(module, (nn.Linear, nn.Conv1d, nn.ConvTranspose1d)):
            nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
            if module.padding_idx is not None:
                module.weight[module.padding_idx] = 0

    def apply_weight_norm(self):
        nn.utils.weight_norm(self.hifi_gan.conv_pre)
        for layer in self.hifi_gan.upsampler:
            nn.utils.weight_norm(layer)
        for layer in self.hifi_gan.resblocks:
            layer.apply_weight_norm()
        nn.utils.weight_norm(self.hifi_gan.conv_post)

    def remove_weight_norm(self):
        nn.utils.remove_weight_norm(self.hifi_gan.conv_pre)
        for layer in self.hifi_gan.upsampler:
            nn.utils.remove_weight_norm(layer)
        for layer in self.hifi_gan.resblocks:
            layer.remove_weight_norm()
        nn.utils.remove_weight_norm(self.hifi_gan.conv_post)

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan.forward(input_ids, spkr_id, lang_id)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTextToUnitForConditionalGeneration]. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`
spkr_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*
tgt_lang	The language id to use as target language for translation. TYPE: `str`, *optional*

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self, input_ids: mindspore.Tensor, spkr_id: mindspore.Tensor, lang_id: mindspore.Tensor
) -> Tuple[mindspore.Tensor]:
    """
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTextToUnitForConditionalGeneration`]. [What are input
            IDs?](../glossary#input-ids)
        spkr_id (`int`, *optional*):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        tgt_lang (`str`, *optional*):
            The language id to use as target language for translation.
    """
    hidden_states = ops.transpose(self.unit_embedding(input_ids), 1, 2)
    spkr = ops.transpose(self.speaker_embedding(spkr_id), 1, 2)
    lang = ops.transpose(self.language_embedding(lang_id), 1, 2)

    log_dur_pred = self.dur_predictor(ops.transpose(hidden_states, 1, 2))
    dur_out = ops.clamp(ops.round((ops.exp(log_dur_pred) - 1)).long(), min=1)
    # B x C x T
    if hidden_states.shape[0] == 1:
        hidden_states = ops.repeat_interleave(hidden_states, dur_out.view(-1), dim=2)
    else:
        # if batched sample, need to interleave per sample, and pad -> loss of parallelism
        if hidden_states.shape[0] > 1 and self.training:
            logger.warning(
                """`self.training=True` and you use batching. You lose parallelism during the hifigan
                           forward pass because the samples are interleaved."""
            )
        hidden_states = [
            ops.transpose(ops.repeat_interleave(hidden_state, duration.tolist(), dim=-1), 0, 1)
            for (hidden_state, duration) in zip(hidden_states, dur_out)
        ]

        # hidden_states = nn.utils.rnn.pad_sequence(hidden_states, batch_first=True).transpose(1, 2)
        hidden_states = ops.stack(hidden_states).swapaxes(1, 2)

    spkr = spkr.tile((1, 1, hidden_states.shape[-1]))
    lang = lang.tile((1, 1, hidden_states.shape[-1]))
    hidden_states = ops.cat([lang, hidden_states, spkr], dim=1)

    hidden_states = self.hifi_gan(hidden_states)

    unit_lengths = self._get_dur_output_lengths(input_ids, dur_out)
    lengths = self._get_output_hifigan_lengths(unit_lengths)

    return hidden_states, lengths

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TConformerConvolutionModule

Bases: Module

Convolution block used in the conformer block

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TConformerConvolutionModule(nn.Module):
    """Convolution block used in the conformer block"""

    def __init__(self, config):
        super().__init__()
        if (config.conv_depthwise_kernel_size - 1) % 2 == 1:
            raise ValueError("`config.conv_depthwise_kernel_size` should be a odd number for 'SAME' padding")
        self.layer_norm = nn.LayerNorm(config.hidden_size)
        self.pointwise_conv1 = nn.Conv1d(
            config.hidden_size,
            2 * config.hidden_size,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False,
        )
        self.glu = nn.GLU(dim=1)
        self.depthwise_conv = nn.Conv1d(
            config.hidden_size,
            config.hidden_size,
            config.conv_depthwise_kernel_size,
            stride=1,
            padding="same",
            groups=config.hidden_size,
            bias=False,
        )
        self.batch_norm = nn.BatchNorm1d(config.hidden_size)
        self.activation = ACT2FN[config.speech_encoder_hidden_act]
        self.pointwise_conv2 = nn.Conv1d(
            config.hidden_size,
            config.hidden_size,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False,
        )
        self.dropout = nn.Dropout(config.speech_encoder_dropout)

    def forward(self, hidden_states, attention_mask=None):
        hidden_states = self.layer_norm(hidden_states)

        # Ensure that we do not leak padded positions in depthwise convolution.
        # Put 0 where necessary
        if attention_mask is not None:
            hidden_states = hidden_states.masked_fill(~attention_mask.bool().unsqueeze(-1), 0.0)

        # exchange the temporal dimension and the feature dimension
        hidden_states = ops.transpose(hidden_states, 1, 2)

        # GLU mechanism
        # => (batch, 2*channel, dim)
        hidden_states = self.pointwise_conv1(hidden_states)
        # => (batch, channel, dim)
        hidden_states = self.glu(hidden_states)

        # 1D Depthwise Conv
        hidden_states = self.depthwise_conv(hidden_states)
        hidden_states = self.batch_norm(hidden_states)
        hidden_states = self.activation(hidden_states)

        hidden_states = self.pointwise_conv2(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = ops.transpose(hidden_states, 1, 2)
        return hidden_states

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TConformerEncoderLayer

Bases: Module

Conformer block based on https://arxiv.org/abs/2005.08100.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TConformerEncoderLayer(nn.Module):
    """Conformer block based on https://arxiv.org/abs/2005.08100."""

    def __init__(self, config):
        super().__init__()
        embed_dim = config.hidden_size
        dropout = config.speech_encoder_dropout

        # Feed-forward 1
        self.ffn1_layer_norm = nn.LayerNorm(embed_dim)
        self.ffn1 = SeamlessM4TConformerFeedForward(config)

        # Self-Attention
        self.self_attn_layer_norm = nn.LayerNorm(embed_dim)
        self.self_attn_dropout = nn.Dropout(dropout)
        self.self_attn = SeamlessM4TConformerSelfAttention(config)

        # Conformer Convolution
        self.conv_module = SeamlessM4TConformerConvolutionModule(config)

        # Feed-forward 2
        self.ffn2_layer_norm = nn.LayerNorm(embed_dim)
        self.ffn2 = SeamlessM4TConformerFeedForward(config)
        self.final_layer_norm = nn.LayerNorm(embed_dim)

    def forward(
        self,
        hidden_states,
        attention_mask: Optional[mindspore.Tensor] = None,
        relative_position_embeddings: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
        conv_attention_mask: Optional[mindspore.Tensor] = None,
    ):
        # 1. Feed-Forward 1 layer
        residual = hidden_states
        hidden_states = self.ffn1_layer_norm(hidden_states)
        hidden_states = self.ffn1(hidden_states)
        hidden_states = hidden_states * 0.5 + residual
        residual = hidden_states

        # 2. Self-Attention layer
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weigts = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            relative_position_embeddings=relative_position_embeddings,
            output_attentions=output_attentions,
        )
        hidden_states = self.self_attn_dropout(hidden_states)
        hidden_states = hidden_states + residual

        # 3. Convolutional Layer
        residual = hidden_states
        hidden_states = self.conv_module(hidden_states, attention_mask=conv_attention_mask)
        hidden_states = residual + hidden_states

        # 4. Feed-Forward 2 Layer
        residual = hidden_states
        hidden_states = self.ffn2_layer_norm(hidden_states)
        hidden_states = self.ffn2(hidden_states)
        hidden_states = hidden_states * 0.5 + residual
        hidden_states = self.final_layer_norm(hidden_states)

        return hidden_states, attn_weigts

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TConformerRelPositionalEmbedding

Bases: Module

Relative positional encoding module.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TConformerRelPositionalEmbedding(nn.Module):
    """Relative positional encoding module."""

    def __init__(self, config):
        super().__init__()
        self.max_len = config.max_source_positions
        self.d_model = config.hidden_size
        self.pe = None
        self.extend_pe(mindspore.tensor(0.0).broadcast_to((1, self.max_len)))

    def extend_pe(self, x):
        # Reset the positional encodings
        if self.pe is not None:
            # self.pe contains both positive and negative parts
            # the length of self.pe is 2 * input_len - 1
            if self.pe.shape[1] >= x.shape[1] * 2 - 1:
                if self.pe.dtype != x.dtype:
                    self.pe = self.pe.to(dtype=x.dtype)
                return
        # Suppose `i` is the position of query vector and `j` is the
        # position of key vector. We use positive relative positions when keys
        # are to the left (i>j) and negative relative positions otherwise (i<j).
        pe_positive = ops.zeros(x.shape[1], self.d_model)
        pe_negative = ops.zeros(x.shape[1], self.d_model)
        position = ops.arange(0, x.shape[1], dtype=mindspore.int64).float().unsqueeze(1)
        div_term = ops.exp(
            ops.arange(0, self.d_model, 2, dtype=mindspore.int64).float() * -(math.log(10000.0) / self.d_model)
        )
        pe_positive[:, 0::2] = ops.sin(position * div_term)
        pe_positive[:, 1::2] = ops.cos(position * div_term)
        pe_negative[:, 0::2] = ops.sin(-1 * position * div_term)
        pe_negative[:, 1::2] = ops.cos(-1 * position * div_term)

        # Reverse the order of positive indices and concat both positive and
        # negative indices. This is used to support the shifting trick
        # as in https://arxiv.org/abs/1901.02860
        pe_positive = ops.flip(pe_positive, [0]).unsqueeze(0)
        pe_negative = pe_negative[1:].unsqueeze(0)
        pe = ops.cat([pe_positive, pe_negative], dim=1)
        self.pe = pe.to(dtype=x.dtype)

    def forward(self, hidden_states: mindspore.Tensor):
        self.extend_pe(hidden_states)
        start_idx = self.pe.shape[1] // 2 - hidden_states.shape[1] + 1
        end_idx = self.pe.shape[1] // 2 + hidden_states.shape[1]
        relative_position_embeddings = self.pe[:, start_idx:end_idx]

        return relative_position_embeddings

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TConformerRotaryPositionalEmbedding

Bases: Module

Rotary positional embedding Reference : https://blog.eleuther.ai/rotary-embeddings/ Paper: https://arxiv.org/pdf/2104.09864.pdf

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TConformerRotaryPositionalEmbedding(nn.Module):
    """Rotary positional embedding
    Reference : https://blog.eleuther.ai/rotary-embeddings/ Paper: https://arxiv.org/pdf/2104.09864.pdf
    """

    def __init__(self, config):
        super().__init__()
        dim = config.hidden_size // config.speech_encoder_attention_heads
        base = config.rotary_embedding_base

        inv_freq = 1.0 / (base ** (ops.arange(0, dim, 2, dtype=mindspore.int64).float() / dim))
        self.register_buffer("inv_freq", inv_freq)
        self.cached_sequence_length = None
        self.cached_rotary_positional_embedding = None

    def forward(self, hidden_states):
        sequence_length = hidden_states.shape[1]

        if sequence_length == self.cached_sequence_length and self.cached_rotary_positional_embedding is not None:
            return self.cached_rotary_positional_embedding

        self.cached_sequence_length = sequence_length
        # Embeddings are computed in the dtype of the inv_freq constant
        time_stamps = ops.arange(sequence_length).type_as(self.inv_freq)
        freqs = ops.einsum("i,j->ij", time_stamps, self.inv_freq)
        embeddings = ops.cat((freqs, freqs), dim=-1)

        cos_embeddings = embeddings.cos()[:, None, None, :]
        sin_embeddings = embeddings.sin()[:, None, None, :]
        # Computed embeddings are cast to the dtype of the hidden state inputs
        self.cached_rotary_positional_embedding = ops.stack([cos_embeddings, sin_embeddings]).type_as(hidden_states)
        return self.cached_rotary_positional_embedding

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TConformerSelfAttention

Bases: Module

Construct a SeamlessM4TConformerSelfAttention object. Can be enhanced with rotary or relative position embeddings.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TConformerSelfAttention(nn.Module):
    """Construct a SeamlessM4TConformerSelfAttention object.
    Can be enhanced with rotary or relative position embeddings.
    """

    def __init__(self, config, use_position_embeddings=True):
        super().__init__()

        self.head_size = config.hidden_size // config.speech_encoder_attention_heads
        self.num_heads = config.speech_encoder_attention_heads
        self.position_embeddings_type = config.position_embeddings_type if use_position_embeddings else None

        self.linear_q = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_k = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_v = nn.Linear(config.hidden_size, config.hidden_size)
        self.linear_out = nn.Linear(config.hidden_size, config.hidden_size)

        self.dropout = nn.Dropout(p=config.speech_encoder_dropout)

        if self.position_embeddings_type == "relative":
            # linear transformation for positional encoding
            self.linear_pos = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
            # these two learnable bias are used in matrix c and matrix d
            # as described in https://arxiv.org/abs/1901.02860 Section 3.3
            self.pos_bias_u = nn.Parameter(ops.zeros(self.num_heads, self.head_size))
            self.pos_bias_v = nn.Parameter(ops.zeros(self.num_heads, self.head_size))

    # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerSelfAttention.forward
    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        relative_position_embeddings: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        # self-attention mechanism
        batch_size, sequence_length, hidden_size = hidden_states.shape

        # make sure query/key states can be != value states
        query_key_states = hidden_states
        value_states = hidden_states

        if self.position_embeddings_type == "rotary":
            if relative_position_embeddings is None:
                raise ValueError(
                    "`relative_position_embeddings` has to be defined when `self.position_embeddings_type == 'rotary'"
                )
            query_key_states = self._apply_rotary_embedding(query_key_states, relative_position_embeddings)

        # project query_key_states and value_states
        query = self.linear_q(query_key_states).view(batch_size, -1, self.num_heads, self.head_size)
        key = self.linear_k(query_key_states).view(batch_size, -1, self.num_heads, self.head_size)
        value = self.linear_v(value_states).view(batch_size, -1, self.num_heads, self.head_size)

        # => (batch, head, time1, d_k)
        query = ops.transpose(query, 1, 2)
        key = ops.transpose(key, 1, 2)
        value = ops.transpose(value, 1, 2)

        if self.position_embeddings_type == "relative":
            if relative_position_embeddings is None:
                raise ValueError(
                    "`relative_position_embeddings` has to be defined when `self.position_embeddings_type =="
                    " 'relative'"
                )
            # apply relative_position_embeddings to qk scores
            # as proposed in Transformer_XL: https://arxiv.org/abs/1901.02860
            scores = self._apply_relative_embeddings(
                query=query, key=key, relative_position_embeddings=relative_position_embeddings
            )
        else:
            scores = ops.matmul(query, ops.transpose(key, -2, -1)) / math.sqrt(self.head_size)

        # apply attention_mask if necessary
        if attention_mask is not None:
            scores = scores + attention_mask

        # => (batch, head, time1, time2)
        probs = ops.softmax(scores, dim=-1)
        probs = self.dropout(probs)

        # => (batch, head, time1, d_k)
        hidden_states = ops.matmul(probs, value)

        # => (batch, time1, hidden_size)
        hidden_states = ops.transpose(hidden_states, 1, 2).reshape(batch_size, -1, self.num_heads * self.head_size)
        hidden_states = self.linear_out(hidden_states)

        return hidden_states, probs

    # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerSelfAttention._apply_rotary_embedding
    def _apply_rotary_embedding(self, hidden_states, relative_position_embeddings):
        batch_size, sequence_length, hidden_size = hidden_states.shape
        hidden_states = hidden_states.view(batch_size, sequence_length, self.num_heads, self.head_size)

        cos = relative_position_embeddings[0, :sequence_length, ...]
        sin = relative_position_embeddings[1, :sequence_length, ...]

        # rotate hidden_states with rotary embeddings
        hidden_states = ops.transpose(hidden_states, 0, 1)
        rotated_states_begin = hidden_states[..., : self.head_size // 2]
        rotated_states_end = hidden_states[..., self.head_size // 2 :]
        rotated_states = ops.cat((-rotated_states_end, rotated_states_begin), dim=rotated_states_begin.ndim - 1)
        hidden_states = (hidden_states * cos) + (rotated_states * sin)
        hidden_states = ops.transpose(hidden_states, 0, 1)

        hidden_states = hidden_states.view(batch_size, sequence_length, self.num_heads * self.head_size)

        return hidden_states

    # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerSelfAttention._apply_relative_embeddings
    def _apply_relative_embeddings(self, query, key, relative_position_embeddings):
        # 1. project positional embeddings
        # => (batch, head, 2*time1-1, d_k)
        proj_relative_position_embeddings = self.linear_pos(relative_position_embeddings)
        proj_relative_position_embeddings = proj_relative_position_embeddings.view(
            relative_position_embeddings.shape[0], -1, self.num_heads, self.head_size
        )
        proj_relative_position_embeddings = ops.transpose(proj_relative_position_embeddings, 1, 2)
        proj_relative_position_embeddings = ops.transpose(proj_relative_position_embeddings, 2, 3)

        # 2. Add bias to query
        # => (batch, head, time1, d_k)
        query = ops.transpose(query, 1, 2)
        q_with_bias_u = ops.transpose((query + self.pos_bias_u), 1, 2)
        q_with_bias_v = ops.transpose((query + self.pos_bias_v), 1, 2)

        # 3. attention score: first compute matrix a and matrix c
        # as described in https://arxiv.org/abs/1901.02860 Section 3.3
        # => (batch, head, time1, time2)
        scores_ac = ops.matmul(q_with_bias_u, ops.transpose(key, -2, -1))

        # 4. then compute matrix b and matrix d
        # => (batch, head, time1, 2*time1-1)
        scores_bd = ops.matmul(q_with_bias_v, proj_relative_position_embeddings)

        # 5. shift matrix b and matrix d
        zero_pad = ops.zeros((*scores_bd.shape[:3], 1), dtype=scores_bd.dtype)
        scores_bd_padded = ops.cat([zero_pad, scores_bd], dim=-1)
        scores_bd_padded_shape = scores_bd.shape[:2] + (scores_bd.shape[3] + 1, scores_bd.shape[2])
        scores_bd_padded = scores_bd_padded.view(*scores_bd_padded_shape)
        scores_bd = scores_bd_padded[:, :, 1:].view_as(scores_bd)
        scores_bd = scores_bd[:, :, :, : scores_bd.shape[-1] // 2 + 1]

        # 6. sum matrices
        # => (batch, head, time1, time2)
        scores = (scores_ac + scores_bd) / math.sqrt(self.head_size)

        return scores

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TDecoder

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel):
    def __init__(
        self,
        config: SeamlessM4TConfig,
        embed_tokens: Optional[nn.Embedding] = None,
    ):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size
        self.max_target_positions = config.max_position_embeddings
        embed_scale = math.sqrt(config.hidden_size) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            # if embed_tokens defined, use its shape instead
            self.embed_tokens = SeamlessM4TScaledWordEmbedding(
                embed_tokens.num_embeddings, embed_tokens.embedding_dim, self.padding_idx, embed_scale=embed_scale
            )
            self.embed_tokens.weight = embed_tokens.weight
        else:
            self.embed_tokens = SeamlessM4TScaledWordEmbedding(
                self.vocab_size, config.hidden_size, self.padding_idx, embed_scale=embed_scale
            )

        self.embed_positions = SeamlessM4TSinusoidalPositionalEmbedding(
            self.max_target_positions,
            config.hidden_size,
            padding_idx=self.padding_idx,
        )

        layers = []
        for _ in range(config.decoder_layers):
            layers.append(
                SeamlessM4TDecoderLayer(
                    config,
                    decoder_attention_heads=config.decoder_attention_heads,
                    decoder_ffn_dim=config.decoder_ffn_dim,
                )
            )
        self.layers = nn.ModuleList(layers)
        self.layer_norm = nn.LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                of the decoder.
            encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
                Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
                selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
                that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
                all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input = input_ids
            input_shape = input.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
            input = inputs_embeds[:, :, -1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

        # expand encoder attention mask
        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
            encoder_attention_mask = _prepare_4d_attention_mask(
                encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
            )

        # embed positions
        positions = self.embed_positions(input, past_key_values_length=past_key_values_length)

        hidden_states = inputs_embeds + positions

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`..."
                )
                use_cache = False

        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = ops.rand([])
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    attention_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    None,
                    output_attentions,
                    use_cache,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=attention_mask,
                    encoder_hidden_states=encoder_hidden_states,
                    encoder_attention_mask=encoder_attention_mask,
                    past_key_value=past_key_value,
                    output_attentions=output_attentions,
                    use_cache=use_cache,
                )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[1],)

            if output_attentions:
                all_self_attns += (layer_outputs[2],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[3],)

        hidden_states = self.layer_norm(hidden_states)

        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TDecoder.forward(input_ids=None, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional* DEFAULT: None
encoder_attention_mask	Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional* DEFAULT: None
past_key_values	Tuple of tuple(mindspore.Tensor) of length config.n_layers, with each tuple having 2 tensors of shape (batch_size, num_heads, sequence_length, embed_size_per_head)) and 2 additional tensors of shape (batch_size, num_heads, encoder_sequence_length, embed_size_per_head). Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see past_key_values input) to speed up sequential decoding. If past_key_values are used, the user can optionally input only the last decoder_input_ids (those that don't have their past key value states given to this model) of shape (batch_size, 1) instead of all decoder_input_ids of shape (batch_size, sequence_length). TYPE: `tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True` DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
            Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
            selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
            cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
            that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
            all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
    elif input_ids is not None:
        input = input_ids
        input_shape = input.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
        input = inputs_embeds[:, :, -1]
    else:
        raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    attention_mask = _prepare_4d_causal_attention_mask(
        attention_mask, input_shape, inputs_embeds, past_key_values_length
    )

    # expand encoder attention mask
    if encoder_hidden_states is not None and encoder_attention_mask is not None:
        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
        encoder_attention_mask = _prepare_4d_attention_mask(
            encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
        )

    # embed positions
    positions = self.embed_positions(input, past_key_values_length=past_key_values_length)

    hidden_states = inputs_embeds + positions

    hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`..."
            )
            use_cache = False

    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None
    all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
    next_decoder_cache = () if use_cache else None

    for idx, decoder_layer in enumerate(self.layers):
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        if output_hidden_states:
            all_hidden_states += (hidden_states,)
        if self.training:
            dropout_probability = ops.rand([])
            if dropout_probability < self.layerdrop:
                continue

        past_key_value = past_key_values[idx] if past_key_values is not None else None

        if self.gradient_checkpointing and self.training:
            layer_outputs = self._gradient_checkpointing_func(
                decoder_layer.__call__,
                hidden_states,
                attention_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                None,
                output_attentions,
                use_cache,
            )
        else:
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                encoder_attention_mask=encoder_attention_mask,
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )
        hidden_states = layer_outputs[0]

        if use_cache:
            next_decoder_cache += (layer_outputs[1],)

        if output_attentions:
            all_self_attns += (layer_outputs[2],)

            if encoder_hidden_states is not None:
                all_cross_attentions += (layer_outputs[3],)

    hidden_states = self.layer_norm(hidden_states)

    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    next_cache = next_decoder_cache if use_cache else None
    if not return_dict:
        return tuple(
            v
            for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TDecoderLayer

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TDecoderLayer(nn.Module):
    def __init__(self, config: SeamlessM4TConfig, decoder_ffn_dim=None, decoder_attention_heads=None):
        super().__init__()
        decoder_ffn_dim = config.decoder_ffn_dim if decoder_ffn_dim is None else decoder_ffn_dim
        decoder_attention_heads = (
            config.decoder_attention_heads if decoder_attention_heads is None else decoder_attention_heads
        )

        self.embed_dim = config.hidden_size
        self.self_attn = SeamlessM4TAttention(
            embed_dim=self.embed_dim,
            num_heads=decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.attn_dropout = nn.Dropout(config.dropout)

        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.cross_attention = SeamlessM4TAttention(
            self.embed_dim, decoder_attention_heads, config.attention_dropout, is_decoder=True
        )
        self.cross_attention_layer_norm = nn.LayerNorm(self.embed_dim)

        self.ffn = SeamlessM4TFeedForwardNetwork(config, ffn_dim=decoder_ffn_dim)

        self.ffn_layer_norm = nn.LayerNorm(config.hidden_size)
        self.ffn_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`):
                attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
                large negative values.
            encoder_hidden_states (`mindspore.Tensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`mindspore.Tensor`):
                encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by
                very large negative values.
            past_key_value (`Tuple(mindspore.Tensor)`):
                cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        # Self Attention
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        # add present self-attn cache to positions 1,2 of present_key_value tuple
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        # Cross-Attention Block
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.cross_attention_layer_norm(hidden_states)

            # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None

            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention(
                hidden_states=hidden_states,
                encoder_hidden_states=encoder_hidden_states,
                past_key_value=cross_attn_past_key_value,
                attention_mask=encoder_attention_mask,
                output_attentions=output_attentions,
            )
            hidden_states = self.attn_dropout(hidden_states)
            hidden_states = residual + hidden_states

            # add cross-attn to positions 3,4 of present_key_value tuple
            present_key_value += cross_attn_present_key_value

        # Fully Connected
        residual = hidden_states

        hidden_states = self.ffn_layer_norm(hidden_states)

        hidden_states = self.ffn(hidden_states)
        hidden_states = self.ffn_dropout(hidden_states)

        hidden_states = residual + hidden_states

        outputs = (hidden_states, present_key_value)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        return outputs

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TDecoderLayer.forward(hidden_states, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False, use_cache=True)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
encoder_hidden_states	cross attention input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor` DEFAULT: None
encoder_attention_mask	encoder attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
past_key_value	cached past key and value projection states TYPE: `Tuple(mindspore.Tensor)` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: Optional[bool] = False,
    use_cache: Optional[bool] = True,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`):
            input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`):
            attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
            large negative values.
        encoder_hidden_states (`mindspore.Tensor`):
            cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
        encoder_attention_mask (`mindspore.Tensor`):
            encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by
            very large negative values.
        past_key_value (`Tuple(mindspore.Tensor)`):
            cached past key and value projection states
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)

    # Self Attention
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    # add present self-attn cache to positions 1,2 of present_key_value tuple
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        past_key_value=self_attn_past_key_value,
        attention_mask=attention_mask,
        output_attentions=output_attentions,
    )
    hidden_states = self.attn_dropout(hidden_states)
    hidden_states = residual + hidden_states

    # Cross-Attention Block
    cross_attn_present_key_value = None
    cross_attn_weights = None
    if encoder_hidden_states is not None:
        residual = hidden_states
        hidden_states = self.cross_attention_layer_norm(hidden_states)

        # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None

        hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention(
            hidden_states=hidden_states,
            encoder_hidden_states=encoder_hidden_states,
            past_key_value=cross_attn_past_key_value,
            attention_mask=encoder_attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        # add cross-attn to positions 3,4 of present_key_value tuple
        present_key_value += cross_attn_present_key_value

    # Fully Connected
    residual = hidden_states

    hidden_states = self.ffn_layer_norm(hidden_states)

    hidden_states = self.ffn(hidden_states)
    hidden_states = self.ffn_dropout(hidden_states)

    hidden_states = residual + hidden_states

    outputs = (hidden_states, present_key_value)

    if output_attentions:
        outputs += (self_attn_weights, cross_attn_weights)

    return outputs

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TEncoder

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TEncoder(SeamlessM4TPreTrainedModel):
    def __init__(
        self,
        config: SeamlessM4TConfig,
        embed_tokens: Optional[nn.Embedding] = None,
        is_t2u_encoder: bool = False,
    ):
        super().__init__(config)

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop
        self.padding_idx = config.pad_token_id
        embed_dim = config.hidden_size

        self.is_t2u_encoder = is_t2u_encoder
        self.max_source_positions = config.max_position_embeddings

        if not self.is_t2u_encoder:
            embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

            self.embed_tokens = SeamlessM4TScaledWordEmbedding(
                config.vocab_size, embed_dim, self.padding_idx, embed_scale=embed_scale
            )

            if embed_tokens is not None:
                self.embed_tokens.weight = embed_tokens.weight

            self.embed_positions = SeamlessM4TSinusoidalPositionalEmbedding(
                self.max_source_positions,
                embed_dim,
                self.padding_idx,
            )

        layers = []
        for _ in range(config.encoder_layers):
            layers.append(
                SeamlessM4TEncoderLayer(
                    config,
                    encoder_attention_heads=config.encoder_attention_heads,
                    encoder_ffn_dim=config.encoder_ffn_dim,
                )
            )

        self.layers = nn.ModuleList(layers)

        self.layer_norm = nn.LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Tuple, BaseModelOutput]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is not None and self.is_t2u_encoder:
            raise ValueError(
                "You cannot pass input_ids to the encoder of the text_to_units model. Pass inputs_embeds instead."
            )

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            input = input_ids
            input_shape = input.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input = inputs_embeds[:, :, -1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        if not self.is_t2u_encoder:
            embed_pos = self.embed_positions(input)

            hidden_states = inputs_embeds + embed_pos
        else:
            hidden_states = inputs_embeds

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        # expand attention_mask
        if attention_mask is not None:
            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
            attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            to_drop = False
            if self.training:
                dropout_probability = ops.rand([])
                if dropout_probability < self.layerdrop:  # skip the layer
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.forward,
                        hidden_states,
                        attention_mask,
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        attention_mask,
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TEncoder.forward(input_ids=None, attention_mask=None, inputs_embeds=None, output_attentions=None, output_hidden_states=None, return_dict=None, **kwargs)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
    **kwargs,
) -> Union[Tuple, BaseModelOutput]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if input_ids is not None and self.is_t2u_encoder:
        raise ValueError(
            "You cannot pass input_ids to the encoder of the text_to_units model. Pass inputs_embeds instead."
        )

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    elif input_ids is not None:
        input = input_ids
        input_shape = input.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input = inputs_embeds[:, :, -1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    if not self.is_t2u_encoder:
        embed_pos = self.embed_positions(input)

        hidden_states = inputs_embeds + embed_pos
    else:
        hidden_states = inputs_embeds

    hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

    # expand attention_mask
    if attention_mask is not None:
        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
        attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

    encoder_states = () if output_hidden_states else None
    all_attentions = () if output_attentions else None

    for idx, encoder_layer in enumerate(self.layers):
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        to_drop = False
        if self.training:
            dropout_probability = ops.rand([])
            if dropout_probability < self.layerdrop:  # skip the layer
                to_drop = True

        if to_drop:
            layer_outputs = (None, None)
        else:
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    encoder_layer.forward,
                    hidden_states,
                    attention_mask,
                    output_attentions,
                )
            else:
                layer_outputs = encoder_layer(
                    hidden_states,
                    attention_mask,
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

        if output_attentions:
            all_attentions = all_attentions + (layer_outputs[1],)

    hidden_states = self.layer_norm(hidden_states)

    if output_hidden_states:
        encoder_states = encoder_states + (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
    return BaseModelOutput(
        last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
    )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TEncoderLayer

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TEncoderLayer(nn.Module):
    def __init__(self, config: SeamlessM4TConfig, encoder_ffn_dim=None, encoder_attention_heads=None):
        super().__init__()
        encoder_ffn_dim = config.encoder_ffn_dim if encoder_ffn_dim is None else encoder_ffn_dim
        encoder_attention_heads = (
            config.encoder_attention_heads if encoder_attention_heads is None else encoder_attention_heads
        )

        self.embed_dim = config.hidden_size
        self.self_attn = SeamlessM4TAttention(
            embed_dim=self.embed_dim,
            num_heads=encoder_attention_heads,
            dropout=config.attention_dropout,
        )
        self.attn_dropout = nn.Dropout(config.dropout)
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)

        self.ffn = SeamlessM4TFeedForwardNetwork(config, ffn_dim=encoder_ffn_dim)

        self.ffn_layer_norm = nn.LayerNorm(config.hidden_size)
        self.ffn_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: mindspore.Tensor,
        output_attentions: bool = False,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`):
                attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
                large negative values.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        residual = hidden_states

        hidden_states = self.ffn_layer_norm(hidden_states)

        hidden_states = self.ffn(hidden_states)
        hidden_states = self.ffn_dropout(hidden_states)

        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TEncoderLayer.forward(hidden_states, attention_mask, output_attentions=False)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor`

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: mindspore.Tensor,
    output_attentions: bool = False,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`):
            input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`):
            attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very
            large negative values.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)
    hidden_states, attn_weights, _ = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        output_attentions=output_attentions,
    )
    hidden_states = self.attn_dropout(hidden_states)
    hidden_states = residual + hidden_states

    residual = hidden_states

    hidden_states = self.ffn_layer_norm(hidden_states)

    hidden_states = self.ffn(hidden_states)
    hidden_states = self.ffn_dropout(hidden_states)

    hidden_states = residual + hidden_states

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (attn_weights,)

    return outputs

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TForSpeechToSpeech(SeamlessM4TPreTrainedModel):
    _keys_to_ignore_on_load_missing = ["text_encoder"]
    main_input_name = "input_features"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
        self.speech_encoder = SeamlessM4TSpeechEncoder(config)
        self.text_decoder = SeamlessM4TDecoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.t2u_model = SeamlessM4TTextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4TCodeHifiGan(config)

    def get_encoder(self):
        return self.speech_encoder

    def get_decoder(self):
        return self.text_decoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_decoder.embed_tokens = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_features: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This is the same forward method as `SeamlessM4TForSpeechToText`. It doesn't use `self.t2u_model`."
                "If you want to generate speech, use the `generate` method."
            )

            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        spkr_id: Optional[int] = 0,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
        """
        Generates translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_features, num_beams=4, speech_do_sample=True)` will successively perform
        beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Args:
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            spkr_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.

            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.


        Returns:
            `Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]`:
            - If `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
            - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
              sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        """
        batch_size = len(input_features) if input_features is not None else len(kwargs.get("inputs_embeds"))

        if tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
        else:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                    more languages for text translation than for speech synthesis."""
                    )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        text_generation_output = super().generate(input_features, **kwargs_text)
        sequences = text_generation_output.sequences

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        # get last_hidden_state from encoder
        encoder_hidden_states = self.speech_encoder(input_features=input_features, attention_mask=attention_mask)[0]

        # input modality = speech so new attention mask for the decoder
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
            )

        # take care of num_return_sequences
        # take most probable hidden states per batch of return_sequences
        # (batch_size*num_return_sequences, ...) -> (batch_size,...)
        if num_return_sequences > 1:
            idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
            idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
            idx_most_probable_sequences_per_batch = (
                idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
            )
            sequences = sequences[idx_most_probable_sequences_per_batch]

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # Compute t2u decoder_input_ids
        t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
        t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
        t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
        kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids

        # second generation
        unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
        output_unit_ids = unit_ids.copy()

        # get rid of t2u_decoder_input_ids
        unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
        # replace eos per pad
        unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

        if return_intermediate_token_ids:
            return SeamlessM4TGenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.generate(input_features=None, return_intermediate_token_ids=None, tgt_lang=None, spkr_id=0, **kwargs)

Generates translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_features, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)` DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
spkr_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
kwargs	Remaining dictionary of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4TGenerationOutput]	Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]:
Union[Tensor, SeamlessM4TGenerationOutput]	If return_intermediate_token_ids, returns [SeamlessM4TGenerationOutput].
Union[Tensor, SeamlessM4TGenerationOutput]	If not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

@no_grad()
def generate(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    spkr_id: Optional[int] = 0,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
    """
    Generates translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_features, num_beams=4, speech_do_sample=True)` will successively perform
    beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Args:
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        spkr_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.

        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.


    Returns:
        `Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]`:
        - If `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
        - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
          sequence_length)`and and `waveform_lengths` which gives the length of each sample.
    """
    batch_size = len(input_features) if input_features is not None else len(kwargs.get("inputs_embeds"))

    if tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
    else:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                more languages for text translation than for speech synthesis."""
                )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
    text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    text_generation_output = super().generate(input_features, **kwargs_text)
    sequences = text_generation_output.sequences

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    # get last_hidden_state from encoder
    encoder_hidden_states = self.speech_encoder(input_features=input_features, attention_mask=attention_mask)[0]

    # input modality = speech so new attention mask for the decoder
    if attention_mask is not None:
        sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
        attention_mask = _compute_new_attention_mask(
            hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
        )

    # take care of num_return_sequences
    # take most probable hidden states per batch of return_sequences
    # (batch_size*num_return_sequences, ...) -> (batch_size,...)
    if num_return_sequences > 1:
        idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
        idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
        idx_most_probable_sequences_per_batch = (
            idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
        )
        sequences = sequences[idx_most_probable_sequences_per_batch]

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # Compute t2u decoder_input_ids
    t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
    t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
    t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
    kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids

    # second generation
    unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
    output_unit_ids = unit_ids.copy()

    # get rid of t2u_decoder_input_ids
    unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
    # replace eos per pad
    unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

    if return_intermediate_token_ids:
        return SeamlessM4TGenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TForSpeechToText(SeamlessM4TPreTrainedModel):
    _keys_to_ignore_on_load_missing = ["text_decoder", "t2u_model", "vocoder"]
    main_input_name = "input_features"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config: SeamlessM4TConfig):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
        self.speech_encoder = SeamlessM4TSpeechEncoder(config)
        self.text_decoder = SeamlessM4TDecoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_encoder(self):
        return self.speech_encoder

    def get_decoder(self):
        return self.text_decoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_decoder.embed_tokens = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_features: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    def generate(
        self,
        input_features=None,
        tgt_lang=None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        **kwargs,
    ):
        """
        Generates sequences of token ids.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.

            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model.

        Return:
            [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
            [`~utils.ModelOutput`] types are:
                - [`~generation.GenerateEncoderDecoderOutput`],
                - [`~generation.GenerateBeamEncoderDecoderOutput`]
        """
        text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        input_features = input_features if input_features is not None else kwargs.pop("inputs")
        if tgt_lang is not None:
            inputs = kwargs.get("input_embeds") if input_features is None else input_features
            inputs = (
                inputs
                if inputs is not None
                else kwargs.get("encoder_outputs", {"last_hidden_state": None})["last_hidden_state"]
            )
            batch_size = len(inputs)

            if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
                # also accept __xxx__
                tgt_lang = tgt_lang.replace("__", "")
                if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                        {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                    )
                # tgt_lang gets priority over decoder input ids
                text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
                text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
            else:
                raise ValueError(
                    """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                    the target language to the right token id. Make sure to load the right generation config."""
                )
        else:
            # only a warning, otherwise errors appear in the tests
            logger.warning(
                """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
                a correct generation, otherwise the generation will probably make no sense."""
            )
        return super().generate(
            input_features,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            decoder_input_ids=text_decoder_input_ids,
            **kwargs,
        )

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.generate(input_features=None, tgt_lang=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, **kwargs)

Generates sequences of token ids.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)` DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

Return

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. The possible [~utils.ModelOutput] types are: - [~generation.GenerateEncoderDecoderOutput], - [~generation.GenerateBeamEncoderDecoderOutput]

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def generate(
    self,
    input_features=None,
    tgt_lang=None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    **kwargs,
):
    """
    Generates sequences of token ids.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.

        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model.

    Return:
        [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
        or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
        [`~utils.ModelOutput`] types are:
            - [`~generation.GenerateEncoderDecoderOutput`],
            - [`~generation.GenerateBeamEncoderDecoderOutput`]
    """
    text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    input_features = input_features if input_features is not None else kwargs.pop("inputs")
    if tgt_lang is not None:
        inputs = kwargs.get("input_embeds") if input_features is None else input_features
        inputs = (
            inputs
            if inputs is not None
            else kwargs.get("encoder_outputs", {"last_hidden_state": None})["last_hidden_state"]
        )
        batch_size = len(inputs)

        if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                    {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                )
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
        else:
            raise ValueError(
                """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                the target language to the right token id. Make sure to load the right generation config."""
            )
    else:
        # only a warning, otherwise errors appear in the tests
        logger.warning(
            """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
            a correct generation, otherwise the generation will probably make no sense."""
        )
    return super().generate(
        input_features,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        decoder_input_ids=text_decoder_input_ids,
        **kwargs,
    )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel):
    _keys_to_ignore_on_load_missing = ["speech_encoder"]
    main_input_name = "input_ids"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config: SeamlessM4TConfig):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4TEncoder(config, self.shared)
        self.text_decoder = SeamlessM4TDecoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.t2u_model = SeamlessM4TTextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4TCodeHifiGan(config)

    def get_encoder(self):
        return self.text_encoder

    def get_decoder(self):
        return self.text_decoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This is the same forward method as `SeamlessM4TForTextToText`."
                "It doesn't use the text-to-unit model `SeamlessM4TTextToUnitForConditionalGeneration`."
                "If you want to generate speech, use the `.generate` method."
            )
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        spkr_id: Optional[int] = 0,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
        """
        Generates translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_ids, num_beams=4, speech_do_sample=True)` will successively perform
        beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            spkr_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.


        Returns:
            `Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]`:
            - If `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
            - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
              sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        """
        batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

        if tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
        else:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                    more languages for text translation than for speech synthesis."""
                    )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")

        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        text_generation_output = super().generate(input_ids, **kwargs_text)
        sequences = text_generation_output.sequences

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

        # take care of num_return_sequences
        # take most probable hidden states per batch of return_sequences
        # (batch_size*num_return_sequences, ...) -> (batch_size,...)
        if num_return_sequences > 1:
            idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
            idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
            idx_most_probable_sequences_per_batch = (
                idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
            )
            sequences = sequences[idx_most_probable_sequences_per_batch]

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # Compute t2u decoder_input_ids
        t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
        t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
        t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
        kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids
        # second generation
        unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
        output_unit_ids = unit_ids.copy()

        # get rid of t2u_decoder_input_ids
        unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
        # replace eos per pad
        unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

        if return_intermediate_token_ids:
            return SeamlessM4TGenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.generate(input_ids=None, return_intermediate_token_ids=None, tgt_lang=None, spkr_id=0, **kwargs)

Generates translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_ids, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
spkr_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
kwargs	Remaining dictionary of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4TGenerationOutput]	Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]:
Union[Tensor, SeamlessM4TGenerationOutput]	If return_intermediate_token_ids, returns [SeamlessM4TGenerationOutput].
Union[Tensor, SeamlessM4TGenerationOutput]	If not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

@no_grad()
def generate(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    spkr_id: Optional[int] = 0,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
    """
    Generates translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_ids, num_beams=4, speech_do_sample=True)` will successively perform
    beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        spkr_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.


    Returns:
        `Union[SeamlessM4TGenerationOutput, Tuple[Tensor]]`:
        - If `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
        - If not `return_intermediate_token_ids`, returns a tuple composed of waveforms of shape `(batch_size,
          sequence_length)`and and `waveform_lengths` which gives the length of each sample.
    """
    batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

    if tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")
    else:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                more languages for text translation than for speech synthesis."""
                )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")

    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
    text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    text_generation_output = super().generate(input_ids, **kwargs_text)
    sequences = text_generation_output.sequences

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

    # take care of num_return_sequences
    # take most probable hidden states per batch of return_sequences
    # (batch_size*num_return_sequences, ...) -> (batch_size,...)
    if num_return_sequences > 1:
        idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
        idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
        idx_most_probable_sequences_per_batch = (
            idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
        )
        sequences = sequences[idx_most_probable_sequences_per_batch]

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # Compute t2u decoder_input_ids
    t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
    t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
    t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
    kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids
    # second generation
    unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
    output_unit_ids = unit_ids.copy()

    # get rid of t2u_decoder_input_ids
    unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
    # replace eos per pad
    unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

    if return_intermediate_token_ids:
        return SeamlessM4TGenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToText

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TForTextToText(SeamlessM4TPreTrainedModel):
    _keys_to_ignore_on_load_missing = ["speech_encoder", "t2u_model", "vocoder"]
    main_input_name = "input_ids"

    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config: SeamlessM4TConfig):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4TEncoder(config, self.shared)
        self.text_decoder = SeamlessM4TDecoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_encoder(self):
        return self.text_encoder

    def get_decoder(self):
        return self.text_decoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    def generate(
        self,
        input_ids=None,
        tgt_lang=None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        **kwargs,
    ):
        """
        Generates sequences of token ids.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            input_ids (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model.

        Return:
            [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
            [`~utils.ModelOutput`] types are:
                - [`~generation.GenerateEncoderDecoderOutput`],
                - [`~generation.GenerateBeamEncoderDecoderOutput`]
        """
        # prepare text_decoder_input_ids
        text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        if tgt_lang is not None:
            batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

            if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
                # also accept __xxx__
                tgt_lang = tgt_lang.replace("__", "")
                if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                        {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                    )
                # tgt_lang gets priority over decoder input ids
                text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
                text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
            else:
                raise ValueError(
                    """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                    the target language to the right token id. Make sure to load the right generation config."""
                )
        else:
            # only a warning, otherwise errors appear in the tests
            logger.warning(
                """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
                a correct generation, otherwise the generation will probably make no sense."""
            )

        return super().generate(
            input_ids,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            decoder_input_ids=text_decoder_input_ids,
            **kwargs,
        )

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToText.generate(input_ids=None, tgt_lang=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, **kwargs)

Generates sequences of token ids.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of varying shape depending on the modality, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

Return

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. The possible [~utils.ModelOutput] types are: - [~generation.GenerateEncoderDecoderOutput], - [~generation.GenerateBeamEncoderDecoderOutput]

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def generate(
    self,
    input_ids=None,
    tgt_lang=None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    **kwargs,
):
    """
    Generates sequences of token ids.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        input_ids (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model.

    Return:
        [`~utils.ModelOutput`] or `mindspore.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
        or when `config.return_dict_in_generate=True`) or a `mindspore.Tensor`. The possible
        [`~utils.ModelOutput`] types are:
            - [`~generation.GenerateEncoderDecoderOutput`],
            - [`~generation.GenerateBeamEncoderDecoderOutput`]
    """
    # prepare text_decoder_input_ids
    text_decoder_input_ids = kwargs.pop("decoder_input_ids", None)
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    if tgt_lang is not None:
        batch_size = len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds"))

        if hasattr(self.generation_config, "text_decoder_lang_to_code_id"):
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            if tgt_lang not in self.generation_config.text_decoder_lang_to_code_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model. Please specify a `tgt_lang` in
                    {', '.join(self.generation_config.text_decoder_lang_to_code_id.keys())}"""
                )
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)
        else:
            raise ValueError(
                """This model generation config doesn't have a `text_decoder_lang_to_code_id` key which maps
                the target language to the right token id. Make sure to load the right generation config."""
            )
    else:
        # only a warning, otherwise errors appear in the tests
        logger.warning(
            """You must either specify a `tgt_lang` or pass a correct `text_decoder_input_ids` to get
            a correct generation, otherwise the generation will probably make no sense."""
        )

    return super().generate(
        input_ids,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        decoder_input_ids=text_decoder_input_ids,
        **kwargs,
    )

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TGenerationOutput dataclass

Bases: ModelOutput

Class defining the generated outputs from [SeamlessM4TModel], [SeamlessM4TForTextToText], [SeamlessM4TForTextToSpeech], [SeamlessM4TForSpeechToSpeech] and [SeamlessM4TForTextToSpeech].

PARAMETER	DESCRIPTION
waveform	The final audio waveform predicted by the model. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
waveform_lengths	The length in samples of each element in the waveform batch. TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None
sequences	The generated translated sequences. This is the output of the text-to-text or the speech-to-text models. The second dimension (sequence_length) is either equal to max_length or shorter if all batches finished early due to the eos_token_id. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
unit_sequences	The generated translated unit sequences. This is the output of the text-to-units model. The second dimension (unit_sequence_length) is either equal to t2u_max_length or shorter if all batches finished early due to the t2u_eos_token_id. TYPE: `mindspore.Tensor` of shape `(batch_size, unit_sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

@dataclass
class SeamlessM4TGenerationOutput(ModelOutput):
    """
    Class defining the generated outputs from [`SeamlessM4TModel`], [`SeamlessM4TForTextToText`],
    [`SeamlessM4TForTextToSpeech`], [`SeamlessM4TForSpeechToSpeech`] and [`SeamlessM4TForTextToSpeech`].

    Args:
        waveform (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            The final audio waveform predicted by the model.
        waveform_lengths (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            The length in samples of each element in the `waveform` batch.
        sequences (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            The generated translated sequences. This is the output of the text-to-text or the speech-to-text models.
            The second dimension (sequence_length) is either equal to `max_length` or shorter if all batches finished
            early due to the `eos_token_id`.
        unit_sequences (`mindspore.Tensor` of shape `(batch_size, unit_sequence_length)`, *optional*):
            The generated translated unit sequences. This is the output of the text-to-units model. The second
            dimension (unit_sequence_length) is either equal to `t2u_max_length` or shorter if all batches finished
            early due to the `t2u_eos_token_id`.
    """

    waveform: Optional[mindspore.Tensor] = None
    waveform_lengths: Optional[mindspore.Tensor] = None
    sequences: Optional[Tuple[mindspore.Tensor]] = None
    unit_sequences: Optional[Tuple[mindspore.Tensor]] = None

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4THifiGan

Bases: Module

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4THifiGan(nn.Module):
    def __init__(self, config: SeamlessM4TConfig):
        super().__init__()
        model_in_dim = config.unit_embed_dim + config.lang_embed_dim + config.spkr_embed_dim
        self.leaky_relu_slope = config.leaky_relu_slope
        self.num_kernels = len(config.resblock_kernel_sizes)
        self.num_upsamples = len(config.upsample_rates)
        self.conv_pre = nn.Conv1d(
            model_in_dim,
            config.upsample_initial_channel,
            kernel_size=7,
            stride=1,
            padding=3,
        )

        self.upsampler = nn.ModuleList()
        for i, (upsample_rate, kernel_size) in enumerate(zip(config.upsample_rates, config.upsample_kernel_sizes)):
            self.upsampler.append(
                nn.ConvTranspose1d(
                    config.upsample_initial_channel // (2**i),
                    config.upsample_initial_channel // (2 ** (i + 1)),
                    kernel_size=kernel_size,
                    stride=upsample_rate,
                    padding=(kernel_size - upsample_rate) // 2,
                )
            )

        self.resblocks = nn.ModuleList()
        for i in range(len(self.upsampler)):
            channels = config.upsample_initial_channel // (2 ** (i + 1))
            for kernel_size, dilation in zip(config.resblock_kernel_sizes, config.resblock_dilation_sizes):
                self.resblocks.append(HifiGanResidualBlock(channels, kernel_size, dilation, config.leaky_relu_slope))

        self.conv_post = nn.Conv1d(channels, 1, kernel_size=7, stride=1, padding=3)

    def forward(self, input_embeds: mindspore.Tensor) -> mindspore.Tensor:
        r"""
        Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch
        of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech
        waveform.

        Args:
            spectrogram (`mindspore.Tensor`):
                Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length,
                model_in_dim)`, or un-batched and of shape `(sequence_length, model_in_dim)`. Note that `model_in_dim`
                is the sum of `config.unit_embed_dim`, `config.lang_embed_dim` and `config.spkr_embed_dim`.

        Returns:
            `mindspore.Tensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
            shape `(batch_size, num_frames,)`. If un-batched, will be of shape `(num_frames,)`.
        """

        hidden_states = self.conv_pre(input_embeds)
        for i in range(self.num_upsamples):
            hidden_states = nn.functional.leaky_relu(hidden_states, self.leaky_relu_slope)
            hidden_states = self.upsampler[i](hidden_states)

            res_state = self.resblocks[i * self.num_kernels](hidden_states)
            for j in range(1, self.num_kernels):
                res_state += self.resblocks[i * self.num_kernels + j](hidden_states)
            hidden_states = res_state / self.num_kernels

        hidden_states = nn.functional.leaky_relu(hidden_states)
        hidden_states = self.conv_post(hidden_states)
        hidden_states = ops.tanh(hidden_states)

        # remove seq-len dim since this collapses to 1
        waveform = hidden_states.squeeze(1)

        return waveform

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4THifiGan.forward(input_embeds)

Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech waveform.

PARAMETER	DESCRIPTION
spectrogram	Tensor containing the log-mel spectrograms. Can be batched and of shape (batch_size, sequence_length, model_in_dim), or un-batched and of shape (sequence_length, model_in_dim). Note that model_in_dim is the sum of config.unit_embed_dim, config.lang_embed_dim and config.spkr_embed_dim. TYPE: `mindspore.Tensor`

RETURNS	DESCRIPTION
Tensor	mindspore.Tensor: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
Tensor	shape (batch_size, num_frames,). If un-batched, will be of shape (num_frames,).

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def forward(self, input_embeds: mindspore.Tensor) -> mindspore.Tensor:
    r"""
    Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch
    of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech
    waveform.

    Args:
        spectrogram (`mindspore.Tensor`):
            Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length,
            model_in_dim)`, or un-batched and of shape `(sequence_length, model_in_dim)`. Note that `model_in_dim`
            is the sum of `config.unit_embed_dim`, `config.lang_embed_dim` and `config.spkr_embed_dim`.

    Returns:
        `mindspore.Tensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of
        shape `(batch_size, num_frames,)`. If un-batched, will be of shape `(num_frames,)`.
    """

    hidden_states = self.conv_pre(input_embeds)
    for i in range(self.num_upsamples):
        hidden_states = nn.functional.leaky_relu(hidden_states, self.leaky_relu_slope)
        hidden_states = self.upsampler[i](hidden_states)

        res_state = self.resblocks[i * self.num_kernels](hidden_states)
        for j in range(1, self.num_kernels):
            res_state += self.resblocks[i * self.num_kernels + j](hidden_states)
        hidden_states = res_state / self.num_kernels

    hidden_states = nn.functional.leaky_relu(hidden_states)
    hidden_states = self.conv_post(hidden_states)
    hidden_states = ops.tanh(hidden_states)

    # remove seq-len dim since this collapses to 1
    waveform = hidden_states.squeeze(1)

    return waveform

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel

Bases: SeamlessM4TPreTrainedModel

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TModel(SeamlessM4TPreTrainedModel):
    _tied_weights_keys = [
        "lm_head.weight",
        "text_encoder.embed_tokens.weight",
        "text_decoder.embed_tokens.weight",
    ]

    def __init__(self, config, current_modality="text"):
        super().__init__(config)

        self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)

        self.text_encoder = SeamlessM4TEncoder(config, self.shared)
        self.speech_encoder = SeamlessM4TSpeechEncoder(config)
        self.text_decoder = SeamlessM4TDecoder(config, self.shared)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

        self.current_modality = current_modality
        if current_modality == "speech":
            self.main_input_name = "input_features"

        # these models already call post_init in their initialization
        self.t2u_model = SeamlessM4TTextToUnitForConditionalGeneration(config)
        self.vocoder = SeamlessM4TCodeHifiGan(config)

    def set_modality(self, modality="text"):
        if modality == "text":
            self.main_input_name = "input_ids"
            self.current_modality = "text"
        elif modality == "speech":
            self.main_input_name = "input_features"
            self.current_modality = "speech"
        else:
            raise ValueError(f"`modality={modality}` is not a valid modality. It must be `text` or `speech`.")

    def get_encoder(self):
        if self.current_modality == "text":
            return self.text_encoder
        else:
            return self.speech_encoder

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def get_input_embeddings(self):
        return self.text_decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.text_encoder.embed_tokens = value
        self.text_decoder.embed_tokens = value
        self.shared = value

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.text_encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.lm_head, self.shared)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        input_features: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Seq2SeqLMOutput, Tuple[mindspore.Tensor]]:
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        if input_ids is None and input_features is None and inputs_embeds is None and encoder_outputs is None:
            raise ValueError(
                "`input_ids`,`input_features`, `inputs_embeds` and `encoder_outputs` are all empty. Make sure at least one of them is not."
            )
        elif input_features is not None:
            if input_ids is not None:
                logger.warning(
                    "`input_ids` is not `None` but `input_features` has been given."
                    "`input_features` will be used in priority through the `speech_encoder`. "
                    "Make sure that `input_features` and `input_ids` are mutually exclusive."
                )

            if inputs_embeds is not None:
                logger.warning(
                    "`inputs_embeds` is not `None` but `input_features` has been given."
                    "`input_features` will be used in priority through `speech_encoder`. "
                    "`inputs_embeds` will be ignored."
                )

            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This calls the same method `forward` as `SeamlessM4TForTextToText` and `SeamlessM4TForSpeechToText`"
                "depending on the input modality. If you want to generate speech, use the `generate` method."
            )

            self.set_modality("speech")

            encoder_outputs = self.speech_encoder(
                input_features=input_features,
                attention_mask=attention_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        elif input_ids is not None or inputs_embeds is not None:
            # if encoder_outputs is not None, it's probably used within a .generate method so no need to warn
            logger.warning(
                "This calls the same method `forward` as `SeamlessM4TForTextToText` and `SeamlessM4TForSpeechToText`"
                "depending on the input modality. If you want to generate speech, use the `generate` method."
            )
            self.set_modality("text")
            encoder_outputs = self.text_encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        encoder_attention_mask = attention_mask
        # input modality = speech so new attention mask
        if self.current_modality == "speech" and attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            encoder_attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_outputs[0], seq_lens=sub_sampled_lengths
            )

        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.text_decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        lm_logits = self.lm_head(decoder_outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            outputs = decoder_outputs + encoder_outputs
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    @no_grad()
    def generate(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        input_features: Optional[mindspore.Tensor] = None,
        return_intermediate_token_ids: Optional[bool] = None,
        tgt_lang: Optional[str] = None,
        spkr_id: Optional[int] = 0,
        generate_speech: Optional[bool] = True,
        **kwargs,
    ) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
        """
        Generates translated token ids and/or translated audio waveforms.

        <Tip>

        This method successively calls the `.generate` function of two different sub-models. You can specify keyword
        arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
        that will be passed to one of them.

        For example, calling `.generate(input_ids=input_ids, num_beams=4, speech_do_sample=True)` will successively
        perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>


        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
                [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional*):
                Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
                [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
            return_intermediate_token_ids (`bool`, *optional*):
                If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
                to get translated text alongside the audio. Note that if `generate_speech=True`, this parameter will be
                ignored.
            tgt_lang (`str`, *optional*):
                The language to use as target language for translation.
            spkr_id (`int`, *optional*, defaults to 0):
                The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
            generate_speech (`bool`, *optional*, defaults to `True`):
                If `False`, will only returns the text tokens and won't generate speech.

            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
                arguments are of two types:

                    - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                    except for `decoder_input_ids` which will only be passed through the text components.
                    - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                    text model and speech model respectively. It has the priority over the keywords without a prefix.

                    This means you can, for example, specify a generation strategy for one generation but not for the
                    other.

        Returns:
            `Union[SeamlessM4TGenerationOutput, Tuple[Tensor], ModelOutput]`:
            - If `generate_speech` and `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
            - If `generate_speech` and not `return_intermediate_token_ids`, returns a tuple composed of waveforms of
              shape `(batch_size, sequence_length)`and and `waveform_lengths` which gives the length of each sample.
            - If `generate_speech=False`, it will returns `ModelOutput`.
        """
        if input_ids is None and input_features is None and kwargs.get("inputs_embeds", None) is None:
            raise ValueError(
                "`input_ids`,`input_features` and `inputs_embeds` are all empty. Make sure at least one of them is not."
            )

        if generate_speech and tgt_lang is None:
            raise ValueError("You must specify a `tgt_lang` to generate translated speech.")

        if tgt_lang is not None:
            # also accept __xxx__
            tgt_lang = tgt_lang.replace("__", "")
            for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
                lang_code_to_id = getattr(self.generation_config, key, None)
                if lang_code_to_id is None:
                    raise ValueError(
                        f"""This model generation config doesn't have a `{key}` key which maps the target language
                        to the right token id. Make sure to load the right generation config."""
                    )
                elif tgt_lang not in lang_code_to_id:
                    raise ValueError(
                        f"""`tgt_lang={tgt_lang}` is not supported by this model.
                    Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                    more languages for text translation than for speech synthesis."""
                    )

        batch_size = (
            len(input_features)
            if input_features is not None
            else (len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds")))
        )

        kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
        kwargs_text["output_hidden_states"] = True
        kwargs_text["return_dict_in_generate"] = True
        kwargs_text["output_scores"] = True

        text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
        # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
        if tgt_lang is not None:
            # tgt_lang gets priority over decoder input ids
            text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
            text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

        kwargs_text["decoder_input_ids"] = text_decoder_input_ids

        # first generation
        if input_features is not None:
            self.set_modality("speech")
            if input_ids is not None:
                logger.warning(
                    "`input_features` and `input_ids` are both non empty. `input_features` will be used in priority "
                    "through the speech encoder. Make sure `input_features=None` if you want to use the text encoder."
                )
            text_generation_output = super().generate(input_features=input_features, **kwargs_text)
        else:
            self.set_modality("text")
            text_generation_output = super().generate(input_ids=input_ids, input_features=None, **kwargs_text)
        sequences = text_generation_output.sequences

        if not generate_speech:
            return text_generation_output

        # prepare second generation
        num_return_sequences = len(sequences) // batch_size
        attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

        # get encoder last hidden states
        if self.current_modality == "speech":
            # get last_hidden_state from encoder - must do a pass through the speech encoder
            encoder_hidden_states = self.speech_encoder(
                input_features=input_features, attention_mask=attention_mask
            ).last_hidden_state

            # input modality = speech so new attention mask for the decoder
            if attention_mask is not None:
                sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
                attention_mask = _compute_new_attention_mask(
                    hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
                )
        else:
            encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

        # take care of num_return_sequences
        # take most probable hidden states per batch of return_sequences
        # (batch_size*num_return_sequences, ...) -> (batch_size,...)
        if num_return_sequences > 1:
            idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
            idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
            idx_most_probable_sequences_per_batch = (
                idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
            )
            sequences = sequences[idx_most_probable_sequences_per_batch]

        # get decoder last hidden state - must do a pass through the text decoder
        t2u_input_embeds = self.text_decoder(
            input_ids=sequences,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=attention_mask,
        ).last_hidden_state

        pad_token_id = self.generation_config.pad_token_id

        # Compute new attention mask
        seq_lens = (sequences != pad_token_id).int().sum(1)
        t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
        kwargs_speech["attention_mask"] = t2u_model_attention_mask

        # Compute t2u decoder_input_ids
        t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
        t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
        t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
        kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids

        # second generation
        unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
        output_unit_ids = unit_ids.copy()

        # get rid of t2u_decoder_input_ids
        unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
        # replace eos per pad
        unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
        # offset of control symbols
        unit_ids = ops.where(
            unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
        )

        vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
        vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

        spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

        waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

        if return_intermediate_token_ids:
            return SeamlessM4TGenerationOutput(
                waveform=waveform,
                waveform_lengths=waveform_lengths,
                sequences=sequences,
                unit_sequences=output_unit_ids,
            )

        return waveform, waveform_lengths

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        # cut decoder_input_ids if past is used
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            # cached cross_attention states don't have to be reordered -> they are always the same
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:],
            )
        return reordered_past

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.generate(input_ids=None, input_features=None, return_intermediate_token_ids=None, tgt_lang=None, spkr_id=0, generate_speech=True, **kwargs)

Generates translated token ids and/or translated audio waveforms.

This method successively calls the .generate function of two different sub-models. You can specify keyword arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments that will be passed to one of them.

For example, calling .generate(input_ids=input_ids, num_beams=4, speech_do_sample=True) will successively perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Indices can be obtained using [SeamlessM4TTokenizer] or [SeamlessM4TProcessor]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
input_features	Input audio features. This should be returnes by the [SeamlessM4TFeatureExtractor] class or the [SeamlessM4TProcessor] class. See [SeamlessM4TFeatureExtractor.__call__] for details. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional* DEFAULT: None
return_intermediate_token_ids	If True, also returns the intermediate generated text and unit tokens. Set to True if you also want to get translated text alongside the audio. Note that if generate_speech=True, this parameter will be ignored. TYPE: `bool`, *optional* DEFAULT: None
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional* DEFAULT: None
spkr_id	The id of the speaker used for speech synthesis. Must be lower than config.vocoder_num_spkrs. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
generate_speech	If False, will only returns the text tokens and won't generate speech. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
kwargs	Remaining dictionary of keyword arguments that will be passed to [GenerationMixin.generate]. Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Union[Tensor, SeamlessM4TGenerationOutput]	Union[SeamlessM4TGenerationOutput, Tuple[Tensor], ModelOutput]:
Union[Tensor, SeamlessM4TGenerationOutput]	If generate_speech and return_intermediate_token_ids, returns [SeamlessM4TGenerationOutput].
Union[Tensor, SeamlessM4TGenerationOutput]	If generate_speech and not return_intermediate_token_ids, returns a tuple composed of waveforms of shape (batch_size, sequence_length)and and waveform_lengths which gives the length of each sample.
Union[Tensor, SeamlessM4TGenerationOutput]	If generate_speech=False, it will returns ModelOutput.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

@no_grad()
def generate(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    input_features: Optional[mindspore.Tensor] = None,
    return_intermediate_token_ids: Optional[bool] = None,
    tgt_lang: Optional[str] = None,
    spkr_id: Optional[int] = 0,
    generate_speech: Optional[bool] = True,
    **kwargs,
) -> Union[mindspore.Tensor, SeamlessM4TGenerationOutput]:
    """
    Generates translated token ids and/or translated audio waveforms.

    <Tip>

    This method successively calls the `.generate` function of two different sub-models. You can specify keyword
    arguments at two different levels: general arguments that will be passed to both models, or prefixed arguments
    that will be passed to one of them.

    For example, calling `.generate(input_ids=input_ids, num_beams=4, speech_do_sample=True)` will successively
    perform beam-search decoding on the text model, and multinomial beam-search sampling on the speech model.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>


    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        input_features (`mindspore.Tensor` of shape `(batch_size, sequence_length, num_banks)`, *optional*):
            Input audio features. This should be returnes by the [`SeamlessM4TFeatureExtractor`] class or the
            [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details.
        return_intermediate_token_ids (`bool`, *optional*):
            If `True`, also returns the intermediate generated text and unit tokens. Set to `True` if you also want
            to get translated text alongside the audio. Note that if `generate_speech=True`, this parameter will be
            ignored.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        spkr_id (`int`, *optional*, defaults to 0):
            The id of the speaker used for speech synthesis. Must be lower than `config.vocoder_num_spkrs`.
        generate_speech (`bool`, *optional*, defaults to `True`):
            If `False`, will only returns the text tokens and won't generate speech.

        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`GenerationMixin.generate`]. Keyword
            arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.

    Returns:
        `Union[SeamlessM4TGenerationOutput, Tuple[Tensor], ModelOutput]`:
        - If `generate_speech` and `return_intermediate_token_ids`, returns [`SeamlessM4TGenerationOutput`].
        - If `generate_speech` and not `return_intermediate_token_ids`, returns a tuple composed of waveforms of
          shape `(batch_size, sequence_length)`and and `waveform_lengths` which gives the length of each sample.
        - If `generate_speech=False`, it will returns `ModelOutput`.
    """
    if input_ids is None and input_features is None and kwargs.get("inputs_embeds", None) is None:
        raise ValueError(
            "`input_ids`,`input_features` and `inputs_embeds` are all empty. Make sure at least one of them is not."
        )

    if generate_speech and tgt_lang is None:
        raise ValueError("You must specify a `tgt_lang` to generate translated speech.")

    if tgt_lang is not None:
        # also accept __xxx__
        tgt_lang = tgt_lang.replace("__", "")
        for key in ["text_decoder_lang_to_code_id", "t2u_lang_code_to_id", "vocoder_lang_code_to_id"]:
            lang_code_to_id = getattr(self.generation_config, key, None)
            if lang_code_to_id is None:
                raise ValueError(
                    f"""This model generation config doesn't have a `{key}` key which maps the target language
                    to the right token id. Make sure to load the right generation config."""
                )
            elif tgt_lang not in lang_code_to_id:
                raise ValueError(
                    f"""`tgt_lang={tgt_lang}` is not supported by this model.
                Please specify a `tgt_lang` in {','.join(lang_code_to_id.keys())}. Note that SeamlessM4T supports
                more languages for text translation than for speech synthesis."""
                )

    batch_size = (
        len(input_features)
        if input_features is not None
        else (len(input_ids) if input_ids is not None else len(kwargs.get("inputs_embeds")))
    )

    kwargs_text, kwargs_speech = format_speech_generation_kwargs(kwargs)
    kwargs_text["output_hidden_states"] = True
    kwargs_text["return_dict_in_generate"] = True
    kwargs_text["output_scores"] = True

    text_decoder_input_ids = kwargs_text.get("decoder_input_ids")
    # overwrite text_decoder_input_ids if tgt_lang is passed. The latter gets priority over decoder_input_ids.
    if tgt_lang is not None:
        # tgt_lang gets priority over decoder input ids
        text_tgt_lang_id = self.generation_config.text_decoder_lang_to_code_id.get(tgt_lang)
        text_decoder_input_ids = mindspore.tensor([[text_tgt_lang_id]] * batch_size)

    kwargs_text["decoder_input_ids"] = text_decoder_input_ids

    # first generation
    if input_features is not None:
        self.set_modality("speech")
        if input_ids is not None:
            logger.warning(
                "`input_features` and `input_ids` are both non empty. `input_features` will be used in priority "
                "through the speech encoder. Make sure `input_features=None` if you want to use the text encoder."
            )
        text_generation_output = super().generate(input_features=input_features, **kwargs_text)
    else:
        self.set_modality("text")
        text_generation_output = super().generate(input_ids=input_ids, input_features=None, **kwargs_text)
    sequences = text_generation_output.sequences

    if not generate_speech:
        return text_generation_output

    # prepare second generation
    num_return_sequences = len(sequences) // batch_size
    attention_mask = kwargs_speech.get("attention_mask", kwargs_text.get("attention_mask", None))

    # get encoder last hidden states
    if self.current_modality == "speech":
        # get last_hidden_state from encoder - must do a pass through the speech encoder
        encoder_hidden_states = self.speech_encoder(
            input_features=input_features, attention_mask=attention_mask
        ).last_hidden_state

        # input modality = speech so new attention mask for the decoder
        if attention_mask is not None:
            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(attention_mask)
            attention_mask = _compute_new_attention_mask(
                hidden_states=encoder_hidden_states, seq_lens=sub_sampled_lengths
            )
    else:
        encoder_hidden_states = text_generation_output.encoder_hidden_states[-1]

    # take care of num_return_sequences
    # take most probable hidden states per batch of return_sequences
    # (batch_size*num_return_sequences, ...) -> (batch_size,...)
    if num_return_sequences > 1:
        idx_most_probable_sequences_per_batch = text_generation_output.sequences_scores.view(batch_size, -1)
        idx_most_probable_sequences_per_batch = idx_most_probable_sequences_per_batch.argmax(-1)
        idx_most_probable_sequences_per_batch = (
            idx_most_probable_sequences_per_batch + ops.arange(batch_size) * num_return_sequences
        )
        sequences = sequences[idx_most_probable_sequences_per_batch]

    # get decoder last hidden state - must do a pass through the text decoder
    t2u_input_embeds = self.text_decoder(
        input_ids=sequences,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=attention_mask,
    ).last_hidden_state

    pad_token_id = self.generation_config.pad_token_id

    # Compute new attention mask
    seq_lens = (sequences != pad_token_id).int().sum(1)
    t2u_model_attention_mask = _compute_new_attention_mask(t2u_input_embeds, seq_lens)
    kwargs_speech["attention_mask"] = t2u_model_attention_mask

    # Compute t2u decoder_input_ids
    t2u_decoder_input_ids = kwargs_speech.get("decoder_input_ids")
    t2u_tgt_lang_id = self.generation_config.t2u_lang_code_to_id.get(tgt_lang)
    t2u_decoder_input_ids = mindspore.tensor([[self.config.t2u_eos_token_id, t2u_tgt_lang_id]] * batch_size)
    kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids

    # second generation
    unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech)
    output_unit_ids = unit_ids.copy()

    # get rid of t2u_decoder_input_ids
    unit_ids = unit_ids[:, kwargs_speech["decoder_input_ids"].shape[1] :]
    # replace eos per pad
    unit_ids[unit_ids == self.config.t2u_eos_token_id] = self.config.t2u_pad_token_id
    # offset of control symbols
    unit_ids = ops.where(
        unit_ids == self.config.t2u_pad_token_id, unit_ids, unit_ids - self.config.vocoder_offset
    )

    vocoder_tgt_lang_id = self.generation_config.vocoder_lang_code_to_id.get(tgt_lang)
    vocoder_tgt_lang_id = mindspore.tensor([[vocoder_tgt_lang_id]] * len(unit_ids))

    spkr_id = mindspore.tensor([[spkr_id]] * len(unit_ids))

    waveform, waveform_lengths = self.vocoder(input_ids=unit_ids, spkr_id=spkr_id, lang_id=vocoder_tgt_lang_id)

    if return_intermediate_token_ids:
        return SeamlessM4TGenerationOutput(
            waveform=waveform,
            waveform_lengths=waveform_lengths,
            sequences=sequences,
            unit_sequences=output_unit_ids,
        )

    return waveform, waveform_lengths

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TPreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TPreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """

    config_class = SeamlessM4TConfig
    base_model_prefix = "seamless_m4t"
    supports_gradient_checkpointing = True
    _no_split_modules = ["SeamlessM4TEncoderLayer", "SeamlessM4TDecoderLayer", "SeamlessM4TConformerEncoderLayer"]

    def _init_weights(self, module):
        """Initialize the weights"""
        std = self.config.initializer_range
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, mean=0.0, std=std)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=std)
            if module.padding_idx is not None:
                module.weight[module.padding_idx] = 0
        elif isinstance(module, SeamlessM4TConformerSelfAttention):
            if hasattr(module, "pos_bias_u"):
                nn.init.xavier_uniform_(module.pos_bias_u)
            if hasattr(module, "pos_bias_v"):
                nn.init.xavier_uniform_(module.pos_bias_v)
        elif isinstance(module, SeamlessM4TConformerPositionalConvEmbedding):
            nn.init.normal_(
                module.conv.weight,
                mean=0,
                std=2 * math.sqrt(1 / (module.conv.kernel_size[0] * module.conv.in_channels)),
            )
            nn.init.constant_(module.conv.bias, 0)
        elif isinstance(module, SeamlessM4TConformerFeatureProjection):
            k = math.sqrt(1 / module.projection.in_features)
            nn.init.uniform_(module.projection.weight, a=-k, b=k)
            nn.init.uniform_(module.projection.bias, a=-k, b=k)
        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
            nn.init.zeros_(module.bias)
            nn.init.ones_(module.weight)
        elif isinstance(module, nn.Conv1d):
            nn.init.kaiming_normal_(module.weight)
            if module.bias is not None:
                k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
                nn.init.uniform_(module.bias, a=-k, b=k)

    def _compute_sub_sample_lengths_from_attention_mask(self, attention_mask):
        kernel_size, stride = self.config.adaptor_kernel_size, self.config.adaptor_stride
        pad = kernel_size // 2
        seq_lens = attention_mask.shape[1] - (1 - attention_mask.int()).sum(1)

        seq_lens = ((seq_lens + 2 * pad - kernel_size) / stride) + 1

        return seq_lens.floor()

    def compute_last_hidden_states_per_sample(
        self,
        hidden_states: Tuple[Tuple[mindspore.Tensor]],
        beam_indices: Optional[mindspore.Tensor] = None,
    ) -> mindspore.Tensor:
        """
        Computes the last hidden states.

        Parameters:
            hidden_states (`Tuple[Tuple[mindspore.Tensor]]`):
                The generated hidden states. Tuple (one element for each generated token) of tuples (one element for
                each layer of the decoder) of mindspore.Tensor of shape (batch_size*num_beams*num_return_sequences,
                generated_length, hidden_size).
            beam_indices (`mindspore.Tensor`, *optional*):
                Beam indices of generated token id at each generation step. `mindspore.Tensor` of shape
                `(batch_size*num_return_sequences, sequence_length)`. Only required if a `num_beams>1` at
                generate-time.

        Return:
            `mindspore.Tensor`: A `mindspore.Tensor` of shape `(batch_size*num_return_sequences, sequence_length, hidden_size)`
            containing
                the last hidden states.
        ```"""
        # 1. First, let's compute last_hidden_states from hidden_states.
        # For each generation step, takes the hidden state from the last layer.
        # shape: (batch_size*vocab_size*num_return_sequences, # generation_steps, hidden_dim)
        last_hidden_states = ops.concat([hidden_states[-1] for hidden_states in hidden_states], dim=1)

        # 2. In absence of `beam_indices`, we can assume that we come from e.g. greedy search, which is equivalent
        # to a beam search approach were the first (and only) beam is always selected
        # in that case, return directly last_hidden_states
        if beam_indices is None:
            return last_hidden_states

        # 3. cut beam_indices to longest beam length
        beam_indices_mask = beam_indices < 0
        max_beam_length = (1 - beam_indices_mask.long()).sum(-1).max()
        beam_indices = beam_indices.copy()[:, :max_beam_length]
        beam_indices_mask = beam_indices_mask[:, :max_beam_length]

        # 4. Set indices of beams that finished early to 0; such indices will be masked correctly afterwards anyways
        beam_indices[beam_indices_mask] = 0

        # 5. broadcast_to beam_indices to last_hidden_states dim
        beam_indices = beam_indices.unsqueeze(-1)
        beam_indices = beam_indices.broadcast_to((-1, -1, last_hidden_states.shape[-1]))

        # 6. select the right candidate for each beam
        # in other words, new_last_hidden_states[i,j,k] = last_hidden_states[beam_indices[i,j,k], j, k] for all i, j, k
        last_hidden_states = ops.gather(last_hidden_states, 0, beam_indices)

        return last_hidden_states

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TPreTrainedModel.compute_last_hidden_states_per_sample(hidden_states, beam_indices=None)

Computes the last hidden states.

PARAMETER	DESCRIPTION
hidden_states	The generated hidden states. Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of mindspore.Tensor of shape (batch_size*num_beams*num_return_sequences, generated_length, hidden_size). TYPE: `Tuple[Tuple[mindspore.Tensor]]`
beam_indices	Beam indices of generated token id at each generation step. mindspore.Tensor of shape (batch_size*num_return_sequences, sequence_length). Only required if a num_beams>1 at generate-time. TYPE: `mindspore.Tensor`, *optional* DEFAULT: None

Return

mindspore.Tensor: A mindspore.Tensor of shape (batch_size*num_return_sequences, sequence_length, hidden_size) containing the last hidden states.

```

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def compute_last_hidden_states_per_sample(
    self,
    hidden_states: Tuple[Tuple[mindspore.Tensor]],
    beam_indices: Optional[mindspore.Tensor] = None,
) -> mindspore.Tensor:
    """
    Computes the last hidden states.

    Parameters:
        hidden_states (`Tuple[Tuple[mindspore.Tensor]]`):
            The generated hidden states. Tuple (one element for each generated token) of tuples (one element for
            each layer of the decoder) of mindspore.Tensor of shape (batch_size*num_beams*num_return_sequences,
            generated_length, hidden_size).
        beam_indices (`mindspore.Tensor`, *optional*):
            Beam indices of generated token id at each generation step. `mindspore.Tensor` of shape
            `(batch_size*num_return_sequences, sequence_length)`. Only required if a `num_beams>1` at
            generate-time.

    Return:
        `mindspore.Tensor`: A `mindspore.Tensor` of shape `(batch_size*num_return_sequences, sequence_length, hidden_size)`
        containing
            the last hidden states.
    ```"""
    # 1. First, let's compute last_hidden_states from hidden_states.
    # For each generation step, takes the hidden state from the last layer.
    # shape: (batch_size*vocab_size*num_return_sequences, # generation_steps, hidden_dim)
    last_hidden_states = ops.concat([hidden_states[-1] for hidden_states in hidden_states], dim=1)

    # 2. In absence of `beam_indices`, we can assume that we come from e.g. greedy search, which is equivalent
    # to a beam search approach were the first (and only) beam is always selected
    # in that case, return directly last_hidden_states
    if beam_indices is None:
        return last_hidden_states

    # 3. cut beam_indices to longest beam length
    beam_indices_mask = beam_indices < 0
    max_beam_length = (1 - beam_indices_mask.long()).sum(-1).max()
    beam_indices = beam_indices.copy()[:, :max_beam_length]
    beam_indices_mask = beam_indices_mask[:, :max_beam_length]

    # 4. Set indices of beams that finished early to 0; such indices will be masked correctly afterwards anyways
    beam_indices[beam_indices_mask] = 0

    # 5. broadcast_to beam_indices to last_hidden_states dim
    beam_indices = beam_indices.unsqueeze(-1)
    beam_indices = beam_indices.broadcast_to((-1, -1, last_hidden_states.shape[-1]))

    # 6. select the right candidate for each beam
    # in other words, new_last_hidden_states[i,j,k] = last_hidden_states[beam_indices[i,j,k], j, k] for all i, j, k
    last_hidden_states = ops.gather(last_hidden_states, 0, beam_indices)

    return last_hidden_states

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TScaledWordEmbedding

Bases: Embedding

This module overrides nn.Embeddings' forward by multiplying with embeddings scale.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TScaledWordEmbedding(nn.Embedding):
    """
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    """

    def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, embed_scale: Optional[float] = 1.0):
        super().__init__(num_embeddings, embedding_dim, padding_idx)
        self.embed_scale = embed_scale

    def forward(self, input_ids: mindspore.Tensor):
        return super().forward(input_ids) * self.embed_scale

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TSinusoidalPositionalEmbedding

Bases: Module

This module produces sinusoidal positional embeddings of any length.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

class SeamlessM4TSinusoidalPositionalEmbedding(nn.Module):
    """This module produces sinusoidal positional embeddings of any length."""

    def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None):
        super().__init__()
        self.offset = 2
        self.embedding_dim = embedding_dim
        self.padding_idx = padding_idx
        self.make_weights(num_positions + self.offset, embedding_dim, padding_idx)

    def make_weights(self, num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        emb_weights = self.get_embedding(num_embeddings, embedding_dim, padding_idx)
        if hasattr(self, "weights"):
            # in forward put the weights on the correct dtype and device of the param
            emb_weights = emb_weights.to(dtype=self.weights.dtype)

        self.register_buffer("weights", emb_weights, persistent=False)

    @staticmethod
    def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        """
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        """
        half_dim = embedding_dim // 2
        emb = math.log(10000) / (half_dim - 1)
        emb = ops.exp(ops.arange(half_dim, dtype=mindspore.int64).float() * -emb)
        emb = ops.arange(num_embeddings, dtype=mindspore.int64).float().unsqueeze(1) * emb.unsqueeze(0)
        emb = ops.cat([ops.sin(emb), ops.cos(emb)], dim=1).view(num_embeddings, -1)
        if embedding_dim % 2 == 1:
            # zero pad
            emb = ops.cat([emb, ops.zeros(num_embeddings, 1)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0

        return emb.to(get_default_dtype())

    @no_grad()
    def forward(
        self, input_ids: mindspore.Tensor = None, inputs_embeds: mindspore.Tensor = None, past_key_values_length: int = 0
    ):
        if input_ids is not None:
            bsz, seq_len = input_ids.shape
            # Create the position ids from the input token ids. Any padded tokens remain padded.
            position_ids = create_position_ids_from_input_ids(input_ids, self.padding_idx, past_key_values_length)
        else:
            bsz, seq_len = inputs_embeds.shape[:-1]
            position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length)

        # expand embeddings if needed
        max_pos = self.padding_idx + 1 + seq_len + past_key_values_length
        if max_pos > self.weights.shape[0]:
            self.make_weights(max_pos + self.offset, self.embedding_dim, self.padding_idx)

        return self.weights.index_select(0, position_ids.view(-1)).view(bsz, seq_len, self.weights.shape[-1])

    def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length):
        """
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: mindspore.Tensor

        Returns: mindspore.Tensor
        """
        input_shape = inputs_embeds.shape[:-1]
        sequence_length = input_shape[1]

        position_ids = ops.arange(
            self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=mindspore.int64
        )
        return position_ids.unsqueeze(0).broadcast_to(input_shape) + past_key_values_length

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TSinusoidalPositionalEmbedding.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length)

We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

PARAMETER	DESCRIPTION
inputs_embeds	mindspore.Tensor

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length):
    """
    We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

    Args:
        inputs_embeds: mindspore.Tensor

    Returns: mindspore.Tensor
    """
    input_shape = inputs_embeds.shape[:-1]
    sequence_length = input_shape[1]

    position_ids = ops.arange(
        self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=mindspore.int64
    )
    return position_ids.unsqueeze(0).broadcast_to(input_shape) + past_key_values_length

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TSinusoidalPositionalEmbedding.get_embedding(num_embeddings, embedding_dim, padding_idx=None) staticmethod

Build sinusoidal embeddings.

This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of "Attention Is All You Need".

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

@staticmethod
def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
    """
    Build sinusoidal embeddings.

    This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
    "Attention Is All You Need".
    """
    half_dim = embedding_dim // 2
    emb = math.log(10000) / (half_dim - 1)
    emb = ops.exp(ops.arange(half_dim, dtype=mindspore.int64).float() * -emb)
    emb = ops.arange(num_embeddings, dtype=mindspore.int64).float().unsqueeze(1) * emb.unsqueeze(0)
    emb = ops.cat([ops.sin(emb), ops.cos(emb)], dim=1).view(num_embeddings, -1)
    if embedding_dim % 2 == 1:
        # zero pad
        emb = ops.cat([emb, ops.zeros(num_embeddings, 1)], dim=1)
    if padding_idx is not None:
        emb[padding_idx, :] = 0

    return emb.to(get_default_dtype())

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0)

Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's utils.make_positions.

PARAMETER	DESCRIPTION
x	mindspore.Tensor x:

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0):
    """
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: mindspore.Tensor x:

    Returns: mindspore.Tensor
    """
    # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
    mask = input_ids.ne(padding_idx).int()
    incremental_indices = (ops.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask
    return incremental_indices.long() + padding_idx

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.format_speech_generation_kwargs(kwargs)

Format kwargs for SeamlessM4T models that generate speech, attribute kwargs to either the text generation or the speech generation models.

PARAMETER

DESCRIPTION

kwargs

Keyword arguments are of two types: - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model, except for `decoder_input_ids` which will only be passed through the text components. - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the text model and speech model respectively. It has the priority over the keywords without a prefix. This means you can, for example, specify a generation strategy for one generation but not for the other. TYPE: `dict`)`

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def format_speech_generation_kwargs(kwargs):
    """
    Format kwargs for SeamlessM4T models that generate speech, attribute kwargs to either the text generation or the
    speech generation models.

    Args:
        kwargs (`dict`)`:
             Keyword arguments are of two types:

                - Without a prefix, they will be entered as `**kwargs` for the `generate` method of each sub-model,
                except for `decoder_input_ids` which will only be passed through the text components.
                - With a *text_* or *speech_* prefix, they will be input for the `generate` method of the
                text model and speech model respectively. It has the priority over the keywords without a prefix.

                This means you can, for example, specify a generation strategy for one generation but not for the
                other.
    """
    # attribute kwargs to models
    kwargs_text = {}
    kwargs_speech = {}
    for key, value in kwargs.items():
        if key.startswith("text_"):
            key = key[len("text_") :]
            kwargs_text[key] = value
        elif key.startswith("speech_"):
            key = key[len("speech_") :]
            kwargs_speech[key] = value
        else:
            # If the key is already in a specific config, then it's been set with a
            # submodules specific value and we don't override
            if key not in kwargs_text:
                kwargs_text[key] = value
            if key not in kwargs_speech:
                kwargs_speech[key] = value
    return kwargs_text, kwargs_speech

mindnlp.transformers.models.seamless_m4t.modeling_seamless_m4t.shift_tokens_right(input_ids, pad_token_id, decoder_start_token_id)

Shift input ids one token to the right.

Source code in mindnlp\transformers\models\seamless_m4t\modeling_seamless_m4t.py

def shift_tokens_right(input_ids: mindspore.Tensor, pad_token_id: int, decoder_start_token_id: int):
    """
    Shift input ids one token to the right.
    """
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].copy()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    # replace possible -100 values in labels by `pad_token_id`
    shifted_input_ids = shifted_input_ids.masked_fill(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids

mindnlp.transformers.models.seamless_m4t.configuration_seamless_m4t

SeamlessM4T model configuration

mindnlp.transformers.models.seamless_m4t.configuration_seamless_m4t.SeamlessM4TConfig

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [~SeamlessM4TModel]. It is used to instantiate an SeamlessM4T model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the SeamlessM4T "facebook/hf-seamless-m4t-medium" architecture.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the SeamlessM4T model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [~SeamlessM4TModel], [~SeamlessM4TForTextToSpeech] or [~SeamlessM4TForTextToText]. TYPE: `int`, *optional*, defaults to 256102 DEFAULT: 256102
t2u_vocab_size	Unit vocabulary size of the SeamlessM4T model. Defines the number of different unit tokens that can be represented by the inputs_ids passed when calling the Text-To-Units sub-model of [~SeamlessM4TModel], [~SeamlessM4TForSpeechToSpeech] or [~SeamlessM4TForTextToSpeech]. TYPE: `int`, *optional*, defaults to 10082 DEFAULT: 10082
Parameters	args below are Parameters shared across sub-models TYPE: shared across sub-models
hidden_size	Dimensionality of the "intermediate" layers in the architecture. TYPE: `int`, *optional*, defaults to 1024 DEFAULT: 1024
initializer_range	The standard deviation of the truncated_normal_initializer for initializing all weight matrices. TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02
layer_norm_eps	The epsilon used by the layer normalization layers. TYPE: `float`, *optional*, defaults to 1e-05 DEFAULT: 1e-05
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
max_position_embeddings	The maximum sequence length that this model text encoder and decoder might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 1024 DEFAULT: 1024
is_encoder_decoder	Whether the model is used as an encoder/decoder or not. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
encoder_layerdrop	The LayerDrop probability for the encoders. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.05 DEFAULT: 0.05
decoder_layerdrop	The LayerDrop probability for the decoders. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.05 DEFAULT: 0.05
activation_function	The non-linear activation function (function or string) in the decoder and feed-forward layers. If string, "gelu", "relu", "selu", "swish" and "gelu_new" are supported. TYPE: `str` or `function`, *optional*, defaults to `"relu"` DEFAULT: 'relu'
dropout	The dropout probability for all fully connected layers in the embeddings, encoder, decoder, and pooler. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
attention_dropout	The dropout probability for all attention layers. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
activation_dropout	The dropout probability for all activation layers in the model. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
scale_embedding	Scale embeddings by diving by sqrt(d_model). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
Text	args below are text decoder specific parameters TYPE: encoder and text decoder specific parameters
encoder_layers	Number of hidden layers in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
encoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
encoder_attention_heads	Number of attention heads for each attention layer in the Transformer text encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
decoder_layers	Number of hidden layers in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
decoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
decoder_attention_heads	Number of attention heads for each attention layer in the Transformer text decoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
decoder_start_token_id	If an encoder-decoder model starts decoding with a different token than bos, the id of that token. Only applied in the text decoder. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
max_new_tokens	The maximum numbers of text tokens to generate, ignoring the number of tokens in the prompt. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
pad_token_id	The id of the padding text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
bos_token_id	The id of the beginning-of-stream text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
eos_token_id	The id of the end-of-stream text token. Only applied to the text-decoder model. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
Speech	args below are Speech encoder specific parameters TYPE: encoder specific parameters
speech_encoder_layers	Number of hidden layers in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 24 DEFAULT: 24
speech_encoder_attention_heads	Number of attention heads for each attention layer in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
speech_encoder_intermediate_size	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer speech encoder. TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
speech_encoder_hidden_act	The non-linear activation function (function or string) in the speech encoder. If string, "gelu", "relu", "selu", "swish" and "gelu_new" are supported. TYPE: `str` or `function`, *optional*, defaults to `"swish"` DEFAULT: 'swish'
speech_encoder_dropout	The dropout probability for all layers in the speech encoder. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
add_adapter	Add an adapter layer on top of the speech encoder. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
speech_encoder_layerdrop	The LayerDrop probability for the speech encoder. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
feature_projection_input_dim	Input dimension of the input feature projection of the speech encoder, i.e the dimension after processing input audios with [SeamlessM4TFeatureExtractor]. TYPE: `int`, *optional*, defaults to 160 DEFAULT: 160
num_conv_pos_embeddings	Number of convolutional positional embeddings. Defines the kernel size of 1D convolutional positional embeddings layer of the speech encoder. TYPE: `int`, *optional*, defaults to 128 DEFAULT: 128
num_conv_pos_embedding_groups	Number of groups of 1D convolutional positional embeddings layer of the speech encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
adaptor_kernel_size	Kernel size of the convolutional layers in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
adaptor_stride	Stride of the convolutional layers in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
adaptor_dropout	The dropout probability for all layers in the speech adapter. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
num_adapter_layers	Number of convolutional layers that should be used in the adapter network. Only relevant if add_adapter is True. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
position_embeddings_type	Can be specified to relative or rotary for relative or rotary position embeddings respectively. If left None no relative position embedding is applied. Only applied to the speech encoder. TYPE: `str`, *optional*, defaults to `"relative"` DEFAULT: 'relative'
rotary_embedding_base	If "rotary" position embeddings are used, defines the size of the embedding base. Only applied to the speech encoder. TYPE: `int`, *optional*, defaults to 10000 DEFAULT: 10000
max_source_positions	if "relative" position embeddings are used, defines the maximum source input positions. Only applied to the speech encoder. TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
conv_depthwise_kernel_size	Kernel size of convolutional depthwise 1D layer in Conformer blocks. Only applied to the speech encoder. TYPE: `int`, *optional*, defaults to 31 DEFAULT: 31
Text-To-Unit	args below are Text-To-Unit (t2u) model specific parameters TYPE: t2u) model specific parameters
t2u_bos_token_id	The id of the beginning-of-stream unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 0 DEFAULT: 0
t2u_pad_token_id	The id of the padding unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
t2u_eos_token_id	The id of the end-of-stream unit token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
t2u_decoder_start_token_id	If an encoder-decoder model starts decoding with a different token than bos, the id of that token. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
t2u_max_new_tokens	The maximum numbers of unit tokens to generate, ignoring the number of tokens in the prompt. Only applied to the text-to-unit seq2seq model. TYPE: `int`, *optional*, defaults to 1024 DEFAULT: 1024
t2u_encoder_layers	Number of hidden layers in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
t2u_encoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
t2u_encoder_attention_heads	Number of attention heads for each attention layer in the Transformer text-to-unit encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
t2u_decoder_layers	Number of hidden layers in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
t2u_decoder_ffn_dim	Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 8192 DEFAULT: 8192
t2u_decoder_attention_heads	Number of attention heads for each attention layer in the Transformer text-to-unit decoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
t2u_max_position_embeddings	The maximum sequence length that this model text-to-unit component might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 2048 DEFAULT: 2048
Hifi-Gan	args below are Hifi-Gan Vocoder specific parameters TYPE: Vocoder specific parameters
sampling_rate	The sampling rate at which the output audio will be generated, expressed in hertz (Hz). TYPE: `int`, *optional*, defaults to 16000 DEFAULT: 16000
upsample_initial_channel	The number of input channels into the hifi-gan upsampling network. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 512 DEFAULT: 512
upsample_rates	A tuple of integers defining the stride of each 1D convolutional layer in the vocoder upsampling network. The length of upsample_rates defines the number of convolutional layers and has to match the length of upsample_kernel_sizes. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[5, 4, 4, 2, 2]` DEFAULT: [5, 4, 4, 2, 2]
upsample_kernel_sizes	A tuple of integers defining the kernel size of each 1D convolutional layer in the vocoder upsampling network. The length of upsample_kernel_sizes defines the number of convolutional layers and has to match the length of upsample_rates. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[11, 8, 8, 4, 4]` DEFAULT: [11, 8, 8, 4, 4]
resblock_kernel_sizes	A tuple of integers defining the kernel sizes of the vocoder 1D convolutional layers in the multi-receptive field fusion (MRF) module. Applies to the vocoder only. TYPE: `Tuple[int]` or `List[int]`, *optional*, defaults to `[3, 7, 11]` DEFAULT: [3, 7, 11]
resblock_dilation_sizes	A nested tuple of integers defining the dilation rates of the vocoder dilated 1D convolutional layers in the multi-receptive field fusion (MRF) module. Applies to the vocoder only. TYPE: `Tuple[Tuple[int]]` or `List[List[int]]`, *optional*, defaults to `[[1, 3, 5], [1, 3, 5], [1, 3, 5]]` DEFAULT: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
leaky_relu_slope	The angle of the negative slope used by the leaky ReLU activation in the vocoder. Applies to the vocoder only. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
unit_hifi_gan_vocab_size	Vocabulary size of the SeamlessM4T vocoder. Defines the number of different unit tokens that can be represented by the inputs_ids passed when calling the vocoder of [~SeamlessM4TModel], [~SeamlessM4TForSpeechToSpeech] or [~SeamlessM4TForTextToSpeech]. TYPE: `int`, *optional*, defaults to 10000 DEFAULT: 10000
unit_embed_dim	The projection dimension of the input ids given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 1280 DEFAULT: 1280
lang_embed_dim	The projection dimension of the target language given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
spkr_embed_dim	The projection dimension of the speaker id given to the hifi-gan vocoder. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
vocoder_num_langs	Number of langs supported by the vocoder. Might be different from t2u_num_langs. TYPE: `int`, *optional*, defaults to 36 DEFAULT: 36
vocoder_num_spkrs	Number of speakers supported by the vocoder. TYPE: `int`, *optional*, defaults to 200 DEFAULT: 200
variance_predictor_kernel_size	Kernel size of the duration predictor. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3
var_pred_dropout	The dropout probabilitiy of the duration predictor. Applies to the vocoder only. TYPE: `float`, *optional*, defaults to 0.5 DEFAULT: 0.5
vocoder_offset	Offset the unit token ids by this number to account for symbol tokens. Applies to the vocoder only. TYPE: `int`, *optional*, defaults to 4 DEFAULT: 4

Example

>>> from transformers import SeamlessM4TModel, SeamlessM4TConfig
...
>>> # Initializing a SeamlessM4T "facebook/hf-seamless-m4t-medium" style configuration
>>> configuration = SeamlessM4TConfig()
...
>>> # Initializing a model from the "facebook/hf-seamless-m4t-medium" style configuration
>>> model = SeamlessM4TModel(configuration)
...
>>> # Accessing the model configuration
>>> configuration = model.config

Source code in mindnlp\transformers\models\seamless_m4t\configuration_seamless_m4t.py

class SeamlessM4TConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`~SeamlessM4TModel`]. It is used to instantiate an
    SeamlessM4T model according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the SeamlessM4T
    ["facebook/hf-seamless-m4t-medium"](https://hf-mirror.com/"facebook/hf-seamless-m4t-medium") architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 256102):
            Vocabulary size of the SeamlessM4T model. Defines the number of different tokens that can be represented by
            the `inputs_ids` passed when calling [`~SeamlessM4TModel`], [`~SeamlessM4TForTextToSpeech`] or
            [`~SeamlessM4TForTextToText`].
        t2u_vocab_size (`int`, *optional*, defaults to 10082):
            Unit vocabulary size of the SeamlessM4T model. Defines the number of different unit tokens that can be
            represented by the `inputs_ids` passed when calling the Text-To-Units sub-model of [`~SeamlessM4TModel`],
            [`~SeamlessM4TForSpeechToSpeech`] or [`~SeamlessM4TForTextToSpeech`].
        Parameters shared across sub-models: args below are Parameters shared across sub-models
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the "intermediate" layers in the architecture.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
        max_position_embeddings (`int`, *optional*, defaults to 1024):
            The maximum sequence length that this model text encoder and decoder might ever be used with. Typically set
            this to something large just in case (e.g., 512 or 1024 or 2048).
        is_encoder_decoder (`bool`, *optional*, defaults to `True`):
            Whether the model is used as an encoder/decoder or not.
        encoder_layerdrop (`float`, *optional*, defaults to 0.05):
            The LayerDrop probability for the encoders. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        decoder_layerdrop (`float`, *optional*, defaults to 0.05):
            The LayerDrop probability for the decoders. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        activation_function (`str` or `function`, *optional*, defaults to `"relu"`):
            The non-linear activation function (function or string) in the decoder and feed-forward layers. If string,
            `"gelu"`, `"relu"`, `"selu"`, `"swish"` and `"gelu_new"` are supported.
        dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, decoder, and pooler.
        attention_dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all attention layers.
        activation_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for all activation layers in the model.
        scale_embedding (`bool`, *optional*, defaults to `True`):
            Scale embeddings by diving by sqrt(d_model).
        Text encoder and text decoder specific parameters:  args below are text decoder specific parameters
        encoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer text encoder.
        encoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text encoder.
        encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text encoder.
        decoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer text decoder.
        decoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text decoder.
        decoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text decoder.
        decoder_start_token_id (`int`, *optional*, defaults to 3):
            If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token. Only
            applied in the text decoder.
        max_new_tokens (`int`, *optional*, defaults to 256):
            The maximum numbers of text tokens to generate, ignoring the number of tokens in the prompt.
        pad_token_id (`int`, *optional*, defaults to 0):
            The id of the _padding_ text token. Only applied to the text-decoder model.
        bos_token_id (`int`, *optional*, defaults to 2):
            The id of the _beginning-of-stream_ text token. Only applied to the text-decoder model.
        eos_token_id (`int`, *optional*, defaults to 3):
            The id of the _end-of-stream_ text token. Only applied to the text-decoder model.
        Speech encoder specific parameters: args below are Speech encoder specific parameters
        speech_encoder_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer speech encoder.
        speech_encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer speech encoder.
        speech_encoder_intermediate_size (`int`, *optional*, defaults to 4096):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer speech encoder.
        speech_encoder_hidden_act (`str` or `function`, *optional*, defaults to `"swish"`):
            The non-linear activation function (function or string) in the speech encoder. If string, `"gelu"`,
            `"relu"`, `"selu"`, `"swish"` and `"gelu_new"` are supported.
        speech_encoder_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for all layers in the speech encoder.
        add_adapter (`bool`, *optional*, defaults to `True`):
            Add an adapter layer on top of the speech encoder.
        speech_encoder_layerdrop (`float`, *optional*, defaults to 0.1):
            The LayerDrop probability for the speech encoder. See the [LayerDrop paper](see
            https://arxiv.org/abs/1909.11556) for more details.
        feature_projection_input_dim (`int`, *optional*, defaults to 160):
            Input dimension of the input feature projection of the speech encoder, i.e the dimension after processing
            input audios with [`SeamlessM4TFeatureExtractor`].
        num_conv_pos_embeddings (`int`, *optional*, defaults to 128):
            Number of convolutional positional embeddings. Defines the kernel size of 1D convolutional positional
            embeddings layer of the speech encoder.
        num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
            Number of groups of 1D convolutional positional embeddings layer of the speech encoder.
        adaptor_kernel_size (`int`, *optional*, defaults to 8):
            Kernel size of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
        adaptor_stride (`int`, *optional*, defaults to 8):
            Stride of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
        adaptor_dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all layers in the speech adapter.
        num_adapter_layers (`int`, *optional*, defaults to 1):
            Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is
            True`.
        position_embeddings_type (`str`, *optional*, defaults to `"relative"`):
            Can be specified to `relative` or `rotary` for relative or rotary position embeddings respectively. If left
            `None` no relative position embedding is applied. Only applied to the speech encoder.
        rotary_embedding_base (`int`, *optional*, defaults to 10000):
            If `"rotary"` position embeddings are used, defines the size of the embedding base. Only applied to the
            speech encoder.
        max_source_positions (`int`, *optional*, defaults to 4096):
            if `"relative"` position embeddings are used, defines the maximum source input positions. Only applied to
            the speech encoder.
        conv_depthwise_kernel_size (`int`, *optional*, defaults to 31):
            Kernel size of convolutional depthwise 1D layer in Conformer blocks. Only applied to the speech encoder.
        Text-To-Unit (t2u) model specific parameters: args below are Text-To-Unit (t2u) model specific parameters
        t2u_bos_token_id (`int`, *optional*, defaults to 0):
            The id of the _beginning-of-stream_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_pad_token_id (`int`, *optional*, defaults to 1):
            The id of the _padding_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_eos_token_id (`int`, *optional*, defaults to 2):
            The id of the _end-of-stream_ unit token. Only applied to the text-to-unit seq2seq model.
        t2u_decoder_start_token_id (`int`, *optional*, defaults to 2):
            If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token. Only
            applied to the text-to-unit seq2seq model.
        t2u_max_new_tokens (`int`, *optional*, defaults to 1024):
            The maximum numbers of unit tokens to generate, ignoring the number of tokens in the prompt. Only applied
            to the text-to-unit seq2seq model.
        t2u_encoder_layers (`int`, *optional*, defaults to 6):
            Number of hidden layers in the Transformer text-to-unit encoder.
        t2u_encoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit encoder.
        t2u_encoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text-to-unit encoder.
        t2u_decoder_layers (`int`, *optional*, defaults to 6):
            Number of hidden layers in the Transformer text-to-unit decoder.
        t2u_decoder_ffn_dim (`int`, *optional*, defaults to 8192):
            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer text-to-unit decoder.
        t2u_decoder_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer text-to-unit decoder.
        t2u_max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model text-to-unit component might ever be used with. Typically set
            this to something large just in case (e.g., 512 or 1024 or 2048).
        Hifi-Gan Vocoder specific parameters:  args below are Hifi-Gan Vocoder specific parameters
        sampling_rate (`int`, *optional*, defaults to 16000):
            The sampling rate at which the output audio will be generated, expressed in hertz (Hz).
        upsample_initial_channel (`int`, *optional*, defaults to 512):
            The number of input channels into the hifi-gan upsampling network. Applies to the vocoder only.
        upsample_rates (`Tuple[int]` or `List[int]`, *optional*, defaults to `[5, 4, 4, 2, 2]`):
            A tuple of integers defining the stride of each 1D convolutional layer in the vocoder upsampling network.
            The length of *upsample_rates* defines the number of convolutional layers and has to match the length of
            *upsample_kernel_sizes*. Applies to the vocoder only.
        upsample_kernel_sizes (`Tuple[int]` or `List[int]`, *optional*, defaults to `[11, 8, 8, 4, 4]`):
            A tuple of integers defining the kernel size of each 1D convolutional layer in the vocoder upsampling
            network. The length of *upsample_kernel_sizes* defines the number of convolutional layers and has to match
            the length of *upsample_rates*. Applies to the vocoder only.
        resblock_kernel_sizes (`Tuple[int]` or `List[int]`, *optional*, defaults to `[3, 7, 11]`):
            A tuple of integers defining the kernel sizes of the vocoder 1D convolutional layers in the multi-receptive
            field fusion (MRF) module. Applies to the vocoder only.
        resblock_dilation_sizes (`Tuple[Tuple[int]]` or `List[List[int]]`, *optional*, defaults to `[[1, 3, 5], [1, 3, 5], [1, 3, 5]]`):
            A nested tuple of integers defining the dilation rates of the vocoder dilated 1D convolutional layers in
            the multi-receptive field fusion (MRF) module. Applies to the vocoder only.
        leaky_relu_slope (`float`, *optional*, defaults to 0.1):
            The angle of the negative slope used by the leaky ReLU activation in the vocoder. Applies to the vocoder
            only.
        unit_hifi_gan_vocab_size (`int`, *optional*, defaults to 10000):
            Vocabulary size of the SeamlessM4T vocoder. Defines the number of different unit tokens that can be
            represented by the `inputs_ids` passed when calling the vocoder of [`~SeamlessM4TModel`],
            [`~SeamlessM4TForSpeechToSpeech`] or [`~SeamlessM4TForTextToSpeech`].
        unit_embed_dim (`int`, *optional*, defaults to 1280):
            The projection dimension of the input ids given to the hifi-gan vocoder. Applies to the vocoder only.
        lang_embed_dim (`int`, *optional*, defaults to 256):
            The projection dimension of the target language given to the hifi-gan vocoder. Applies to the vocoder only.
        spkr_embed_dim (`int`, *optional*, defaults to 256):
            The projection dimension of the speaker id given to the hifi-gan vocoder. Applies to the vocoder only.
        vocoder_num_langs (`int`, *optional*, defaults to 36):
            Number of langs supported by the vocoder. Might be different from `t2u_num_langs`.
        vocoder_num_spkrs (`int`, *optional*, defaults to 200):
            Number of speakers supported by the vocoder.
        variance_predictor_kernel_size (`int`, *optional*, defaults to 3):
            Kernel size of the duration predictor. Applies to the vocoder only.
        var_pred_dropout (`float`, *optional*, defaults to 0.5):
            The dropout probabilitiy of the duration predictor. Applies to the vocoder only.
        vocoder_offset (`int`, *optional*, defaults to 4):
            Offset the unit token ids by this number to account for symbol tokens. Applies to the vocoder only.

    Example:
        ```python
        >>> from transformers import SeamlessM4TModel, SeamlessM4TConfig
        ...
        >>> # Initializing a SeamlessM4T "facebook/hf-seamless-m4t-medium" style configuration
        >>> configuration = SeamlessM4TConfig()
        ...
        >>> # Initializing a model from the "facebook/hf-seamless-m4t-medium" style configuration
        >>> model = SeamlessM4TModel(configuration)
        ...
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "seamless_m4t"

    def __init__(
        self,
        vocab_size=256102,
        t2u_vocab_size=10082,
        # shared config
        hidden_size=1024,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        use_cache=True,
        max_position_embeddings=1024,
        is_encoder_decoder=True,
        encoder_layerdrop=0.05,
        decoder_layerdrop=0.05,
        activation_function="relu",
        dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        scale_embedding=True,
        # text encoder|decoder
        encoder_layers=24,
        encoder_ffn_dim=8192,
        encoder_attention_heads=16,
        decoder_layers=24,
        decoder_ffn_dim=8192,
        decoder_attention_heads=16,
        decoder_start_token_id=3,
        max_new_tokens=256,
        pad_token_id=0,
        bos_token_id=2,
        eos_token_id=3,
        # speech_encoder
        speech_encoder_layers=24,
        speech_encoder_attention_heads=16,
        speech_encoder_intermediate_size=4096,
        speech_encoder_hidden_act="swish",
        speech_encoder_dropout=0.0,
        add_adapter=True,
        speech_encoder_layerdrop=0.1,
        feature_projection_input_dim=160,
        num_conv_pos_embeddings=128,
        num_conv_pos_embedding_groups=16,
        adaptor_kernel_size=8,
        adaptor_stride=8,
        adaptor_dropout=0.1,
        num_adapter_layers=1,
        position_embeddings_type="relative",
        rotary_embedding_base=10000,
        max_source_positions=4096,
        conv_depthwise_kernel_size=31,
        # t2u config
        t2u_bos_token_id=0,
        t2u_pad_token_id=1,
        t2u_eos_token_id=2,
        t2u_decoder_start_token_id=2,
        t2u_max_new_tokens=1024,
        t2u_encoder_layers=6,
        t2u_encoder_ffn_dim=8192,
        t2u_encoder_attention_heads=16,
        t2u_decoder_layers=6,
        t2u_decoder_ffn_dim=8192,
        t2u_decoder_attention_heads=16,
        t2u_max_position_embeddings=2048,
        # hifi-gan vocoder config
        sampling_rate=16000,
        upsample_initial_channel=512,
        upsample_rates=[5, 4, 4, 2, 2],
        upsample_kernel_sizes=[11, 8, 8, 4, 4],
        resblock_kernel_sizes=[3, 7, 11],
        resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
        leaky_relu_slope=0.1,
        # specific to Code Hifi-Gan
        unit_hifi_gan_vocab_size=10000,
        unit_embed_dim=1280,
        lang_embed_dim=256,
        spkr_embed_dim=256,
        vocoder_num_langs=36,
        vocoder_num_spkrs=200,
        variance_predictor_kernel_size=3,
        var_pred_dropout=0.5,
        vocoder_offset=4,
        **kwargs,
    ):
        """
        Initializes a SeamlessM4TConfig object with the specified configuration parameters.

        Args:
            self (object): The instance of the class.
            vocab_size (int): The size of the vocabulary.
            t2u_vocab_size (int): The size of the T2U vocabulary.
            hidden_size (int): The size of the hidden layers.
            initializer_range (float): The range for weight initialization.
            layer_norm_eps (float): The epsilon value for layer normalization.
            use_cache (bool): Flag to indicate whether to use cache.
            max_position_embeddings (int): The maximum position embeddings.
            is_encoder_decoder (bool): Flag to indicate if it's an encoder-decoder model.
            encoder_layerdrop (float): The layer drop rate for encoder layers.
            decoder_layerdrop (float): The layer drop rate for decoder layers.
            activation_function (str): The activation function to use.
            dropout (float): The dropout rate.
            attention_dropout (float): The dropout rate for attention layers.
            activation_dropout (float): The dropout rate for activation layers.
            scale_embedding (bool): Flag to indicate whether to scale embeddings.
            encoder_layers (int): The number of encoder layers.
            encoder_ffn_dim (int): The dimension of the encoder feed-forward network.
            encoder_attention_heads (int): The number of attention heads for encoder.
            decoder_layers (int): The number of decoder layers.
            decoder_ffn_dim (int): The dimension of the decoder feed-forward network.
            decoder_attention_heads (int): The number of attention heads for decoder.
            decoder_start_token_id (int): The start token ID for decoder.
            max_new_tokens (int): The maximum number of new tokens.
            pad_token_id (int): The ID of the padding token.
            bos_token_id (int): The ID of the beginning of sentence token.
            eos_token_id (int): The ID of the end of sentence token.
            speech_encoder_layers (int): The number of layers in the speech encoder.
            speech_encoder_attention_heads (int): The number of attention heads for speech encoder.
            speech_encoder_intermediate_size (int): The size of the intermediate layer in speech encoder.
            speech_encoder_hidden_act (str): The activation function for the hidden layers in speech encoder.
            speech_encoder_dropout (float): The dropout rate for the speech encoder.
            add_adapter (bool): Flag to indicate whether to add adapter layers.
            speech_encoder_layerdrop (float): The layer drop rate for speech encoder.
            feature_projection_input_dim (int): The input dimension for feature projection.
            num_conv_pos_embeddings (int): The number of convolutional positional embeddings.
            num_conv_pos_embedding_groups (int): The number of groups for convolutional positional embeddings.
            adaptor_kernel_size (int): The kernel size for the adaptor.
            adaptor_stride (int): The stride for the adaptor.
            adaptor_dropout (float): The dropout rate for the adaptor.
            num_adapter_layers (int): The number of adapter layers.
            position_embeddings_type (str): The type of position embeddings.
            rotary_embedding_base (int): The base value for rotary embeddings.
            max_source_positions (int): The maximum source positions.
            conv_depthwise_kernel_size (int): The kernel size for depthwise convolution.
            t2u_bos_token_id (int): The ID of the beginning of sentence token for T2U.
            t2u_pad_token_id (int): The ID of the padding token for T2U.
            t2u_eos_token_id (int): The ID of the end of sentence token for T2U.
            t2u_decoder_start_token_id (int): The start token ID for the T2U decoder.
            t2u_max_new_tokens (int): The maximum number of new tokens for T2U.
            t2u_encoder_layers (int): The number of layers in the T2U encoder.
            t2u_encoder_ffn_dim (int): The dimension of the T2U encoder feed-forward network.
            t2u_encoder_attention_heads (int): The number of attention heads for T2U encoder.
            t2u_decoder_layers (int): The number of layers in the T2U decoder.
            t2u_decoder_ffn_dim (int): The dimension of the T2U decoder feed-forward network.
            t2u_decoder_attention_heads (int): The number of attention heads for T2U decoder.
            t2u_max_position_embeddings (int): The maximum position embeddings for T2U.
            sampling_rate (int): The sampling rate for audio processing.
            upsample_initial_channel (int): The initial number of channels for upsampling.
            upsample_rates (list): The rates for upsampling.
            upsample_kernel_sizes (list): The kernel sizes for upsampling.
            resblock_kernel_sizes (list): The kernel sizes for the residual blocks.
            resblock_dilation_sizes (list): The dilation sizes for the residual blocks.
            leaky_relu_slope (float): The slope for leaky ReLU activation.
            unit_hifi_gan_vocab_size (int): The vocabulary size for the HiFi-GAN unit.
            unit_embed_dim (int): The embedding dimension for the HiFi-GAN unit.
            lang_embed_dim (int): The embedding dimension for language.
            spkr_embed_dim (int): The embedding dimension for speaker.
            vocoder_num_langs (int): The number of languages for the vocoder.
            vocoder_num_spkrs (int): The number of speakers for the vocoder.
            variance_predictor_kernel_size (int): The kernel size for the variance predictor.
            var_pred_dropout (float): The dropout rate for the variance predictor.
            vocoder_offset (int): The offset value for the vocoder.

        Returns:
            None.

        Raises:
            None.
        """
        # overall_config
        self.vocab_size = vocab_size
        self.t2u_vocab_size = t2u_vocab_size
        self.hidden_size = hidden_size
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.max_position_embeddings = max_position_embeddings
        self.use_cache = use_cache
        self.max_new_tokens = max_new_tokens
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.activation_function = activation_function
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.scale_embedding = scale_embedding
        # for proper config init
        self.num_attention_heads = decoder_attention_heads
        self.num_hidden_layers = decoder_layers

        # text|unit encoder|decoder
        self.encoder_layers = encoder_layers
        self.encoder_ffn_dim = encoder_ffn_dim
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_layers = decoder_layers
        self.decoder_ffn_dim = decoder_ffn_dim
        self.decoder_attention_heads = decoder_attention_heads

        # speech_encoder
        self.speech_encoder_layers = speech_encoder_layers
        self.speech_encoder_hidden_act = speech_encoder_hidden_act
        self.speech_encoder_dropout = speech_encoder_dropout
        self.speech_encoder_attention_heads = speech_encoder_attention_heads
        self.speech_encoder_layerdrop = speech_encoder_layerdrop
        self.speech_encoder_intermediate_size = speech_encoder_intermediate_size
        self.feature_projection_input_dim = feature_projection_input_dim
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.adaptor_kernel_size = adaptor_kernel_size
        self.adaptor_stride = adaptor_stride
        self.adaptor_dropout = adaptor_dropout
        self.num_adapter_layers = num_adapter_layers
        self.position_embeddings_type = position_embeddings_type
        self.rotary_embedding_base = rotary_embedding_base
        self.max_source_positions = max_source_positions
        self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
        self.add_adapter = add_adapter

        # t2u config
        self.t2u_bos_token_id = t2u_bos_token_id
        self.t2u_pad_token_id = t2u_pad_token_id
        self.t2u_eos_token_id = t2u_eos_token_id
        self.t2u_decoder_start_token_id = t2u_decoder_start_token_id
        self.t2u_max_new_tokens = t2u_max_new_tokens
        self.t2u_encoder_layers = t2u_encoder_layers
        self.t2u_encoder_ffn_dim = t2u_encoder_ffn_dim
        self.t2u_encoder_attention_heads = t2u_encoder_attention_heads
        self.t2u_decoder_layers = t2u_decoder_layers
        self.t2u_decoder_ffn_dim = t2u_decoder_ffn_dim
        self.t2u_decoder_attention_heads = t2u_decoder_attention_heads
        self.t2u_max_position_embeddings = t2u_max_position_embeddings

        # hifi-gan vocoder config
        # original parameters specific to Hifi-Gan
        self.sampling_rate = sampling_rate
        self.upsample_initial_channel = upsample_initial_channel
        self.upsample_rates = upsample_rates
        self.upsample_kernel_sizes = upsample_kernel_sizes
        self.resblock_kernel_sizes = resblock_kernel_sizes
        self.resblock_dilation_sizes = resblock_dilation_sizes
        self.leaky_relu_slope = leaky_relu_slope

        # specific to Code Hifi-Gan
        self.unit_hifi_gan_vocab_size = unit_hifi_gan_vocab_size
        self.unit_embed_dim = unit_embed_dim
        self.lang_embed_dim = lang_embed_dim
        self.spkr_embed_dim = spkr_embed_dim
        self.vocoder_num_langs = vocoder_num_langs
        self.vocoder_num_spkrs = vocoder_num_spkrs
        self.variance_predictor_kernel_size = variance_predictor_kernel_size
        self.var_pred_dropout = var_pred_dropout
        self.vocoder_offset = vocoder_offset

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            is_encoder_decoder=is_encoder_decoder,
            max_position_embeddings=max_position_embeddings,
            **kwargs,
        )

mindnlp.transformers.models.seamless_m4t.configuration_seamless_m4t.SeamlessM4TConfig.__init__(vocab_size=256102, t2u_vocab_size=10082, hidden_size=1024, initializer_range=0.02, layer_norm_eps=1e-05, use_cache=True, max_position_embeddings=1024, is_encoder_decoder=True, encoder_layerdrop=0.05, decoder_layerdrop=0.05, activation_function='relu', dropout=0.1, attention_dropout=0.1, activation_dropout=0.0, scale_embedding=True, encoder_layers=24, encoder_ffn_dim=8192, encoder_attention_heads=16, decoder_layers=24, decoder_ffn_dim=8192, decoder_attention_heads=16, decoder_start_token_id=3, max_new_tokens=256, pad_token_id=0, bos_token_id=2, eos_token_id=3, speech_encoder_layers=24, speech_encoder_attention_heads=16, speech_encoder_intermediate_size=4096, speech_encoder_hidden_act='swish', speech_encoder_dropout=0.0, add_adapter=True, speech_encoder_layerdrop=0.1, feature_projection_input_dim=160, num_conv_pos_embeddings=128, num_conv_pos_embedding_groups=16, adaptor_kernel_size=8, adaptor_stride=8, adaptor_dropout=0.1, num_adapter_layers=1, position_embeddings_type='relative', rotary_embedding_base=10000, max_source_positions=4096, conv_depthwise_kernel_size=31, t2u_bos_token_id=0, t2u_pad_token_id=1, t2u_eos_token_id=2, t2u_decoder_start_token_id=2, t2u_max_new_tokens=1024, t2u_encoder_layers=6, t2u_encoder_ffn_dim=8192, t2u_encoder_attention_heads=16, t2u_decoder_layers=6, t2u_decoder_ffn_dim=8192, t2u_decoder_attention_heads=16, t2u_max_position_embeddings=2048, sampling_rate=16000, upsample_initial_channel=512, upsample_rates=[5, 4, 4, 2, 2], upsample_kernel_sizes=[11, 8, 8, 4, 4], resblock_kernel_sizes=[3, 7, 11], resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]], leaky_relu_slope=0.1, unit_hifi_gan_vocab_size=10000, unit_embed_dim=1280, lang_embed_dim=256, spkr_embed_dim=256, vocoder_num_langs=36, vocoder_num_spkrs=200, variance_predictor_kernel_size=3, var_pred_dropout=0.5, vocoder_offset=4, **kwargs)

Initializes a SeamlessM4TConfig object with the specified configuration parameters.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
vocab_size	The size of the vocabulary. TYPE: int DEFAULT: 256102
t2u_vocab_size	The size of the T2U vocabulary. TYPE: int DEFAULT: 10082
hidden_size	The size of the hidden layers. TYPE: int DEFAULT: 1024
initializer_range	The range for weight initialization. TYPE: float DEFAULT: 0.02
layer_norm_eps	The epsilon value for layer normalization. TYPE: float DEFAULT: 1e-05
use_cache	Flag to indicate whether to use cache. TYPE: bool DEFAULT: True
max_position_embeddings	The maximum position embeddings. TYPE: int DEFAULT: 1024
is_encoder_decoder	Flag to indicate if it's an encoder-decoder model. TYPE: bool DEFAULT: True
encoder_layerdrop	The layer drop rate for encoder layers. TYPE: float DEFAULT: 0.05
decoder_layerdrop	The layer drop rate for decoder layers. TYPE: float DEFAULT: 0.05
activation_function	The activation function to use. TYPE: str DEFAULT: 'relu'
dropout	The dropout rate. TYPE: float DEFAULT: 0.1
attention_dropout	The dropout rate for attention layers. TYPE: float DEFAULT: 0.1
activation_dropout	The dropout rate for activation layers. TYPE: float DEFAULT: 0.0
scale_embedding	Flag to indicate whether to scale embeddings. TYPE: bool DEFAULT: True
encoder_layers	The number of encoder layers. TYPE: int DEFAULT: 24
encoder_ffn_dim	The dimension of the encoder feed-forward network. TYPE: int DEFAULT: 8192
encoder_attention_heads	The number of attention heads for encoder. TYPE: int DEFAULT: 16
decoder_layers	The number of decoder layers. TYPE: int DEFAULT: 24
decoder_ffn_dim	The dimension of the decoder feed-forward network. TYPE: int DEFAULT: 8192
decoder_attention_heads	The number of attention heads for decoder. TYPE: int DEFAULT: 16
decoder_start_token_id	The start token ID for decoder. TYPE: int DEFAULT: 3
max_new_tokens	The maximum number of new tokens. TYPE: int DEFAULT: 256
pad_token_id	The ID of the padding token. TYPE: int DEFAULT: 0
bos_token_id	The ID of the beginning of sentence token. TYPE: int DEFAULT: 2
eos_token_id	The ID of the end of sentence token. TYPE: int DEFAULT: 3
speech_encoder_layers	The number of layers in the speech encoder. TYPE: int DEFAULT: 24
speech_encoder_attention_heads	The number of attention heads for speech encoder. TYPE: int DEFAULT: 16
speech_encoder_intermediate_size	The size of the intermediate layer in speech encoder. TYPE: int DEFAULT: 4096
speech_encoder_hidden_act	The activation function for the hidden layers in speech encoder. TYPE: str DEFAULT: 'swish'
speech_encoder_dropout	The dropout rate for the speech encoder. TYPE: float DEFAULT: 0.0
add_adapter	Flag to indicate whether to add adapter layers. TYPE: bool DEFAULT: True
speech_encoder_layerdrop	The layer drop rate for speech encoder. TYPE: float DEFAULT: 0.1
feature_projection_input_dim	The input dimension for feature projection. TYPE: int DEFAULT: 160
num_conv_pos_embeddings	The number of convolutional positional embeddings. TYPE: int DEFAULT: 128
num_conv_pos_embedding_groups	The number of groups for convolutional positional embeddings. TYPE: int DEFAULT: 16
adaptor_kernel_size	The kernel size for the adaptor. TYPE: int DEFAULT: 8
adaptor_stride	The stride for the adaptor. TYPE: int DEFAULT: 8
adaptor_dropout	The dropout rate for the adaptor. TYPE: float DEFAULT: 0.1
num_adapter_layers	The number of adapter layers. TYPE: int DEFAULT: 1
position_embeddings_type	The type of position embeddings. TYPE: str DEFAULT: 'relative'
rotary_embedding_base	The base value for rotary embeddings. TYPE: int DEFAULT: 10000
max_source_positions	The maximum source positions. TYPE: int DEFAULT: 4096
conv_depthwise_kernel_size	The kernel size for depthwise convolution. TYPE: int DEFAULT: 31
t2u_bos_token_id	The ID of the beginning of sentence token for T2U. TYPE: int DEFAULT: 0
t2u_pad_token_id	The ID of the padding token for T2U. TYPE: int DEFAULT: 1
t2u_eos_token_id	The ID of the end of sentence token for T2U. TYPE: int DEFAULT: 2
t2u_decoder_start_token_id	The start token ID for the T2U decoder. TYPE: int DEFAULT: 2
t2u_max_new_tokens	The maximum number of new tokens for T2U. TYPE: int DEFAULT: 1024
t2u_encoder_layers	The number of layers in the T2U encoder. TYPE: int DEFAULT: 6
t2u_encoder_ffn_dim	The dimension of the T2U encoder feed-forward network. TYPE: int DEFAULT: 8192
t2u_encoder_attention_heads	The number of attention heads for T2U encoder. TYPE: int DEFAULT: 16
t2u_decoder_layers	The number of layers in the T2U decoder. TYPE: int DEFAULT: 6
t2u_decoder_ffn_dim	The dimension of the T2U decoder feed-forward network. TYPE: int DEFAULT: 8192
t2u_decoder_attention_heads	The number of attention heads for T2U decoder. TYPE: int DEFAULT: 16
t2u_max_position_embeddings	The maximum position embeddings for T2U. TYPE: int DEFAULT: 2048
sampling_rate	The sampling rate for audio processing. TYPE: int DEFAULT: 16000
upsample_initial_channel	The initial number of channels for upsampling. TYPE: int DEFAULT: 512
upsample_rates	The rates for upsampling. TYPE: list DEFAULT: [5, 4, 4, 2, 2]
upsample_kernel_sizes	The kernel sizes for upsampling. TYPE: list DEFAULT: [11, 8, 8, 4, 4]
resblock_kernel_sizes	The kernel sizes for the residual blocks. TYPE: list DEFAULT: [3, 7, 11]
resblock_dilation_sizes	The dilation sizes for the residual blocks. TYPE: list DEFAULT: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
leaky_relu_slope	The slope for leaky ReLU activation. TYPE: float DEFAULT: 0.1
unit_hifi_gan_vocab_size	The vocabulary size for the HiFi-GAN unit. TYPE: int DEFAULT: 10000
unit_embed_dim	The embedding dimension for the HiFi-GAN unit. TYPE: int DEFAULT: 1280
lang_embed_dim	The embedding dimension for language. TYPE: int DEFAULT: 256
spkr_embed_dim	The embedding dimension for speaker. TYPE: int DEFAULT: 256
vocoder_num_langs	The number of languages for the vocoder. TYPE: int DEFAULT: 36
vocoder_num_spkrs	The number of speakers for the vocoder. TYPE: int DEFAULT: 200
variance_predictor_kernel_size	The kernel size for the variance predictor. TYPE: int DEFAULT: 3
var_pred_dropout	The dropout rate for the variance predictor. TYPE: float DEFAULT: 0.5
vocoder_offset	The offset value for the vocoder. TYPE: int DEFAULT: 4

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\configuration_seamless_m4t.py

def __init__(
    self,
    vocab_size=256102,
    t2u_vocab_size=10082,
    # shared config
    hidden_size=1024,
    initializer_range=0.02,
    layer_norm_eps=1e-5,
    use_cache=True,
    max_position_embeddings=1024,
    is_encoder_decoder=True,
    encoder_layerdrop=0.05,
    decoder_layerdrop=0.05,
    activation_function="relu",
    dropout=0.1,
    attention_dropout=0.1,
    activation_dropout=0.0,
    scale_embedding=True,
    # text encoder|decoder
    encoder_layers=24,
    encoder_ffn_dim=8192,
    encoder_attention_heads=16,
    decoder_layers=24,
    decoder_ffn_dim=8192,
    decoder_attention_heads=16,
    decoder_start_token_id=3,
    max_new_tokens=256,
    pad_token_id=0,
    bos_token_id=2,
    eos_token_id=3,
    # speech_encoder
    speech_encoder_layers=24,
    speech_encoder_attention_heads=16,
    speech_encoder_intermediate_size=4096,
    speech_encoder_hidden_act="swish",
    speech_encoder_dropout=0.0,
    add_adapter=True,
    speech_encoder_layerdrop=0.1,
    feature_projection_input_dim=160,
    num_conv_pos_embeddings=128,
    num_conv_pos_embedding_groups=16,
    adaptor_kernel_size=8,
    adaptor_stride=8,
    adaptor_dropout=0.1,
    num_adapter_layers=1,
    position_embeddings_type="relative",
    rotary_embedding_base=10000,
    max_source_positions=4096,
    conv_depthwise_kernel_size=31,
    # t2u config
    t2u_bos_token_id=0,
    t2u_pad_token_id=1,
    t2u_eos_token_id=2,
    t2u_decoder_start_token_id=2,
    t2u_max_new_tokens=1024,
    t2u_encoder_layers=6,
    t2u_encoder_ffn_dim=8192,
    t2u_encoder_attention_heads=16,
    t2u_decoder_layers=6,
    t2u_decoder_ffn_dim=8192,
    t2u_decoder_attention_heads=16,
    t2u_max_position_embeddings=2048,
    # hifi-gan vocoder config
    sampling_rate=16000,
    upsample_initial_channel=512,
    upsample_rates=[5, 4, 4, 2, 2],
    upsample_kernel_sizes=[11, 8, 8, 4, 4],
    resblock_kernel_sizes=[3, 7, 11],
    resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
    leaky_relu_slope=0.1,
    # specific to Code Hifi-Gan
    unit_hifi_gan_vocab_size=10000,
    unit_embed_dim=1280,
    lang_embed_dim=256,
    spkr_embed_dim=256,
    vocoder_num_langs=36,
    vocoder_num_spkrs=200,
    variance_predictor_kernel_size=3,
    var_pred_dropout=0.5,
    vocoder_offset=4,
    **kwargs,
):
    """
    Initializes a SeamlessM4TConfig object with the specified configuration parameters.

    Args:
        self (object): The instance of the class.
        vocab_size (int): The size of the vocabulary.
        t2u_vocab_size (int): The size of the T2U vocabulary.
        hidden_size (int): The size of the hidden layers.
        initializer_range (float): The range for weight initialization.
        layer_norm_eps (float): The epsilon value for layer normalization.
        use_cache (bool): Flag to indicate whether to use cache.
        max_position_embeddings (int): The maximum position embeddings.
        is_encoder_decoder (bool): Flag to indicate if it's an encoder-decoder model.
        encoder_layerdrop (float): The layer drop rate for encoder layers.
        decoder_layerdrop (float): The layer drop rate for decoder layers.
        activation_function (str): The activation function to use.
        dropout (float): The dropout rate.
        attention_dropout (float): The dropout rate for attention layers.
        activation_dropout (float): The dropout rate for activation layers.
        scale_embedding (bool): Flag to indicate whether to scale embeddings.
        encoder_layers (int): The number of encoder layers.
        encoder_ffn_dim (int): The dimension of the encoder feed-forward network.
        encoder_attention_heads (int): The number of attention heads for encoder.
        decoder_layers (int): The number of decoder layers.
        decoder_ffn_dim (int): The dimension of the decoder feed-forward network.
        decoder_attention_heads (int): The number of attention heads for decoder.
        decoder_start_token_id (int): The start token ID for decoder.
        max_new_tokens (int): The maximum number of new tokens.
        pad_token_id (int): The ID of the padding token.
        bos_token_id (int): The ID of the beginning of sentence token.
        eos_token_id (int): The ID of the end of sentence token.
        speech_encoder_layers (int): The number of layers in the speech encoder.
        speech_encoder_attention_heads (int): The number of attention heads for speech encoder.
        speech_encoder_intermediate_size (int): The size of the intermediate layer in speech encoder.
        speech_encoder_hidden_act (str): The activation function for the hidden layers in speech encoder.
        speech_encoder_dropout (float): The dropout rate for the speech encoder.
        add_adapter (bool): Flag to indicate whether to add adapter layers.
        speech_encoder_layerdrop (float): The layer drop rate for speech encoder.
        feature_projection_input_dim (int): The input dimension for feature projection.
        num_conv_pos_embeddings (int): The number of convolutional positional embeddings.
        num_conv_pos_embedding_groups (int): The number of groups for convolutional positional embeddings.
        adaptor_kernel_size (int): The kernel size for the adaptor.
        adaptor_stride (int): The stride for the adaptor.
        adaptor_dropout (float): The dropout rate for the adaptor.
        num_adapter_layers (int): The number of adapter layers.
        position_embeddings_type (str): The type of position embeddings.
        rotary_embedding_base (int): The base value for rotary embeddings.
        max_source_positions (int): The maximum source positions.
        conv_depthwise_kernel_size (int): The kernel size for depthwise convolution.
        t2u_bos_token_id (int): The ID of the beginning of sentence token for T2U.
        t2u_pad_token_id (int): The ID of the padding token for T2U.
        t2u_eos_token_id (int): The ID of the end of sentence token for T2U.
        t2u_decoder_start_token_id (int): The start token ID for the T2U decoder.
        t2u_max_new_tokens (int): The maximum number of new tokens for T2U.
        t2u_encoder_layers (int): The number of layers in the T2U encoder.
        t2u_encoder_ffn_dim (int): The dimension of the T2U encoder feed-forward network.
        t2u_encoder_attention_heads (int): The number of attention heads for T2U encoder.
        t2u_decoder_layers (int): The number of layers in the T2U decoder.
        t2u_decoder_ffn_dim (int): The dimension of the T2U decoder feed-forward network.
        t2u_decoder_attention_heads (int): The number of attention heads for T2U decoder.
        t2u_max_position_embeddings (int): The maximum position embeddings for T2U.
        sampling_rate (int): The sampling rate for audio processing.
        upsample_initial_channel (int): The initial number of channels for upsampling.
        upsample_rates (list): The rates for upsampling.
        upsample_kernel_sizes (list): The kernel sizes for upsampling.
        resblock_kernel_sizes (list): The kernel sizes for the residual blocks.
        resblock_dilation_sizes (list): The dilation sizes for the residual blocks.
        leaky_relu_slope (float): The slope for leaky ReLU activation.
        unit_hifi_gan_vocab_size (int): The vocabulary size for the HiFi-GAN unit.
        unit_embed_dim (int): The embedding dimension for the HiFi-GAN unit.
        lang_embed_dim (int): The embedding dimension for language.
        spkr_embed_dim (int): The embedding dimension for speaker.
        vocoder_num_langs (int): The number of languages for the vocoder.
        vocoder_num_spkrs (int): The number of speakers for the vocoder.
        variance_predictor_kernel_size (int): The kernel size for the variance predictor.
        var_pred_dropout (float): The dropout rate for the variance predictor.
        vocoder_offset (int): The offset value for the vocoder.

    Returns:
        None.

    Raises:
        None.
    """
    # overall_config
    self.vocab_size = vocab_size
    self.t2u_vocab_size = t2u_vocab_size
    self.hidden_size = hidden_size
    self.initializer_range = initializer_range
    self.layer_norm_eps = layer_norm_eps
    self.max_position_embeddings = max_position_embeddings
    self.use_cache = use_cache
    self.max_new_tokens = max_new_tokens
    self.encoder_layerdrop = encoder_layerdrop
    self.decoder_layerdrop = decoder_layerdrop
    self.activation_function = activation_function
    self.dropout = dropout
    self.attention_dropout = attention_dropout
    self.activation_dropout = activation_dropout
    self.scale_embedding = scale_embedding
    # for proper config init
    self.num_attention_heads = decoder_attention_heads
    self.num_hidden_layers = decoder_layers

    # text|unit encoder|decoder
    self.encoder_layers = encoder_layers
    self.encoder_ffn_dim = encoder_ffn_dim
    self.encoder_attention_heads = encoder_attention_heads
    self.decoder_layers = decoder_layers
    self.decoder_ffn_dim = decoder_ffn_dim
    self.decoder_attention_heads = decoder_attention_heads

    # speech_encoder
    self.speech_encoder_layers = speech_encoder_layers
    self.speech_encoder_hidden_act = speech_encoder_hidden_act
    self.speech_encoder_dropout = speech_encoder_dropout
    self.speech_encoder_attention_heads = speech_encoder_attention_heads
    self.speech_encoder_layerdrop = speech_encoder_layerdrop
    self.speech_encoder_intermediate_size = speech_encoder_intermediate_size
    self.feature_projection_input_dim = feature_projection_input_dim
    self.num_conv_pos_embeddings = num_conv_pos_embeddings
    self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
    self.adaptor_kernel_size = adaptor_kernel_size
    self.adaptor_stride = adaptor_stride
    self.adaptor_dropout = adaptor_dropout
    self.num_adapter_layers = num_adapter_layers
    self.position_embeddings_type = position_embeddings_type
    self.rotary_embedding_base = rotary_embedding_base
    self.max_source_positions = max_source_positions
    self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
    self.add_adapter = add_adapter

    # t2u config
    self.t2u_bos_token_id = t2u_bos_token_id
    self.t2u_pad_token_id = t2u_pad_token_id
    self.t2u_eos_token_id = t2u_eos_token_id
    self.t2u_decoder_start_token_id = t2u_decoder_start_token_id
    self.t2u_max_new_tokens = t2u_max_new_tokens
    self.t2u_encoder_layers = t2u_encoder_layers
    self.t2u_encoder_ffn_dim = t2u_encoder_ffn_dim
    self.t2u_encoder_attention_heads = t2u_encoder_attention_heads
    self.t2u_decoder_layers = t2u_decoder_layers
    self.t2u_decoder_ffn_dim = t2u_decoder_ffn_dim
    self.t2u_decoder_attention_heads = t2u_decoder_attention_heads
    self.t2u_max_position_embeddings = t2u_max_position_embeddings

    # hifi-gan vocoder config
    # original parameters specific to Hifi-Gan
    self.sampling_rate = sampling_rate
    self.upsample_initial_channel = upsample_initial_channel
    self.upsample_rates = upsample_rates
    self.upsample_kernel_sizes = upsample_kernel_sizes
    self.resblock_kernel_sizes = resblock_kernel_sizes
    self.resblock_dilation_sizes = resblock_dilation_sizes
    self.leaky_relu_slope = leaky_relu_slope

    # specific to Code Hifi-Gan
    self.unit_hifi_gan_vocab_size = unit_hifi_gan_vocab_size
    self.unit_embed_dim = unit_embed_dim
    self.lang_embed_dim = lang_embed_dim
    self.spkr_embed_dim = spkr_embed_dim
    self.vocoder_num_langs = vocoder_num_langs
    self.vocoder_num_spkrs = vocoder_num_spkrs
    self.variance_predictor_kernel_size = variance_predictor_kernel_size
    self.var_pred_dropout = var_pred_dropout
    self.vocoder_offset = vocoder_offset

    super().__init__(
        pad_token_id=pad_token_id,
        bos_token_id=bos_token_id,
        eos_token_id=eos_token_id,
        decoder_start_token_id=decoder_start_token_id,
        is_encoder_decoder=is_encoder_decoder,
        max_position_embeddings=max_position_embeddings,
        **kwargs,
    )

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t

Tokenization classes for SeamlessM4T.

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer

Bases: PreTrainedTokenizer

Construct a SeamlessM4T tokenizer.

Adapted from [RobertaTokenizer] and [XLNetTokenizer]. Based on SentencePiece.

The tokenization method is <language code> <tokens> <eos> for source language documents, and <eos> <language code> <tokens> <eos> for target language documents.

Example

>>> from transformers import SeamlessM4TTokenizer
...
>>> tokenizer = SeamlessM4TTokenizer.from_pretrained(
...     "facebook/hf-seamless-m4t-medium", src_lang="eng", tgt_lang="fra"
... )
>>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
>>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
>>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="ms")

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the cls_token. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
tokenizer_file	The path to a tokenizer file to use instead of the vocab file. TYPE: `str`, *optional* DEFAULT: None
src_lang	The language to use as source language for translation. TYPE: `str`, *optional*, defaults to `"eng"` DEFAULT: 'eng'
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional*, defaults to `"fra"` DEFAULT: 'fra'
sp_model_kwargs	Additional keyword arguments to pass to the model initialization. TYPE: `Dict[str, Any]`, *optional* DEFAULT: None
additional_special_tokens	A tuple or a list of additional special tokens. Can be used to specify the list of languages that will be supported by the tokenizer. TYPE: tuple or list of `str` or `tokenizers.AddedToken`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

class SeamlessM4TTokenizer(PreTrainedTokenizer):
    """
    Construct a SeamlessM4T tokenizer.

    Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
    [SentencePiece](https://github.com/google/sentencepiece).

    The tokenization method is `<language code> <tokens> <eos>` for source language documents, and `<eos> <language
    code> <tokens> <eos>` for target language documents.

    Example:
        ```python
        >>> from transformers import SeamlessM4TTokenizer
        ...
        >>> tokenizer = SeamlessM4TTokenizer.from_pretrained(
        ...     "facebook/hf-seamless-m4t-medium", src_lang="eng", tgt_lang="fra"
        ... )
        >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
        >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
        >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="ms")
        ```

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        tokenizer_file (`str`, *optional*):
            The path to a tokenizer file to use instead of the vocab file.
        src_lang (`str`, *optional*, defaults to `"eng"`):
            The language to use as source language for translation.
        tgt_lang (`str`, *optional*, defaults to `"fra"`):
            The language to use as target language for translation.
        sp_model_kwargs (`Dict[str, Any]`, *optional*):
            Additional keyword arguments to pass to the model initialization.
        additional_special_tokens (tuple or list of `str` or `tokenizers.AddedToken`, *optional*):
            A tuple or a list of additional special tokens. Can be used to specify the list of languages that will be
            supported by the tokenizer.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    model_input_names = ["input_ids", "attention_mask"]

    prefix_tokens: List[int] = []
    suffix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        tokenizer_file=None,
        src_lang="eng",
        tgt_lang="fra",
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        additional_special_tokens=None,
        **kwargs,
    ):
        """
        Initializes an instance of the SeamlessM4TTokenizer class.

        Args:
            self: The instance of the class.
            vocab_file (str): The path to the vocabulary file.
            bos_token (str, optional): The token representing the beginning of a sequence. Defaults to '<s>'.
            eos_token (str, optional): The token representing the end of a sequence. Defaults to '</s>'.
            sep_token (str, optional): The token used to separate two sequences. Defaults to '</s>'.
            cls_token (str, optional): The token representing the classification of a sequence. Defaults to '<s>'.
            unk_token (str, optional): The token representing an unknown word. Defaults to '<unk>'.
            pad_token (str, optional): The token used for padding sequences. Defaults to '<pad>'.
            tokenizer_file (str, optional): The path to the tokenizer file. Defaults to None.
            src_lang (str, optional): The source language. Defaults to 'eng'.
            tgt_lang (str, optional): The target language. Defaults to 'fra'.
            sp_model_kwargs (Optional[Dict[str, Any]], optional): Additional arguments for the sentencepiece model.
                Defaults to None.
            additional_special_tokens (List[str], optional): Additional special tokens. Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
        # Add this unused argument to keep some important Copied from statements
        self.legacy = False
        self.vocab_file = vocab_file

        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

        # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
        # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
        # spm  | '<unk>'   | '<s>' | '</s>' | 'an' | 'en' | '_d' | 'er' | 'in' | '_s' | '_a'
        # fairseq  | '<pad>'   | '<unk>' | '<s>' | '</s>' | 'an' | 'en' | '▁d' | 'er' | 'in' | '▁s'

        # Mimic fairseq token-to-id alignment for the first 4 token
        self._added_tokens_decoder = {
            0: AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token,
            1: AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token,
            2: AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token,
            3: AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token,
        }

        # The first "real" token "an" has position 4 in the original fairseq vocab and position 3 in the spm vocab
        self.fairseq_offset = 1

        self.sp_model_size = len(self.sp_model)

        self._src_lang = f"__{src_lang}__" if "__" not in src_lang else src_lang
        self._tgt_lang = f"__{tgt_lang}__" if "__" not in tgt_lang else tgt_lang

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            cls_token=cls_token,
            pad_token=pad_token,
            tokenizer_file=tokenizer_file,
            src_lang=src_lang,
            tgt_lang=tgt_lang,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

        self.set_src_lang_special_tokens(self._src_lang)
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.__getstate__
    def __getstate__(self):
        """
        Return the state of the SeamlessM4TTokenizer object.

        Args:
            self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.

        Returns:
            dict: A dictionary containing the current state of the object,
                with the following keys:

                - '__dict__': A dictionary containing the object's instance variables.
                - 'sp_model': The value of the 'sp_model' instance variable set to None.
                - 'sp_model_proto': The serialized model proto of the 'sp_model' instance variable.

        Raises:
            None.
        """
        state = self.__dict__.copy()
        state["sp_model"] = None
        state["sp_model_proto"] = self.sp_model.serialized_model_proto()
        return state

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.__setstate__
    def __setstate__(self, d):
        """
        Method to set the state of the SeamlessM4TTokenizer instance.

        Args:
            self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.
            d (dict): A dictionary containing the state information to be set on the instance.

        Returns:
            None.

        Raises:
           None.
        """
        self.__dict__ = d

        # for backward compatibility
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

    @property
    def vocab_size(self):
        """
        This method returns the size of the vocabulary used by the SeamlessM4TTokenizer.

        Args:
            self: An instance of the SeamlessM4TTokenizer class.

        Returns:
            int: The size of the vocabulary used by the SeamlessM4TTokenizer.

        Raises:
            None.
        """
        return len(self.sp_model)

    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
        text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair_target: Optional[
            Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
        ] = None,
        padding: Union[bool, str, PaddingStrategy] = True,
        pad_to_multiple_of: Optional[int] = 2,
        src_lang: Optional[str] = None,
        tgt_lang: Optional[str] = None,
        **kwargs,
    ):
        """
        Args:
            text (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_pair (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
                list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
                you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_pair_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
                list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
                you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
                 Select a strategy to pad the returned sequences (according to the model's padding side and padding
                 index) among:

                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
                sequence if provided).
                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
                acceptable input length for the model if that argument is not provided.
                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
                lengths).
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the sequence to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta).
            src_lang (`str`, *optional*):
                A string representing the source language. If not specified, the last `src_lang` specified (either
                during initialization or when calling this tokenizer) will be used.
            tgt_lang (`str`, *optional*):
                A string representing the target language. If not specified, the last `tgt_lang` specified (either
                during initialization or when calling this tokenizer) will be used.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`PreTrainedTokenizer.__call__`].
        """
        if src_lang is not None:
            self.src_lang = src_lang
        if tgt_lang is not None:
            self.tgt_lang = tgt_lang

        output = super().__call__(
            text=text,
            text_pair=text_pair,
            text_target=text_target,
            text_pair_target=text_pair_target,
            padding=padding,
            pad_to_multiple_of=pad_to_multiple_of,
            **kwargs,
        )

        return BatchEncoding(output, tensor_type=kwargs.get("return_tensors"))

    @property
    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.src_lang
    def src_lang(self) -> str:
        """
        Returns the source language of the SeamlessM4TTokenizer instance.

        Args:
            self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.

        Returns:
            str: The source language of the tokenized text.

        Raises:
            None.

        This property method returns the source language of the tokenized text. The source language refers to the
        language in which the original text was written.

        Note:
            The source language is stored internally as a private attribute '_src_lang'. This method retrieves the value
            of '_src_lang' and returns it as a string.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizer()
            >>> tokenizer.src_lang
            'en'
            ```
        In the example above, the 'src_lang' property method is called on an instance of the SeamlessM4TTokenizer class,
        returning the source language 'en'.
        """
        return self._src_lang

    @src_lang.setter
    def src_lang(self, new_src_lang: str) -> None:
        """
        Sets the source language for the SeamlessM4TTokenizer instance.

        Args:
            self (SeamlessM4TTokenizer): The current instance of the SeamlessM4TTokenizer class.
            new_src_lang (str): The new source language to be set. It should be a string representing the language code.

        Returns:
            None: This method updates the source language attribute of the instance.

        Raises:
            None.
        """
        if "__" not in new_src_lang:
            self._src_lang = f"__{new_src_lang}__"
        else:
            self._src_lang = new_src_lang
        self.set_src_lang_special_tokens(self._src_lang)

    @property
    def tgt_lang(self) -> str:
        """
        Returns the target language of the SeamlessM4TTokenizer instance.

        Args:
            self: An instance of the SeamlessM4TTokenizer class.

        Returns:
            str: The target language of the tokenizer.
                It represents the language into which the input text will be translated.

        Raises:
            None.

        """
        return self._tgt_lang

    @tgt_lang.setter
    def tgt_lang(self, new_tgt_lang: str) -> None:
        """
        Set the target language for the SeamlessM4TTokenizer.

        Args:
            self: The instance of the SeamlessM4TTokenizer class.
            new_tgt_lang (str): The new target language to set. It should be a string representing the target language code.
                If the target language does not contain '__' (double underscore), it will be prefixed and suffixed with '__'
                to indicate that it is a special token. Otherwise, the target language will be set as is.

        Returns:
            None.

        Raises:
            None.
        """
        if "__" not in new_tgt_lang:
            self._tgt_lang = f"__{new_tgt_lang}__"
        else:
            self._tgt_lang = new_tgt_lang
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.get_special_tokens_mask
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        prefix_ones = [1] * len(self.prefix_tokens)
        suffix_ones = [1] * len(self.suffix_tokens)
        if token_ids_1 is None:
            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An NLLB sequence has the following format, where `X` represents the sequence:

        - `input_ids` (for encoder) `X [eos, src_lang_code]`
        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`

        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0 + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.create_token_type_ids_from_sequences
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
        make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.

        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def _build_translation_inputs(
        self, raw_inputs, return_tensors: str, src_lang: Optional[str], tgt_lang: Optional[str], **extra_kwargs
    ):
        """Used by translation pipeline, to prepare inputs for the generate function"""
        if src_lang is None or tgt_lang is None:
            raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model.")
        self.src_lang = src_lang
        inputs = self(raw_inputs, add_special_tokens=True, return_tensors=return_tensors, **extra_kwargs)
        if "__" not in tgt_lang:
            tgt_lang = f"__{tgt_lang}__"
        tgt_lang_id = self.convert_tokens_to_ids(tgt_lang)
        inputs["forced_bos_token_id"] = tgt_lang_id
        return inputs

    def get_vocab(self):
        """
        Method: get_vocab

        Description:
        This method returns the vocabulary for the SeamlessM4TTokenizer instance.

        Args:
            self: The instance of the SeamlessM4TTokenizer class.

        Returns:
            vocab: A dictionary containing the vocabulary, where the keys are tokens and the values are their
                corresponding IDs.

        Raises:
            None.
        """
        vocab = {
            self.convert_ids_to_tokens(i): i for i in range(self.fairseq_offset, self.vocab_size + self.fairseq_offset)
        }
        vocab.update(self.added_tokens_encoder)
        return vocab

    @property
    def unk_token_length(self):
        """
        Returns the length of the unknown token.

        Args:
            self (SeamlessM4TTokenizer): An instance of the SeamlessM4TTokenizer class.

        Returns:
            int: The length of the unknown token.

        Raises:
            None.

        This method calculates and returns the length of the unknown token present in the SeamlessM4TTokenizer instance.
        The unknown token is obtained by encoding the string representation of the 'unk_token' attribute using the
        'sp_model' encoding method. The length of the resulting encoded token is then returned as an integer value.

        Note that this method takes no additional parameters besides the mandatory 'self' parameter, which represents
        the instance of the SeamlessM4TTokenizer class on which the method is called.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizer()
            >>> tokenizer.unk_token = "unknown"
            >>> tokenizer.unk_token_length()
            7
            ```
        """
        return len(self.sp_model.encode(str(self.unk_token)))

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor
    def get_spm_processor(self, from_slow=False):
        """
        Retrieves the SentencePieceProcessor tokenizer for the SeamlessM4TTokenizer class.

        Args:
            self (SeamlessM4TTokenizer): An instance of the SeamlessM4TTokenizer class.
            from_slow (bool, optional): A flag indicating whether to load the tokenizer from the slow version or not.
                Defaults to False.

        Returns:
            None

        Raises:
            None.
        """
        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        if self.legacy or from_slow:  # no dependency on protobuf
            tokenizer.Load(self.vocab_file)
            return tokenizer

        with open(self.vocab_file, "rb") as f:
            sp_model = f.read()
            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
            model = model_pb2.ModelProto.FromString(sp_model)
            normalizer_spec = model_pb2.NormalizerSpec()
            normalizer_spec.add_dummy_prefix = False
            model.normalizer_spec.MergeFrom(normalizer_spec)
            sp_model = model.SerializeToString()
            tokenizer.LoadFromSerializedProto(sp_model)
        return tokenizer

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize
    def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
        """
        Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
        first token is special.
        """
        if self.legacy or len(text) == 0:
            return super().tokenize(text, **kwargs)

        tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
            tokens = tokens[1:]
        return tokens

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._tokenize
    def _tokenize(self, text, **kwargs):
        """
        Returns a tokenized string.

        We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any
        SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give
        `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the
        `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.
        `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.
        """
        tokens = self.sp_model.encode(text, out_type=str)
        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):
            return tokens

        # 1. Encode string + prefix ex: "<unk> Hey"
        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
        # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']
        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        spm_id = self.sp_model.PieceToId(token)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.sp_model.IdToPiece(index - self.fairseq_offset)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (strings for sub-words) in a single string."""
        if tokens[0].startswith(SPIECE_UNDERLINE):
            tokens[0] = tokens[0][1:]

        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
        return out_string

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.save_vocabulary
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary of the SeamlessM4TTokenizer object to a specified directory.

        Args:
            self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.
            save_directory (str): The path of the directory where the vocabulary will be saved.
            filename_prefix (Optional[str]): An optional prefix to be added to the filename of the saved vocabulary.
                Default is None.

        Returns:
            Tuple[str]: A tuple containing the path of the saved vocabulary file.

        Raises:
            OSError: If the save_directory is not a valid directory.
            IOError: If the vocabulary file cannot be copied or saved.

        Note:
            - The save_directory should be an existing directory.
            - If the save_directory already contains a file with the same name as the vocabulary file,
            it will be overwritten.
            - If the self.vocab_file is not an existing file, the vocabulary will be saved directly to the
            specified directory.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizer()
            >>> save_directory = '/path/to/save_directory'
            >>> filename_prefix = 'my_vocab'
            >>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
            >>> print(saved_file)  # Output: ('/path/to/save_directory/my_vocab-vocab.txt',)
            ```
        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.prepare_seq2seq_batch with eng_Latn->eng, fra_Latn->fra
    def prepare_seq2seq_batch(
        self,
        src_texts: List[str],
        src_lang: str = "eng",
        tgt_texts: Optional[List[str]] = None,
        tgt_lang: str = "fra",
        **kwargs,
    ) -> BatchEncoding:
        """
        Prepare Seq2Seq Batch method in the SeamlessM4TTokenizer class.

        This method prepares a batch of inputs for sequence-to-sequence models.

        Args:
            self: The instance of the class.
            src_texts (List[str]): A list of source texts to be encoded.
            src_lang (str, optional): The language of the source texts. Defaults to 'eng'.
            tgt_texts (Optional[List[str]], optional): A list of target texts to be encoded. Defaults to None.
            tgt_lang (str, optional): The language of the target texts. Defaults to 'fra'.
            **kwargs: Additional keyword arguments.

        Returns:
            BatchEncoding: A BatchEncoding object containing the prepared batch of inputs.

        Raises:
            None.

        """
        self.src_lang = src_lang
        self.tgt_lang = tgt_lang
        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer._switch_to_input_mode
    def _switch_to_input_mode(self):
        """
        Switches the tokenizer to input mode.

        Args:
            self (SeamlessM4TTokenizer): An instance of the SeamlessM4TTokenizer class.
                This parameter is used to access the methods and properties of the class.

        Returns:
            None.

        Raises:
            None.

        Description:
            This method is used to switch the tokenizer to input mode. In input mode, the tokenizer processes the
            source language text and prepares it for translation. Switching to input mode involves setting the source
            language special tokens using the set_src_lang_special_tokens method of the SeamlessM4TTokenizer class.
            The source language is passed as a parameter to the set_src_lang_special_tokens method to configure the
            special tokens specific to the source language.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizer()
            >>> tokenizer._switch_to_input_mode()
            ```
        """
        return self.set_src_lang_special_tokens(self.src_lang)

    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer._switch_to_target_mode
    def _switch_to_target_mode(self):
        """
        Switches the tokenizer to the target mode for the SeamlessM4TTokenizer class.

        Args:
            self: An instance of the SeamlessM4TTokenizer class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.set_tgt_lang_special_tokens(self.tgt_lang)

    def set_src_lang_special_tokens(self, src_lang) -> None:
        """Reset the special tokens to the source lang setting.
        Prefix=[src_lang_code], suffix = [eos]
        """
        self.cur_lang_code = self.convert_tokens_to_ids(src_lang)
        self.init_kwargs["src_lang"] = src_lang

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`src_lang={src_lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.prefix_tokens = [self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]

    # https://github.com/facebookresearch/fairseq2/blob/c53f18e6be6b8b46b722f2249b8397b7eccd7ad3/src/fairseq2/models/nllb/tokenizer.py#L112-L116
    def set_tgt_lang_special_tokens(self, lang: str) -> None:
        """Reset the special tokens to the target lang setting.
        Prefix=[eos, tgt_lang_code] and suffix=[eos].
        """
        self.cur_lang_code = self.convert_tokens_to_ids(lang)
        self.init_kwargs["tgt_lang"] = lang

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`tgt_lang={lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.prefix_tokens = [self.eos_token_id, self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.src_lang: str property writable

Returns the source language of the SeamlessM4TTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer

RETURNS	DESCRIPTION
str	The source language of the tokenized text. TYPE: str

This property method returns the source language of the tokenized text. The source language refers to the language in which the original text was written.

Note

The source language is stored internally as a private attribute '_src_lang'. This method retrieves the value of '_src_lang' and returns it as a string.

Example

>>> tokenizer = SeamlessM4TTokenizer()
>>> tokenizer.src_lang
'en'

In the example above, the 'src_lang' property method is called on an instance of the SeamlessM4TTokenizer class, returning the source language 'en'.

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.tgt_lang: str property writable

Returns the target language of the SeamlessM4TTokenizer instance.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizer class.

RETURNS	DESCRIPTION
str	The target language of the tokenizer. It represents the language into which the input text will be translated. TYPE: str

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.unk_token_length property

Returns the length of the unknown token.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer

RETURNS	DESCRIPTION
int	The length of the unknown token.

This method calculates and returns the length of the unknown token present in the SeamlessM4TTokenizer instance. The unknown token is obtained by encoding the string representation of the 'unk_token' attribute using the 'sp_model' encoding method. The length of the resulting encoded token is then returned as an integer value.

Note that this method takes no additional parameters besides the mandatory 'self' parameter, which represents the instance of the SeamlessM4TTokenizer class on which the method is called.

Example

>>> tokenizer = SeamlessM4TTokenizer()
>>> tokenizer.unk_token = "unknown"
>>> tokenizer.unk_token_length()
7

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.vocab_size property

This method returns the size of the vocabulary used by the SeamlessM4TTokenizer.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizer class.

RETURNS	DESCRIPTION
int	The size of the vocabulary used by the SeamlessM4TTokenizer.

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.__call__(text=None, text_pair=None, text_target=None, text_pair_target=None, padding=True, pad_to_multiple_of=2, src_lang=None, tgt_lang=None, **kwargs)

PARAMETER	DESCRIPTION
text	The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_pair	The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_target	The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_pair_target	The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
padding	Select a strategy to pad the returned sequences (according to the model's padding side and padding index) among: True or 'longest': Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). 'max_length': Pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided. False or 'do_not_pad' (default): No padding (i.e., can output a batch with sequences of different lengths). TYPE: `bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True` DEFAULT: True
pad_to_multiple_of	If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta). TYPE: `int`, *optional* DEFAULT: 2
src_lang	A string representing the source language. If not specified, the last src_lang specified (either during initialization or when calling this tokenizer) will be used. TYPE: `str`, *optional* DEFAULT: None
tgt_lang	A string representing the target language. If not specified, the last tgt_lang specified (either during initialization or when calling this tokenizer) will be used. TYPE: `str`, *optional* DEFAULT: None
kwargs	Remaining dictionary of keyword arguments that will be passed to [PreTrainedTokenizer.__call__]. TYPE: *optional* DEFAULT: {}

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def __call__(
    self,
    text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
    text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
    text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
    text_pair_target: Optional[
        Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
    ] = None,
    padding: Union[bool, str, PaddingStrategy] = True,
    pad_to_multiple_of: Optional[int] = 2,
    src_lang: Optional[str] = None,
    tgt_lang: Optional[str] = None,
    **kwargs,
):
    """
    Args:
        text (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
            (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
            `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_pair (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
            (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
            `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
            list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
            you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_pair_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
            list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
            you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
             Select a strategy to pad the returned sequences (according to the model's padding side and padding
             index) among:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
            sequence if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
            acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
            lengths).
        pad_to_multiple_of (`int`, *optional*):
            If set will pad the sequence to a multiple of the provided value.

            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
            `>= 7.5` (Volta).
        src_lang (`str`, *optional*):
            A string representing the source language. If not specified, the last `src_lang` specified (either
            during initialization or when calling this tokenizer) will be used.
        tgt_lang (`str`, *optional*):
            A string representing the target language. If not specified, the last `tgt_lang` specified (either
            during initialization or when calling this tokenizer) will be used.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`PreTrainedTokenizer.__call__`].
    """
    if src_lang is not None:
        self.src_lang = src_lang
    if tgt_lang is not None:
        self.tgt_lang = tgt_lang

    output = super().__call__(
        text=text,
        text_pair=text_pair,
        text_target=text_target,
        text_pair_target=text_pair_target,
        padding=padding,
        pad_to_multiple_of=pad_to_multiple_of,
        **kwargs,
    )

    return BatchEncoding(output, tensor_type=kwargs.get("return_tensors"))

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.__getstate__()

Return the state of the SeamlessM4TTokenizer object.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the current state of the object, with the following keys: 'dict': A dictionary containing the object's instance variables. 'sp_model': The value of the 'sp_model' instance variable set to None. 'sp_model_proto': The serialized model proto of the 'sp_model' instance variable.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def __getstate__(self):
    """
    Return the state of the SeamlessM4TTokenizer object.

    Args:
        self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.

    Returns:
        dict: A dictionary containing the current state of the object,
            with the following keys:

            - '__dict__': A dictionary containing the object's instance variables.
            - 'sp_model': The value of the 'sp_model' instance variable set to None.
            - 'sp_model_proto': The serialized model proto of the 'sp_model' instance variable.

    Raises:
        None.
    """
    state = self.__dict__.copy()
    state["sp_model"] = None
    state["sp_model_proto"] = self.sp_model.serialized_model_proto()
    return state

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.__init__(vocab_file, bos_token='<s>', eos_token='</s>', sep_token='</s>', cls_token='<s>', unk_token='<unk>', pad_token='<pad>', tokenizer_file=None, src_lang='eng', tgt_lang='fra', sp_model_kwargs=None, additional_special_tokens=None, **kwargs)

Initializes an instance of the SeamlessM4TTokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The path to the vocabulary file. TYPE: str
bos_token	The token representing the beginning of a sequence. Defaults to ''. TYPE: str DEFAULT: '<s>'
eos_token	The token representing the end of a sequence. Defaults to ''. TYPE: str DEFAULT: '</s>'
sep_token	The token used to separate two sequences. Defaults to ''. TYPE: str DEFAULT: '</s>'
cls_token	The token representing the classification of a sequence. Defaults to ''. TYPE: str DEFAULT: '<s>'
unk_token	The token representing an unknown word. Defaults to ''. TYPE: str DEFAULT: '<unk>'
pad_token	The token used for padding sequences. Defaults to ''. TYPE: str DEFAULT: '<pad>'
tokenizer_file	The path to the tokenizer file. Defaults to None. TYPE: str DEFAULT: None
src_lang	The source language. Defaults to 'eng'. TYPE: str DEFAULT: 'eng'
tgt_lang	The target language. Defaults to 'fra'. TYPE: str DEFAULT: 'fra'
sp_model_kwargs	Additional arguments for the sentencepiece model. Defaults to None. TYPE: Optional[Dict[str, Any]] DEFAULT: None
additional_special_tokens	Additional special tokens. Defaults to None. TYPE: List[str] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def __init__(
    self,
    vocab_file,
    bos_token="<s>",
    eos_token="</s>",
    sep_token="</s>",
    cls_token="<s>",
    unk_token="<unk>",
    pad_token="<pad>",
    tokenizer_file=None,
    src_lang="eng",
    tgt_lang="fra",
    sp_model_kwargs: Optional[Dict[str, Any]] = None,
    additional_special_tokens=None,
    **kwargs,
):
    """
    Initializes an instance of the SeamlessM4TTokenizer class.

    Args:
        self: The instance of the class.
        vocab_file (str): The path to the vocabulary file.
        bos_token (str, optional): The token representing the beginning of a sequence. Defaults to '<s>'.
        eos_token (str, optional): The token representing the end of a sequence. Defaults to '</s>'.
        sep_token (str, optional): The token used to separate two sequences. Defaults to '</s>'.
        cls_token (str, optional): The token representing the classification of a sequence. Defaults to '<s>'.
        unk_token (str, optional): The token representing an unknown word. Defaults to '<unk>'.
        pad_token (str, optional): The token used for padding sequences. Defaults to '<pad>'.
        tokenizer_file (str, optional): The path to the tokenizer file. Defaults to None.
        src_lang (str, optional): The source language. Defaults to 'eng'.
        tgt_lang (str, optional): The target language. Defaults to 'fra'.
        sp_model_kwargs (Optional[Dict[str, Any]], optional): Additional arguments for the sentencepiece model.
            Defaults to None.
        additional_special_tokens (List[str], optional): Additional special tokens. Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
    # Add this unused argument to keep some important Copied from statements
    self.legacy = False
    self.vocab_file = vocab_file

    self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

    # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
    # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
    # spm  | '<unk>'   | '<s>' | '</s>' | 'an' | 'en' | '_d' | 'er' | 'in' | '_s' | '_a'
    # fairseq  | '<pad>'   | '<unk>' | '<s>' | '</s>' | 'an' | 'en' | '▁d' | 'er' | 'in' | '▁s'

    # Mimic fairseq token-to-id alignment for the first 4 token
    self._added_tokens_decoder = {
        0: AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token,
        1: AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token,
        2: AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token,
        3: AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token,
    }

    # The first "real" token "an" has position 4 in the original fairseq vocab and position 3 in the spm vocab
    self.fairseq_offset = 1

    self.sp_model_size = len(self.sp_model)

    self._src_lang = f"__{src_lang}__" if "__" not in src_lang else src_lang
    self._tgt_lang = f"__{tgt_lang}__" if "__" not in tgt_lang else tgt_lang

    super().__init__(
        bos_token=bos_token,
        eos_token=eos_token,
        unk_token=unk_token,
        sep_token=sep_token,
        cls_token=cls_token,
        pad_token=pad_token,
        tokenizer_file=tokenizer_file,
        src_lang=src_lang,
        tgt_lang=tgt_lang,
        additional_special_tokens=additional_special_tokens,
        sp_model_kwargs=self.sp_model_kwargs,
        **kwargs,
    )

    self.set_src_lang_special_tokens(self._src_lang)
    self.set_tgt_lang_special_tokens(self._tgt_lang)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.__setstate__(d)

Method to set the state of the SeamlessM4TTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer
d	A dictionary containing the state information to be set on the instance. TYPE: dict

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def __setstate__(self, d):
    """
    Method to set the state of the SeamlessM4TTokenizer instance.

    Args:
        self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.
        d (dict): A dictionary containing the state information to be set on the instance.

    Returns:
        None.

    Raises:
       None.
    """
    self.__dict__ = d

    # for backward compatibility
    if not hasattr(self, "sp_model_kwargs"):
        self.sp_model_kwargs = {}

    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. An NLLB sequence has the following format, where X represents the sequence:

• input_ids (for encoder) X [eos, src_lang_code]
• decoder_input_ids: (for decoder) X [eos, tgt_lang_code]

BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a separator.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. An NLLB sequence has the following format, where `X` represents the sequence:

    - `input_ids` (for encoder) `X [eos, src_lang_code]`
    - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`

    BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
    separator.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (strings for sub-words) in a single string.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (strings for sub-words) in a single string."""
    if tokens[0].startswith(SPIECE_UNDERLINE):
        tokens[0] = tokens[0][1:]

    out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
    return out_string

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not make use of token type ids, therefore a list of zeros is returned.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of zeros.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
    make use of token type ids, therefore a list of zeros is returned.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of zeros.

    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]

    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    prefix_ones = [1] * len(self.prefix_tokens)
    suffix_ones = [1] * len(self.suffix_tokens)
    if token_ids_1 is None:
        return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
    return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.get_spm_processor(from_slow=False)

Retrieves the SentencePieceProcessor tokenizer for the SeamlessM4TTokenizer class.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer
from_slow	A flag indicating whether to load the tokenizer from the slow version or not. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def get_spm_processor(self, from_slow=False):
    """
    Retrieves the SentencePieceProcessor tokenizer for the SeamlessM4TTokenizer class.

    Args:
        self (SeamlessM4TTokenizer): An instance of the SeamlessM4TTokenizer class.
        from_slow (bool, optional): A flag indicating whether to load the tokenizer from the slow version or not.
            Defaults to False.

    Returns:
        None

    Raises:
        None.
    """
    tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    if self.legacy or from_slow:  # no dependency on protobuf
        tokenizer.Load(self.vocab_file)
        return tokenizer

    with open(self.vocab_file, "rb") as f:
        sp_model = f.read()
        model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
        model = model_pb2.ModelProto.FromString(sp_model)
        normalizer_spec = model_pb2.NormalizerSpec()
        normalizer_spec.add_dummy_prefix = False
        model.normalizer_spec.MergeFrom(normalizer_spec)
        sp_model = model.SerializeToString()
        tokenizer.LoadFromSerializedProto(sp_model)
    return tokenizer

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.get_vocab()

Description: This method returns the vocabulary for the SeamlessM4TTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizer class.

RETURNS	DESCRIPTION
vocab	A dictionary containing the vocabulary, where the keys are tokens and the values are their corresponding IDs.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def get_vocab(self):
    """
    Method: get_vocab

    Description:
    This method returns the vocabulary for the SeamlessM4TTokenizer instance.

    Args:
        self: The instance of the SeamlessM4TTokenizer class.

    Returns:
        vocab: A dictionary containing the vocabulary, where the keys are tokens and the values are their
            corresponding IDs.

    Raises:
        None.
    """
    vocab = {
        self.convert_ids_to_tokens(i): i for i in range(self.fairseq_offset, self.vocab_size + self.fairseq_offset)
    }
    vocab.update(self.added_tokens_encoder)
    return vocab

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.prepare_seq2seq_batch(src_texts, src_lang='eng', tgt_texts=None, tgt_lang='fra', **kwargs)

Prepare Seq2Seq Batch method in the SeamlessM4TTokenizer class.

This method prepares a batch of inputs for sequence-to-sequence models.

PARAMETER	DESCRIPTION
self	The instance of the class.
src_texts	A list of source texts to be encoded. TYPE: List[str]
src_lang	The language of the source texts. Defaults to 'eng'. TYPE: str DEFAULT: 'eng'
tgt_texts	A list of target texts to be encoded. Defaults to None. TYPE: Optional[List[str]] DEFAULT: None
tgt_lang	The language of the target texts. Defaults to 'fra'. TYPE: str DEFAULT: 'fra'
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS	DESCRIPTION
BatchEncoding	A BatchEncoding object containing the prepared batch of inputs. TYPE: BatchEncoding

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def prepare_seq2seq_batch(
    self,
    src_texts: List[str],
    src_lang: str = "eng",
    tgt_texts: Optional[List[str]] = None,
    tgt_lang: str = "fra",
    **kwargs,
) -> BatchEncoding:
    """
    Prepare Seq2Seq Batch method in the SeamlessM4TTokenizer class.

    This method prepares a batch of inputs for sequence-to-sequence models.

    Args:
        self: The instance of the class.
        src_texts (List[str]): A list of source texts to be encoded.
        src_lang (str, optional): The language of the source texts. Defaults to 'eng'.
        tgt_texts (Optional[List[str]], optional): A list of target texts to be encoded. Defaults to None.
        tgt_lang (str, optional): The language of the target texts. Defaults to 'fra'.
        **kwargs: Additional keyword arguments.

    Returns:
        BatchEncoding: A BatchEncoding object containing the prepared batch of inputs.

    Raises:
        None.

    """
    self.src_lang = src_lang
    self.tgt_lang = tgt_lang
    return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary of the SeamlessM4TTokenizer object to a specified directory.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizer class. TYPE: SeamlessM4TTokenizer
save_directory	The path of the directory where the vocabulary will be saved. TYPE: str
filename_prefix	An optional prefix to be added to the filename of the saved vocabulary. Default is None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path of the saved vocabulary file.

RAISES	DESCRIPTION
OSError	If the save_directory is not a valid directory.
IOError	If the vocabulary file cannot be copied or saved.

Note

• The save_directory should be an existing directory.
• If the save_directory already contains a file with the same name as the vocabulary file, it will be overwritten.
• If the self.vocab_file is not an existing file, the vocabulary will be saved directly to the specified directory.

Example

>>> tokenizer = SeamlessM4TTokenizer()
>>> save_directory = '/path/to/save_directory'
>>> filename_prefix = 'my_vocab'
>>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
>>> print(saved_file)  # Output: ('/path/to/save_directory/my_vocab-vocab.txt',)

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary of the SeamlessM4TTokenizer object to a specified directory.

    Args:
        self (SeamlessM4TTokenizer): The instance of the SeamlessM4TTokenizer class.
        save_directory (str): The path of the directory where the vocabulary will be saved.
        filename_prefix (Optional[str]): An optional prefix to be added to the filename of the saved vocabulary.
            Default is None.

    Returns:
        Tuple[str]: A tuple containing the path of the saved vocabulary file.

    Raises:
        OSError: If the save_directory is not a valid directory.
        IOError: If the vocabulary file cannot be copied or saved.

    Note:
        - The save_directory should be an existing directory.
        - If the save_directory already contains a file with the same name as the vocabulary file,
        it will be overwritten.
        - If the self.vocab_file is not an existing file, the vocabulary will be saved directly to the
        specified directory.

    Example:
        ```python
        >>> tokenizer = SeamlessM4TTokenizer()
        >>> save_directory = '/path/to/save_directory'
        >>> filename_prefix = 'my_vocab'
        >>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
        >>> print(saved_file)  # Output: ('/path/to/save_directory/my_vocab-vocab.txt',)
        ```
    """
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file, out_vocab_file)
    elif not os.path.isfile(self.vocab_file):
        with open(out_vocab_file, "wb") as fi:
            content_spiece_model = self.sp_model.serialized_model_proto()
            fi.write(content_spiece_model)

    return (out_vocab_file,)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.set_src_lang_special_tokens(src_lang)

Reset the special tokens to the source lang setting. Prefix=[src_lang_code], suffix = [eos]

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def set_src_lang_special_tokens(self, src_lang) -> None:
    """Reset the special tokens to the source lang setting.
    Prefix=[src_lang_code], suffix = [eos]
    """
    self.cur_lang_code = self.convert_tokens_to_ids(src_lang)
    self.init_kwargs["src_lang"] = src_lang

    if self.cur_lang_code == self.unk_token_id:
        logger.warning_once(
            f"`src_lang={src_lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
        )

    self.prefix_tokens = [self.cur_lang_code]
    self.suffix_tokens = [self.eos_token_id]

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.set_tgt_lang_special_tokens(lang)

Reset the special tokens to the target lang setting. Prefix=[eos, tgt_lang_code] and suffix=[eos].

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def set_tgt_lang_special_tokens(self, lang: str) -> None:
    """Reset the special tokens to the target lang setting.
    Prefix=[eos, tgt_lang_code] and suffix=[eos].
    """
    self.cur_lang_code = self.convert_tokens_to_ids(lang)
    self.init_kwargs["tgt_lang"] = lang

    if self.cur_lang_code == self.unk_token_id:
        logger.warning_once(
            f"`tgt_lang={lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
        )

    self.prefix_tokens = [self.eos_token_id, self.cur_lang_code]
    self.suffix_tokens = [self.eos_token_id]

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t.SeamlessM4TTokenizer.tokenize(text, add_special_tokens=False, **kwargs)

Converts a string to a list of tokens. If self.legacy is set to False, a prefix token is added unless the first token is special.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t.py

def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
    """
    Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
    first token is special.
    """
    if self.legacy or len(text) == 0:
        return super().tokenize(text, **kwargs)

    tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

    if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
        tokens = tokens[1:]
    return tokens

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast

Fast Tokenization class for SeamlessM4T.

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast

Bases: PreTrainedTokenizerFast

Construct a "fast" SeamlessM4T tokenizer (backed by HuggingFace's tokenizers library). Based on BPE.

This tokenizer inherits from [PreTrainedTokenizerFast] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

The tokenization method is <language code> <tokens> <eos> for source language documents, and <eos> <language code> <tokens> <eos> for target language documents.

Example

>>> from transformers import SeamlessM4TTokenizerFast
...
>>> tokenizer = SeamlessM4TTokenizerFast.from_pretrained(
...     "facebook/hf-seamless-m4t-medium", src_lang="eng", tgt_lang="fra"
... )
>>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
>>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
>>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="ms")

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`, *optional* DEFAULT: None
tokenizer_file	The path to a tokenizer file to use instead of the vocab file. TYPE: `str`, *optional* DEFAULT: None
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the cls_token. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
src_lang	The language to use as source language for translation. TYPE: `str`, *optional*, defaults to `"eng"` DEFAULT: 'eng'
tgt_lang	The language to use as target language for translation. TYPE: `str`, *optional*, defaults to `"fra"` DEFAULT: 'fra'
additional_special_tokens	A tuple or a list of additional special tokens. TYPE: tuple or list of `str` or `tokenizers.AddedToken`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

class SeamlessM4TTokenizerFast(PreTrainedTokenizerFast):
    """
    Construct a "fast" SeamlessM4T tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [BPE](https://hf-mirror.com/docs/tokenizers/python/latest/components.html?highlight=BPE#models).

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<language code> <tokens> <eos>` for source language documents, and `<eos> <language
    code> <tokens> <eos>` for target language documents.

    Example:
        ```python
        >>> from transformers import SeamlessM4TTokenizerFast
        ...
        >>> tokenizer = SeamlessM4TTokenizerFast.from_pretrained(
        ...     "facebook/hf-seamless-m4t-medium", src_lang="eng", tgt_lang="fra"
        ... )
        >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
        >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
        >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="ms")
        ```

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        tokenizer_file (`str`, *optional*):
            The path to a tokenizer file to use instead of the vocab file.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        src_lang (`str`, *optional*, defaults to `"eng"`):
            The language to use as source language for translation.
        tgt_lang (`str`, *optional*, defaults to `"fra"`):
            The language to use as target language for translation.
        additional_special_tokens (tuple or list of `str` or `tokenizers.AddedToken`, *optional*):
            A tuple or a list of additional special tokens.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = SeamlessM4TTokenizer
    model_input_names = ["input_ids", "attention_mask"]

    prefix_tokens: List[int] = []
    suffix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        src_lang="eng",
        tgt_lang="fra",
        additional_special_tokens=None,
        **kwargs,
    ):
        """
        Initializes the SeamlessM4TTokenizerFast class.

        Args:
            self: An instance of the SeamlessM4TTokenizerFast class.
            vocab_file (str): Path to the vocabulary file.
            tokenizer_file (str): Path to the tokenizer file.
            bos_token (str): The beginning of sequence token. Defaults to '<s>'.
            eos_token (str): The end of sequence token. Defaults to '</s>'.
            sep_token (str): The separator token. Defaults to '</s>'.
            cls_token (str): The classification token. Defaults to '<s>'.
            unk_token (str): The unknown token. Defaults to '<unk>'.
            pad_token (str): The padding token. Defaults to '<pad>'.
            src_lang (str): The source language. Defaults to 'eng'.
            tgt_lang (str): The target language. Defaults to 'fra'.
            additional_special_tokens (List[str]): A list of additional special tokens. Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(
            vocab_file=vocab_file,
            tokenizer_file=tokenizer_file,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            src_lang=src_lang,
            tgt_lang=tgt_lang,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file
        self._src_lang = f"__{src_lang}__" if "__" not in src_lang else src_lang
        self._tgt_lang = f"__{tgt_lang}__" if "__" not in tgt_lang else tgt_lang
        self.set_src_lang_special_tokens(self._src_lang)
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    @property
    def can_save_slow_tokenizer(self) -> bool:
        """
        This method checks if the slow tokenizer can be saved.

        Args:
            self (SeamlessM4TTokenizerFast): The instance of the SeamlessM4TTokenizerFast class.

        Returns:
            bool: Returns True if the vocab_file exists, False otherwise.

        Raises:
            None
        """
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    @property
    # Copied from transformers.models.nllb.tokenization_nllb.NllbTokenizer.src_lang
    def src_lang(self) -> str:
        """
        This method returns the source language used for tokenization.

        Args:
            self: An instance of the SeamlessM4TTokenizerFast class.

        Returns:
            str: The source language used for tokenization.

        Raises:
            None.
        """
        return self._src_lang

    @src_lang.setter
    def src_lang(self, new_src_lang: str) -> None:
        """
        src_lang(self, new_src_lang: str) -> None

        This method sets the source language for the SeamlessM4TTokenizerFast object.

        Args:
            self: The instance of the SeamlessM4TTokenizerFast class.
            new_src_lang (str): The new source language to be set. It should be a string representing the language code.

        Returns:
            None.

        Raises:
            None.
        """
        if "__" not in new_src_lang:
            self._src_lang = f"__{new_src_lang}__"
        else:
            self._src_lang = new_src_lang
        self.set_src_lang_special_tokens(self._src_lang)

    @property
    def tgt_lang(self) -> str:
        """
        tgt_lang method in the SeamlessM4TTokenizerFast class.

        Args:
            self: A reference to the current instance of the class.

        Returns:
            str: The language code representing the target language for tokenization.

        Raises:
            None.
        """
        return self._tgt_lang

    @tgt_lang.setter
    def tgt_lang(self, new_tgt_lang: str) -> None:
        """
        Sets the target language for the SeamlessM4TTokenizerFast object.

        Args:
            self (SeamlessM4TTokenizerFast): The instance of the SeamlessM4TTokenizerFast class.
            new_tgt_lang (str): The new target language to be set. It should be a string representing the language code.

        Returns:
            None.

        Raises:
            None.
        """
        if "__" not in new_tgt_lang:
            self._tgt_lang = f"__{new_tgt_lang}__"
        else:
            self._tgt_lang = new_tgt_lang
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. The special tokens depend on calling set_lang.

        An SeamlessM4T sequence has the following format, where `X` represents the sequence:

        - `input_ids` (for encoder) `[src_lang_code] X [eos]`
        - `decoder_input_ids`: (for decoder) `[eos, tgt_lang_code] X [eos]`

        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0 + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

    # Copied from transformers.models.nllb.tokenization_nllb_fast.NllbTokenizerFast.create_token_type_ids_from_sequences
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
        make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.

        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def _build_translation_inputs(
        self, raw_inputs, return_tensors: str, src_lang: Optional[str], tgt_lang: Optional[str], **extra_kwargs
    ):
        """Used by translation pipeline, to prepare inputs for the generate function"""
        if src_lang is None or tgt_lang is None:
            raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
        self.src_lang = src_lang
        inputs = self(raw_inputs, add_special_tokens=True, return_tensors=return_tensors, **extra_kwargs)
        if "__" not in tgt_lang:
            tgt_lang = f"__{tgt_lang}__"
        tgt_lang_id = self.convert_tokens_to_ids(tgt_lang)
        inputs["forced_bos_token_id"] = tgt_lang_id
        return inputs

    # Copied from transformers.models.nllb.tokenization_nllb_fast.NllbTokenizerFast.prepare_seq2seq_batch with "fra_Latn"->"fra", "eng_Latn"->"eng"
    def prepare_seq2seq_batch(
        self,
        src_texts: List[str],
        src_lang: str = "eng",
        tgt_texts: Optional[List[str]] = None,
        tgt_lang: str = "fra",
        **kwargs,
    ) -> BatchEncoding:
        """
        Prepares a batch for sequence-to-sequence tokenization using the SeamlessM4TTokenizerFast class.

        Args:
            self (SeamlessM4TTokenizerFast): An instance of the SeamlessM4TTokenizerFast class.
            src_texts (List[str]): A list of source texts to be tokenized.
            src_lang (str, optional): The language of the source texts. Defaults to 'eng'.
            tgt_texts (List[str], optional): A list of target texts to be tokenized. Defaults to None.
            tgt_lang (str, optional): The language of the target texts. Defaults to 'fra'.
            **kwargs: Additional keyword arguments that can be passed to the underlying tokenizer.

        Returns:
            BatchEncoding: A batch encoding containing the tokenized sequences.

        Raises:
            None

        This method prepares a batch of source texts and, optionally, target texts for tokenization using the
        SeamlessM4TTokenizerFast class. It takes the source texts, source language, target texts, and target language
        as input parameters. The method returns a BatchEncoding object, which contains the tokenized sequences.

        The 'self' parameter refers to the instance of the SeamlessM4TTokenizerFast class on which the method is called.

        The 'src_texts' parameter is a list of source texts that need to be tokenized.

        The 'src_lang' parameter specifies the language of the source texts. The default value is 'eng'.

        The 'tgt_texts' parameter is an optional list of target texts that need to be tokenized. If not provided,
        it defaults to None.

        The 'tgt_lang' parameter specifies the language of the target texts. The default value is 'fra'.

        Additional keyword arguments can be passed using the '**kwargs' parameter. These arguments will be forwarded
        to the underlying tokenizer.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizerFast()
            >>> src_texts = ["Hello, world!", "How are you?"]
            >>> tgt_texts = ["Bonjour, le monde!", "Comment ça va?"]
            >>> batch = tokenizer.prepare_seq2seq_batch(src_texts, src_lang='eng', tgt_texts=tgt_texts, tgt_lang='fra')
            ```
        """
        self.src_lang = src_lang
        self.tgt_lang = tgt_lang
        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

    # Copied from transformers.models.nllb.tokenization_nllb_fast.NllbTokenizerFast._switch_to_input_mode
    def _switch_to_input_mode(self):
        """
        Method to switch the tokenizer to input mode by setting source language special tokens.

        Args:
            self (SeamlessM4TTokenizerFast): The instance of the SeamlessM4TTokenizerFast class.
                Represents the tokenizer object.

        Returns:
            None.

        Raises:
            None.
        """
        return self.set_src_lang_special_tokens(self.src_lang)

    # Copied from transformers.models.nllb.tokenization_nllb_fast.NllbTokenizerFast._switch_to_target_mode
    def _switch_to_target_mode(self):
        """
        Switches the tokenizer to target mode for SeamlessM4TTokenizerFast.

        Args:
            self:
                The instance of SeamlessM4TTokenizerFast.

                - Type: object
                - Purpose: Represents the instance of the SeamlessM4TTokenizerFast class.
                - Restrictions: None

        Returns:
            None:
                Indicates that no value is returned from this method.

                - Type: None
                - Purpose: The method sets the target language special tokens and does not return any value.

        Raises:
            None
        """
        return self.set_tgt_lang_special_tokens(self.tgt_lang)

    def set_src_lang_special_tokens(self, src_lang) -> None:
        """Reset the special tokens to the source lang setting.
        Prefix=[src_lang_code], suffix = [eos]
        """
        self.cur_lang_code = self.convert_tokens_to_ids(src_lang)

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`tgt_lang={src_lang}` has not be found in the `vocabulary`. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.init_kwargs["src_lang"] = src_lang

        self.prefix_tokens = [self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]

        prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens)
        suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens)

        self._tokenizer.post_processor = processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str,
            pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str,
            special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)),
        )

    def set_tgt_lang_special_tokens(self, lang: str) -> None:
        """
        Reset the special tokens to the target lang setting.
        Prefix=[eos, tgt_lang_code] and suffix=[eos].
        """
        self.cur_lang_code = self.convert_tokens_to_ids(lang)

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`tgt_lang={lang}` has not be found in the `vocabulary`. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.init_kwargs["tgt_lang"] = lang

        self.prefix_tokens = [self.eos_token_id, self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]

        prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens)
        suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens)

        self._tokenizer.post_processor = processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str,
            pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str,
            special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)),
        )

    # Copied from transformers.models.nllb.tokenization_nllb_fast.NllbTokenizerFast.save_vocabulary
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary for a slow tokenizer.

        Args:
            self (SeamlessM4TTokenizerFast): The instance of the class.
            save_directory (str): The directory where the vocabulary will be saved.
            filename_prefix (Optional[str], optional): The prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            ValueError: If the fast tokenizer does not have the necessary information to save the vocabulary
                for a slow tokenizer.
            FileNotFoundError: If the save_directory does not exist.
            IsADirectoryError: If the save_directory is not a directory.

        Note:
            The method assumes that the fast tokenizer has the necessary information to save the vocabulary
            for a slow tokenizer.

        Example:
            ```python
            >>> tokenizer = SeamlessM4TTokenizerFast()
            >>> save_directory = '/path/to/save/directory'
            >>> filename_prefix = 'vocab'
            >>> vocab_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
            >>> # vocab_file is now ('/path/to/save/directory/vocab-file', )
            ```
        """
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory.")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)

    @classmethod
    def _from_pretrained(
        cls,
        resolved_vocab_files,
        pretrained_model_name_or_path,
        init_configuration,
        *init_inputs,
        token=None,
        cache_dir=None,
        local_files_only=False,
        _commit_hash=None,
        _is_local=False,
        **kwargs,
    ):
        """
        Method _from_pretrained in the class SeamlessM4TTokenizerFast.

        Args:
            cls (class): The class itself.
            resolved_vocab_files (dict): A dictionary containing resolved vocabulary files.
            pretrained_model_name_or_path (str): The name or path of the pretrained model.
            init_configuration (dict): Initial configuration settings for the tokenizer.

        Returns:
            None.

        Raises:
            None.
        """
        tokenizer = super()._from_pretrained(
            resolved_vocab_files,
            pretrained_model_name_or_path,
            init_configuration,
            *init_inputs,
            token=token,
            cache_dir=cache_dir,
            local_files_only=local_files_only,
            _commit_hash=_commit_hash,
            _is_local=_is_local,
            **kwargs,
        )

        # ensure also set after from pretrained
        tokenizer.set_src_lang_special_tokens(tokenizer._src_lang)
        tokenizer.set_tgt_lang_special_tokens(tokenizer._tgt_lang)

        return tokenizer

    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
        text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair_target: Optional[
            Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
        ] = None,
        padding: Union[bool, str, PaddingStrategy] = True,
        pad_to_multiple_of: Optional[int] = 2,
        src_lang: Optional[str] = None,
        tgt_lang: Optional[str] = None,
        **kwargs,
    ):
        """
        Args:
            text (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_pair (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
                list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
                you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            text_pair_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
                The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
                list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
                you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
                 Select a strategy to pad the returned sequences (according to the model's padding side and padding
                 index) among:

                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
                sequence if provided).
                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
                acceptable input length for the model if that argument is not provided.
                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
                lengths).
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the sequence to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta).
            src_lang (`str`, *optional*):
                A string representing the source language. If not specified, the last `src_lang` specified (either
                during initialization or when calling this tokenizer) will be used.
            tgt_lang (`str`, *optional*):
                A string representing the target language. If not specified, the last `tgt_lang` specified (either
                during initialization or when calling this tokenizer) will be used.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to [`PreTrainedTokenizerFast.__call__`].
        """
        if src_lang is not None:
            self.src_lang = src_lang
        if tgt_lang is not None:
            self.tgt_lang = tgt_lang

        output = super().__call__(
            text=text,
            text_pair=text_pair,
            text_target=text_target,
            text_pair_target=text_pair_target,
            padding=padding,
            pad_to_multiple_of=pad_to_multiple_of,
            **kwargs,
        )

        return output

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.can_save_slow_tokenizer: bool property

This method checks if the slow tokenizer can be saved.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TTokenizerFast class. TYPE: SeamlessM4TTokenizerFast

RETURNS	DESCRIPTION
bool	Returns True if the vocab_file exists, False otherwise. TYPE: bool

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.src_lang: str property writable

This method returns the source language used for tokenization.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizerFast class.

RETURNS	DESCRIPTION
str	The source language used for tokenization. TYPE: str

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.tgt_lang: str property writable

tgt_lang method in the SeamlessM4TTokenizerFast class.

PARAMETER	DESCRIPTION
self	A reference to the current instance of the class.

RETURNS	DESCRIPTION
str	The language code representing the target language for tokenization. TYPE: str

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.__call__(text=None, text_pair=None, text_target=None, text_pair_target=None, padding=True, pad_to_multiple_of=2, src_lang=None, tgt_lang=None, **kwargs)

PARAMETER	DESCRIPTION
text	The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_pair	The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_target	The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
text_pair_target	The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]`, *optional* DEFAULT: None
padding	Select a strategy to pad the returned sequences (according to the model's padding side and padding index) among: True or 'longest': Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). 'max_length': Pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided. False or 'do_not_pad' (default): No padding (i.e., can output a batch with sequences of different lengths). TYPE: `bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True` DEFAULT: True
pad_to_multiple_of	If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta). TYPE: `int`, *optional* DEFAULT: 2
src_lang	A string representing the source language. If not specified, the last src_lang specified (either during initialization or when calling this tokenizer) will be used. TYPE: `str`, *optional* DEFAULT: None
tgt_lang	A string representing the target language. If not specified, the last tgt_lang specified (either during initialization or when calling this tokenizer) will be used. TYPE: `str`, *optional* DEFAULT: None
kwargs	Remaining dictionary of keyword arguments that will be passed to [PreTrainedTokenizerFast.__call__]. TYPE: *optional* DEFAULT: {}

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def __call__(
    self,
    text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
    text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
    text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
    text_pair_target: Optional[
        Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
    ] = None,
    padding: Union[bool, str, PaddingStrategy] = True,
    pad_to_multiple_of: Optional[int] = 2,
    src_lang: Optional[str] = None,
    tgt_lang: Optional[str] = None,
    **kwargs,
):
    """
    Args:
        text (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
            (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
            `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_pair (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
            (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
            `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
            list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
            you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        text_pair_target (`str`, `List[str]`, `List[List[str]]`, *optional*):
            The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
            list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized),
            you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
             Select a strategy to pad the returned sequences (according to the model's padding side and padding
             index) among:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
            sequence if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
            acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
            lengths).
        pad_to_multiple_of (`int`, *optional*):
            If set will pad the sequence to a multiple of the provided value.

            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
            `>= 7.5` (Volta).
        src_lang (`str`, *optional*):
            A string representing the source language. If not specified, the last `src_lang` specified (either
            during initialization or when calling this tokenizer) will be used.
        tgt_lang (`str`, *optional*):
            A string representing the target language. If not specified, the last `tgt_lang` specified (either
            during initialization or when calling this tokenizer) will be used.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to [`PreTrainedTokenizerFast.__call__`].
    """
    if src_lang is not None:
        self.src_lang = src_lang
    if tgt_lang is not None:
        self.tgt_lang = tgt_lang

    output = super().__call__(
        text=text,
        text_pair=text_pair,
        text_target=text_target,
        text_pair_target=text_pair_target,
        padding=padding,
        pad_to_multiple_of=pad_to_multiple_of,
        **kwargs,
    )

    return output

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.__init__(vocab_file=None, tokenizer_file=None, bos_token='<s>', eos_token='</s>', sep_token='</s>', cls_token='<s>', unk_token='<unk>', pad_token='<pad>', src_lang='eng', tgt_lang='fra', additional_special_tokens=None, **kwargs)

Initializes the SeamlessM4TTokenizerFast class.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizerFast class.
vocab_file	Path to the vocabulary file. TYPE: str DEFAULT: None
tokenizer_file	Path to the tokenizer file. TYPE: str DEFAULT: None
bos_token	The beginning of sequence token. Defaults to ''. TYPE: str DEFAULT: '<s>'
eos_token	The end of sequence token. Defaults to ''. TYPE: str DEFAULT: '</s>'
sep_token	The separator token. Defaults to ''. TYPE: str DEFAULT: '</s>'
cls_token	The classification token. Defaults to ''. TYPE: str DEFAULT: '<s>'
unk_token	The unknown token. Defaults to ''. TYPE: str DEFAULT: '<unk>'
pad_token	The padding token. Defaults to ''. TYPE: str DEFAULT: '<pad>'
src_lang	The source language. Defaults to 'eng'. TYPE: str DEFAULT: 'eng'
tgt_lang	The target language. Defaults to 'fra'. TYPE: str DEFAULT: 'fra'
additional_special_tokens	A list of additional special tokens. Defaults to None. TYPE: List[str] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def __init__(
    self,
    vocab_file=None,
    tokenizer_file=None,
    bos_token="<s>",
    eos_token="</s>",
    sep_token="</s>",
    cls_token="<s>",
    unk_token="<unk>",
    pad_token="<pad>",
    src_lang="eng",
    tgt_lang="fra",
    additional_special_tokens=None,
    **kwargs,
):
    """
    Initializes the SeamlessM4TTokenizerFast class.

    Args:
        self: An instance of the SeamlessM4TTokenizerFast class.
        vocab_file (str): Path to the vocabulary file.
        tokenizer_file (str): Path to the tokenizer file.
        bos_token (str): The beginning of sequence token. Defaults to '<s>'.
        eos_token (str): The end of sequence token. Defaults to '</s>'.
        sep_token (str): The separator token. Defaults to '</s>'.
        cls_token (str): The classification token. Defaults to '<s>'.
        unk_token (str): The unknown token. Defaults to '<unk>'.
        pad_token (str): The padding token. Defaults to '<pad>'.
        src_lang (str): The source language. Defaults to 'eng'.
        tgt_lang (str): The target language. Defaults to 'fra'.
        additional_special_tokens (List[str]): A list of additional special tokens. Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(
        vocab_file=vocab_file,
        tokenizer_file=tokenizer_file,
        bos_token=bos_token,
        eos_token=eos_token,
        sep_token=sep_token,
        cls_token=cls_token,
        unk_token=unk_token,
        pad_token=pad_token,
        src_lang=src_lang,
        tgt_lang=tgt_lang,
        additional_special_tokens=additional_special_tokens,
        **kwargs,
    )

    self.vocab_file = vocab_file
    self._src_lang = f"__{src_lang}__" if "__" not in src_lang else src_lang
    self._tgt_lang = f"__{tgt_lang}__" if "__" not in tgt_lang else tgt_lang
    self.set_src_lang_special_tokens(self._src_lang)
    self.set_tgt_lang_special_tokens(self._tgt_lang)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. The special tokens depend on calling set_lang.

An SeamlessM4T sequence has the following format, where X represents the sequence:

• input_ids (for encoder) [src_lang_code] X [eos]
• decoder_input_ids: (for decoder) [eos, tgt_lang_code] X [eos]

BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a separator.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: list of input IDs with the appropriate special tokens.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. The special tokens depend on calling set_lang.

    An SeamlessM4T sequence has the following format, where `X` represents the sequence:

    - `input_ids` (for encoder) `[src_lang_code] X [eos]`
    - `decoder_input_ids`: (for decoder) `[eos, tgt_lang_code] X [eos]`

    BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
    separator.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not make use of token type ids, therefore a list of zeros is returned.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of zeros.

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
    make use of token type ids, therefore a list of zeros is returned.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of zeros.

    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]

    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.prepare_seq2seq_batch(src_texts, src_lang='eng', tgt_texts=None, tgt_lang='fra', **kwargs)

Prepares a batch for sequence-to-sequence tokenization using the SeamlessM4TTokenizerFast class.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TTokenizerFast class. TYPE: SeamlessM4TTokenizerFast
src_texts	A list of source texts to be tokenized. TYPE: List[str]
src_lang	The language of the source texts. Defaults to 'eng'. TYPE: str DEFAULT: 'eng'
tgt_texts	A list of target texts to be tokenized. Defaults to None. TYPE: List[str] DEFAULT: None
tgt_lang	The language of the target texts. Defaults to 'fra'. TYPE: str DEFAULT: 'fra'
**kwargs	Additional keyword arguments that can be passed to the underlying tokenizer. DEFAULT: {}

RETURNS	DESCRIPTION
BatchEncoding	A batch encoding containing the tokenized sequences. TYPE: BatchEncoding

This method prepares a batch of source texts and, optionally, target texts for tokenization using the SeamlessM4TTokenizerFast class. It takes the source texts, source language, target texts, and target language as input parameters. The method returns a BatchEncoding object, which contains the tokenized sequences.

The 'self' parameter refers to the instance of the SeamlessM4TTokenizerFast class on which the method is called.

The 'src_texts' parameter is a list of source texts that need to be tokenized.

The 'src_lang' parameter specifies the language of the source texts. The default value is 'eng'.

The 'tgt_texts' parameter is an optional list of target texts that need to be tokenized. If not provided, it defaults to None.

The 'tgt_lang' parameter specifies the language of the target texts. The default value is 'fra'.

Additional keyword arguments can be passed using the '**kwargs' parameter. These arguments will be forwarded to the underlying tokenizer.

Example

>>> tokenizer = SeamlessM4TTokenizerFast()
>>> src_texts = ["Hello, world!", "How are you?"]
>>> tgt_texts = ["Bonjour, le monde!", "Comment ça va?"]
>>> batch = tokenizer.prepare_seq2seq_batch(src_texts, src_lang='eng', tgt_texts=tgt_texts, tgt_lang='fra')

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def prepare_seq2seq_batch(
    self,
    src_texts: List[str],
    src_lang: str = "eng",
    tgt_texts: Optional[List[str]] = None,
    tgt_lang: str = "fra",
    **kwargs,
) -> BatchEncoding:
    """
    Prepares a batch for sequence-to-sequence tokenization using the SeamlessM4TTokenizerFast class.

    Args:
        self (SeamlessM4TTokenizerFast): An instance of the SeamlessM4TTokenizerFast class.
        src_texts (List[str]): A list of source texts to be tokenized.
        src_lang (str, optional): The language of the source texts. Defaults to 'eng'.
        tgt_texts (List[str], optional): A list of target texts to be tokenized. Defaults to None.
        tgt_lang (str, optional): The language of the target texts. Defaults to 'fra'.
        **kwargs: Additional keyword arguments that can be passed to the underlying tokenizer.

    Returns:
        BatchEncoding: A batch encoding containing the tokenized sequences.

    Raises:
        None

    This method prepares a batch of source texts and, optionally, target texts for tokenization using the
    SeamlessM4TTokenizerFast class. It takes the source texts, source language, target texts, and target language
    as input parameters. The method returns a BatchEncoding object, which contains the tokenized sequences.

    The 'self' parameter refers to the instance of the SeamlessM4TTokenizerFast class on which the method is called.

    The 'src_texts' parameter is a list of source texts that need to be tokenized.

    The 'src_lang' parameter specifies the language of the source texts. The default value is 'eng'.

    The 'tgt_texts' parameter is an optional list of target texts that need to be tokenized. If not provided,
    it defaults to None.

    The 'tgt_lang' parameter specifies the language of the target texts. The default value is 'fra'.

    Additional keyword arguments can be passed using the '**kwargs' parameter. These arguments will be forwarded
    to the underlying tokenizer.

    Example:
        ```python
        >>> tokenizer = SeamlessM4TTokenizerFast()
        >>> src_texts = ["Hello, world!", "How are you?"]
        >>> tgt_texts = ["Bonjour, le monde!", "Comment ça va?"]
        >>> batch = tokenizer.prepare_seq2seq_batch(src_texts, src_lang='eng', tgt_texts=tgt_texts, tgt_lang='fra')
        ```
    """
    self.src_lang = src_lang
    self.tgt_lang = tgt_lang
    return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary for a slow tokenizer.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: SeamlessM4TTokenizerFast
save_directory	The directory where the vocabulary will be saved. TYPE: str
filename_prefix	The prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
ValueError	If the fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.
FileNotFoundError	If the save_directory does not exist.
IsADirectoryError	If the save_directory is not a directory.

Note

The method assumes that the fast tokenizer has the necessary information to save the vocabulary for a slow tokenizer.

Example

>>> tokenizer = SeamlessM4TTokenizerFast()
>>> save_directory = '/path/to/save/directory'
>>> filename_prefix = 'vocab'
>>> vocab_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
>>> # vocab_file is now ('/path/to/save/directory/vocab-file', )

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary for a slow tokenizer.

    Args:
        self (SeamlessM4TTokenizerFast): The instance of the class.
        save_directory (str): The directory where the vocabulary will be saved.
        filename_prefix (Optional[str], optional): The prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        ValueError: If the fast tokenizer does not have the necessary information to save the vocabulary
            for a slow tokenizer.
        FileNotFoundError: If the save_directory does not exist.
        IsADirectoryError: If the save_directory is not a directory.

    Note:
        The method assumes that the fast tokenizer has the necessary information to save the vocabulary
        for a slow tokenizer.

    Example:
        ```python
        >>> tokenizer = SeamlessM4TTokenizerFast()
        >>> save_directory = '/path/to/save/directory'
        >>> filename_prefix = 'vocab'
        >>> vocab_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
        >>> # vocab_file is now ('/path/to/save/directory/vocab-file', )
        ```
    """
    if not self.can_save_slow_tokenizer:
        raise ValueError(
            "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
            "tokenizer."
        )

    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory.")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
        copyfile(self.vocab_file, out_vocab_file)

    return (out_vocab_file,)

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.set_src_lang_special_tokens(src_lang)

Reset the special tokens to the source lang setting. Prefix=[src_lang_code], suffix = [eos]

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def set_src_lang_special_tokens(self, src_lang) -> None:
    """Reset the special tokens to the source lang setting.
    Prefix=[src_lang_code], suffix = [eos]
    """
    self.cur_lang_code = self.convert_tokens_to_ids(src_lang)

    if self.cur_lang_code == self.unk_token_id:
        logger.warning_once(
            f"`tgt_lang={src_lang}` has not be found in the `vocabulary`. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
        )

    self.init_kwargs["src_lang"] = src_lang

    self.prefix_tokens = [self.cur_lang_code]
    self.suffix_tokens = [self.eos_token_id]

    prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens)
    suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens)

    self._tokenizer.post_processor = processors.TemplateProcessing(
        single=prefix_tokens_str + ["$A"] + suffix_tokens_str,
        pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str,
        special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)),
    )

mindnlp.transformers.models.seamless_m4t.tokenization_seamless_m4t_fast.SeamlessM4TTokenizerFast.set_tgt_lang_special_tokens(lang)

Reset the special tokens to the target lang setting. Prefix=[eos, tgt_lang_code] and suffix=[eos].

Source code in mindnlp\transformers\models\seamless_m4t\tokenization_seamless_m4t_fast.py

def set_tgt_lang_special_tokens(self, lang: str) -> None:
    """
    Reset the special tokens to the target lang setting.
    Prefix=[eos, tgt_lang_code] and suffix=[eos].
    """
    self.cur_lang_code = self.convert_tokens_to_ids(lang)

    if self.cur_lang_code == self.unk_token_id:
        logger.warning_once(
            f"`tgt_lang={lang}` has not be found in the `vocabulary`. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
        )

    self.init_kwargs["tgt_lang"] = lang

    self.prefix_tokens = [self.eos_token_id, self.cur_lang_code]
    self.suffix_tokens = [self.eos_token_id]

    prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens)
    suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens)

    self._tokenizer.post_processor = processors.TemplateProcessing(
        single=prefix_tokens_str + ["$A"] + suffix_tokens_str,
        pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str,
        special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)),
    )

mindnlp.transformers.models.seamless_m4t.feature_extraction_seamless_m4t

Feature extractor class for SeamlessM4T

mindnlp.transformers.models.seamless_m4t.feature_extraction_seamless_m4t.SeamlessM4TFeatureExtractor

Bases: SequenceFeatureExtractor

Constructs a SeamlessM4T feature extractor.

This feature extractor inherits from [SequenceFeatureExtractor] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

This class extracts mel-filter bank features from raw speech.

PARAMETER	DESCRIPTION
feature_size	The feature dimension of the extracted features. TYPE: `int`, *optional*, defaults to 80 DEFAULT: 80
sampling_rate	The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). TYPE: `int`, *optional*, defaults to 16000 DEFAULT: 16000
num_mel_bins	Number of Mel-frequency bins. TYPE: `int`, *optional*, defaults to 80 DEFAULT: 80
padding_value	The value that is used to fill the padding vectors. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
stride	Stride used to reshape audios from shape (batch_size,num_frames,num_mel_bins) to (batch_size,num_frames//stride,num_mel_bins*stride). TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2

Source code in mindnlp\transformers\models\seamless_m4t\feature_extraction_seamless_m4t.py

class SeamlessM4TFeatureExtractor(SequenceFeatureExtractor):
    r"""
    Constructs a SeamlessM4T feature extractor.

    This feature extractor inherits from [`SequenceFeatureExtractor`] which contains most of the main methods. Users
    should refer to this superclass for more information regarding those methods.

    This class extracts mel-filter bank features from raw speech.

    Args:
        feature_size (`int`, *optional*, defaults to 80):
            The feature dimension of the extracted features.
        sampling_rate (`int`, *optional*, defaults to 16000):
            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
        num_mel_bins (`int`, *optional*, defaults to 80):
            Number of Mel-frequency bins.
        padding_value (`float`, *optional*, defaults to 0.0):
            The value that is used to fill the padding vectors.
        stride (`int`, *optional*, defaults to 2):
            Stride used to reshape audios from shape (batch_size,num_frames,num_mel_bins) to
            (batch_size,num_frames//stride,num_mel_bins*stride).
    """
    model_input_names = ["input_features", "attention_mask"]

    def __init__(
        self,
        feature_size=80,
        sampling_rate=16000,
        num_mel_bins=80,
        padding_value=0.0,
        stride=2,
        **kwargs,
    ):
        """
        Initializes an instance of the SeamlessM4TFeatureExtractor class.

        Args:
            self (SeamlessM4TFeatureExtractor): The instance of the class.
            feature_size (int, optional): The size of the extracted feature. Defaults to 80.
            sampling_rate (int, optional): The sampling rate of the audio. Defaults to 16000.
            num_mel_bins (int, optional): The number of mel bins for mel-frequency cepstral coefficients (MFCC).
                Defaults to 80.
            padding_value (float, optional): The value used for padding. Defaults to 0.0.
            stride (int, optional): The stride for feature extraction. Defaults to 2.

        Returns:
            None.

        Raises:
            None.
        """
        self.num_mel_bins = num_mel_bins
        self.return_attention_mask = True
        self.stride = stride

        mel_filters = mel_filter_bank(
            num_frequency_bins=256,
            num_mel_filters=self.num_mel_bins,
            min_frequency=20,
            max_frequency=sampling_rate // 2,
            sampling_rate=sampling_rate,
            norm=None,
            mel_scale="kaldi",
            triangularize_in_mel_space=True,
        )

        self.mel_filters = np.pad(mel_filters, ((0, 1), (0, 0)))
        self.window = window_function(400, "povey", periodic=False)

        super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)

    @staticmethod
    # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
    def zero_mean_unit_var_norm(
        input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
    ) -> List[np.ndarray]:
        """
        Every array in the list is normalized to have zero mean and unit variance
        """
        if attention_mask is not None:
            attention_mask = np.array(attention_mask, np.int32)
            normed_input_values = []

            for vector, length in zip(input_values, attention_mask.sum(-1)):
                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
                if length < normed_slice.shape[0]:
                    normed_slice[length:] = padding_value

                normed_input_values.append(normed_slice)
        else:
            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

        return normed_input_values

    def _extract_fbank_features(
        self,
        waveform: np.ndarray,
    ) -> np.ndarray:
        """
        Get mel-filter bank features using TorchAudio. Note that TorchAudio requires 16-bit signed integers as inputs
        and hence the waveform should not be normalized before feature extraction.
        """
        # by default, it extracts the left channel if stereo
        if len(waveform.shape) == 2:
            waveform = waveform[0]

        waveform = np.squeeze(waveform) * (2**15)  # Kaldi compliance: 16-bit signed integers
        features = spectrogram(
            waveform,
            self.window,
            frame_length=400,
            hop_length=160,
            fft_length=512,
            power=2.0,
            center=False,
            preemphasis=0.97,
            mel_filters=self.mel_filters,
            log_mel="log",
            mel_floor=1.192092955078125e-07,
            remove_dc_offset=True,
        ).T
        return features

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        padding: Union[bool, str, PaddingStrategy] = True,
        pad_to_multiple_of: Optional[int] = 2,
        max_length: Optional[int] = None,
        truncation: bool = False,
        return_tensors: Optional[Union[str, TensorType]] = None,
        sampling_rate: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        do_normalize_per_mel_bins: Optional[bool] = True,
        **kwargs,
    ) -> BatchFeature:
        """
        Main method to featurize and prepare for the model one or several sequence(s).

        Args:
            raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`, `List[List[List[float]]]`):
                The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
                values, a list of numpy arrays, a list of list of float values or a list of a list of list of float
                values. If `raw_speech` is a one-dimensional `np.ndarray` or a `List[float]`, `raw_speech` is
                considered a single-channel, single-sample sound. In all other cases, the first dimension of
                `raw_speech`, whether from an `np.ndarray` or a `List[...]`, corresponds to the number of samples in
                the batch, and the number of channels (i.e. mono or stereo character) is derived from the other
                dimensions (1D -> single-channel waveform batches; 2D-> stereo-channel waveform batches).
            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
                Select a strategy to pad the returned sequences (according to the model's padding side and padding
                index) among:

                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
                sequence if provided).
                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
                acceptable input length for the model if that argument is not provided.
                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
                lengths).
            pad_to_multiple_of (`int`, *optional*, defaults to 2):
                If set will pad the sequence to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
            max_length (`int`, *optional*):
                Maximum length of the returned list and optionally padding length (see above).
            truncation (`bool`):
                Activates truncation to cut input sequences longer than *max_length* to *max_length*.
            return_attention_mask (`bool`, *optional*):
                Whether to return the attention mask. If left to the default, will return the attention mask according
                to the specific feature_extractor's default.

                [What are attention masks?](../glossary#attention-mask)

                <Tip>

                For SeamlessM4T models, `attention_mask` should always be passed for batched inference, to avoid subtle
                bugs.

                </Tip>

            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
            sampling_rate (`int`, *optional*):
                The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
                `sampling_rate` at the forward call to prevent silent errors.
            do_normalize_per_mel_bins (`bool`, *optional*, defaults to `True`):
                Whether or not to zero-mean unit-variance normalize the input per mel-channel.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to the tokenizer or the feature
                extractor.
        """
        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                    f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                    f" {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 3:
            raise ValueError(f"Only mono-channel or stereo-channel audio is supported for input to {self}")

        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        # always return batch
        if not is_batched:
            raw_speech = [raw_speech]

        # extract fbank features
        features = [self._extract_fbank_features(waveform) for waveform in raw_speech]

        if do_normalize_per_mel_bins:
            # torch defaults to ddof=1, and numpy defaults to ddof=0
            features = [
                (x - np.expand_dims(x.mean(0), 0)) / np.sqrt(np.expand_dims(x.var(0, ddof=1), 0) + 1e-7)
                for x in features
            ]

        # convert into correct format for padding
        encoded_inputs = BatchFeature({"input_features": features})

        padded_inputs = self.pad(
            encoded_inputs,
            padding=padding,
            max_length=max_length,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            return_tensors="np",
        )

        # SeamlessM4T needs to process extracted features
        input_features = padded_inputs.get("input_features")
        attention_mask = padded_inputs.get("attention_mask")

        batch_size, num_frames, num_channels = input_features.shape

        remainder = num_frames % self.stride
        if remainder != 0:
            input_features = input_features[:, :num_frames, :]
            attention_mask = attention_mask[:, :num_frames]

        input_features = np.reshape(
            input_features, (batch_size, num_frames // self.stride, num_channels * self.stride)
        )

        indices = np.arange(0, num_frames)
        attention_mask = attention_mask[:, indices % self.stride == 1]

        padded_inputs["input_features"] = input_features
        padded_inputs["attention_mask"] = attention_mask

        if return_tensors is not None:
            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

        return padded_inputs

mindnlp.transformers.models.seamless_m4t.feature_extraction_seamless_m4t.SeamlessM4TFeatureExtractor.__call__(raw_speech, padding=True, pad_to_multiple_of=2, max_length=None, truncation=False, return_tensors=None, sampling_rate=None, return_attention_mask=None, do_normalize_per_mel_bins=True, **kwargs)

Main method to featurize and prepare for the model one or several sequence(s).

PARAMETER	DESCRIPTION
raw_speech	The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float values, a list of numpy arrays, a list of list of float values or a list of a list of list of float values. If raw_speech is a one-dimensional np.ndarray or a List[float], raw_speech is considered a single-channel, single-sample sound. In all other cases, the first dimension of raw_speech, whether from an np.ndarray or a List[...], corresponds to the number of samples in the batch, and the number of channels (i.e. mono or stereo character) is derived from the other dimensions (1D -> single-channel waveform batches; 2D-> stereo-channel waveform batches). TYPE: `np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`, `List[List[List[float]]]`
padding	Select a strategy to pad the returned sequences (according to the model's padding side and padding index) among: True or 'longest': Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). 'max_length': Pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided. False or 'do_not_pad' (default): No padding (i.e., can output a batch with sequences of different lengths). TYPE: `bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True` DEFAULT: True
pad_to_multiple_of	If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
max_length	Maximum length of the returned list and optionally padding length (see above). TYPE: `int`, *optional* DEFAULT: None
truncation	Activates truncation to cut input sequences longer than max_length to max_length. TYPE: `bool` DEFAULT: False
return_attention_mask	Whether to return the attention mask. If left to the default, will return the attention mask according to the specific feature_extractor's default. What are attention masks? For SeamlessM4T models, attention_mask should always be passed for batched inference, to avoid subtle bugs. TYPE: `bool`, *optional* DEFAULT: None
return_tensors	If set, will return tensors instead of list of python integers. Acceptable values are: 'tf': Return TensorFlow tf.constant objects. 'pt': Return PyTorch torch.Tensor objects. 'np': Return Numpy np.ndarray objects. TYPE: `str` or [`~utils.TensorType`], *optional* DEFAULT: None
sampling_rate	The sampling rate at which the raw_speech input was sampled. It is strongly recommended to pass sampling_rate at the forward call to prevent silent errors. TYPE: `int`, *optional* DEFAULT: None
do_normalize_per_mel_bins	Whether or not to zero-mean unit-variance normalize the input per mel-channel. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
kwargs	Remaining dictionary of keyword arguments that will be passed to the tokenizer or the feature extractor. TYPE: *optional* DEFAULT: {}

Source code in mindnlp\transformers\models\seamless_m4t\feature_extraction_seamless_m4t.py

def __call__(
    self,
    raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
    padding: Union[bool, str, PaddingStrategy] = True,
    pad_to_multiple_of: Optional[int] = 2,
    max_length: Optional[int] = None,
    truncation: bool = False,
    return_tensors: Optional[Union[str, TensorType]] = None,
    sampling_rate: Optional[int] = None,
    return_attention_mask: Optional[bool] = None,
    do_normalize_per_mel_bins: Optional[bool] = True,
    **kwargs,
) -> BatchFeature:
    """
    Main method to featurize and prepare for the model one or several sequence(s).

    Args:
        raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`, `List[List[List[float]]]`):
            The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
            values, a list of numpy arrays, a list of list of float values or a list of a list of list of float
            values. If `raw_speech` is a one-dimensional `np.ndarray` or a `List[float]`, `raw_speech` is
            considered a single-channel, single-sample sound. In all other cases, the first dimension of
            `raw_speech`, whether from an `np.ndarray` or a `List[...]`, corresponds to the number of samples in
            the batch, and the number of channels (i.e. mono or stereo character) is derived from the other
            dimensions (1D -> single-channel waveform batches; 2D-> stereo-channel waveform batches).
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
            Select a strategy to pad the returned sequences (according to the model's padding side and padding
            index) among:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
            sequence if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
            acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
            lengths).
        pad_to_multiple_of (`int`, *optional*, defaults to 2):
            If set will pad the sequence to a multiple of the provided value.

            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
            `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
        max_length (`int`, *optional*):
            Maximum length of the returned list and optionally padding length (see above).
        truncation (`bool`):
            Activates truncation to cut input sequences longer than *max_length* to *max_length*.
        return_attention_mask (`bool`, *optional*):
            Whether to return the attention mask. If left to the default, will return the attention mask according
            to the specific feature_extractor's default.

            [What are attention masks?](../glossary#attention-mask)

            <Tip>

            For SeamlessM4T models, `attention_mask` should always be passed for batched inference, to avoid subtle
            bugs.

            </Tip>

        return_tensors (`str` or [`~utils.TensorType`], *optional*):
            If set, will return tensors instead of list of python integers. Acceptable values are:

            - `'tf'`: Return TensorFlow `tf.constant` objects.
            - `'pt'`: Return PyTorch `torch.Tensor` objects.
            - `'np'`: Return Numpy `np.ndarray` objects.
        sampling_rate (`int`, *optional*):
            The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
            `sampling_rate` at the forward call to prevent silent errors.
        do_normalize_per_mel_bins (`bool`, *optional*, defaults to `True`):
            Whether or not to zero-mean unit-variance normalize the input per mel-channel.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to the tokenizer or the feature
            extractor.
    """
    if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
            raise ValueError(
                f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                f" {self.sampling_rate} and not {sampling_rate}."
            )
    else:
        logger.warning(
            "It is strongly recommended to pass the `sampling_rate` argument to this function. "
            "Failing to do so can result in silent errors that might be hard to debug."
        )

    is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
    if is_batched_numpy and len(raw_speech.shape) > 3:
        raise ValueError(f"Only mono-channel or stereo-channel audio is supported for input to {self}")

    is_batched = is_batched_numpy or (
        isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
    )

    if is_batched:
        raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
    elif not is_batched and not isinstance(raw_speech, np.ndarray):
        raw_speech = np.asarray(raw_speech, dtype=np.float32)
    elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
        raw_speech = raw_speech.astype(np.float32)

    # always return batch
    if not is_batched:
        raw_speech = [raw_speech]

    # extract fbank features
    features = [self._extract_fbank_features(waveform) for waveform in raw_speech]

    if do_normalize_per_mel_bins:
        # torch defaults to ddof=1, and numpy defaults to ddof=0
        features = [
            (x - np.expand_dims(x.mean(0), 0)) / np.sqrt(np.expand_dims(x.var(0, ddof=1), 0) + 1e-7)
            for x in features
        ]

    # convert into correct format for padding
    encoded_inputs = BatchFeature({"input_features": features})

    padded_inputs = self.pad(
        encoded_inputs,
        padding=padding,
        max_length=max_length,
        truncation=truncation,
        pad_to_multiple_of=pad_to_multiple_of,
        return_attention_mask=return_attention_mask,
        return_tensors="np",
    )

    # SeamlessM4T needs to process extracted features
    input_features = padded_inputs.get("input_features")
    attention_mask = padded_inputs.get("attention_mask")

    batch_size, num_frames, num_channels = input_features.shape

    remainder = num_frames % self.stride
    if remainder != 0:
        input_features = input_features[:, :num_frames, :]
        attention_mask = attention_mask[:, :num_frames]

    input_features = np.reshape(
        input_features, (batch_size, num_frames // self.stride, num_channels * self.stride)
    )

    indices = np.arange(0, num_frames)
    attention_mask = attention_mask[:, indices % self.stride == 1]

    padded_inputs["input_features"] = input_features
    padded_inputs["attention_mask"] = attention_mask

    if return_tensors is not None:
        padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

    return padded_inputs

mindnlp.transformers.models.seamless_m4t.feature_extraction_seamless_m4t.SeamlessM4TFeatureExtractor.__init__(feature_size=80, sampling_rate=16000, num_mel_bins=80, padding_value=0.0, stride=2, **kwargs)

Initializes an instance of the SeamlessM4TFeatureExtractor class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: SeamlessM4TFeatureExtractor
feature_size	The size of the extracted feature. Defaults to 80. TYPE: int DEFAULT: 80
sampling_rate	The sampling rate of the audio. Defaults to 16000. TYPE: int DEFAULT: 16000
num_mel_bins	The number of mel bins for mel-frequency cepstral coefficients (MFCC). Defaults to 80. TYPE: int DEFAULT: 80
padding_value	The value used for padding. Defaults to 0.0. TYPE: float DEFAULT: 0.0
stride	The stride for feature extraction. Defaults to 2. TYPE: int DEFAULT: 2

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\feature_extraction_seamless_m4t.py

def __init__(
    self,
    feature_size=80,
    sampling_rate=16000,
    num_mel_bins=80,
    padding_value=0.0,
    stride=2,
    **kwargs,
):
    """
    Initializes an instance of the SeamlessM4TFeatureExtractor class.

    Args:
        self (SeamlessM4TFeatureExtractor): The instance of the class.
        feature_size (int, optional): The size of the extracted feature. Defaults to 80.
        sampling_rate (int, optional): The sampling rate of the audio. Defaults to 16000.
        num_mel_bins (int, optional): The number of mel bins for mel-frequency cepstral coefficients (MFCC).
            Defaults to 80.
        padding_value (float, optional): The value used for padding. Defaults to 0.0.
        stride (int, optional): The stride for feature extraction. Defaults to 2.

    Returns:
        None.

    Raises:
        None.
    """
    self.num_mel_bins = num_mel_bins
    self.return_attention_mask = True
    self.stride = stride

    mel_filters = mel_filter_bank(
        num_frequency_bins=256,
        num_mel_filters=self.num_mel_bins,
        min_frequency=20,
        max_frequency=sampling_rate // 2,
        sampling_rate=sampling_rate,
        norm=None,
        mel_scale="kaldi",
        triangularize_in_mel_space=True,
    )

    self.mel_filters = np.pad(mel_filters, ((0, 1), (0, 0)))
    self.window = window_function(400, "povey", periodic=False)

    super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)

mindnlp.transformers.models.seamless_m4t.feature_extraction_seamless_m4t.SeamlessM4TFeatureExtractor.zero_mean_unit_var_norm(input_values, attention_mask, padding_value=0.0) staticmethod

Every array in the list is normalized to have zero mean and unit variance

Source code in mindnlp\transformers\models\seamless_m4t\feature_extraction_seamless_m4t.py

@staticmethod
# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
def zero_mean_unit_var_norm(
    input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
) -> List[np.ndarray]:
    """
    Every array in the list is normalized to have zero mean and unit variance
    """
    if attention_mask is not None:
        attention_mask = np.array(attention_mask, np.int32)
        normed_input_values = []

        for vector, length in zip(input_values, attention_mask.sum(-1)):
            normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
            if length < normed_slice.shape[0]:
                normed_slice[length:] = padding_value

            normed_input_values.append(normed_slice)
    else:
        normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

    return normed_input_values

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t

Audio/Text processor class for SeamlessM4T

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor

Bases: ProcessorMixin

Constructs a SeamlessM4T processor which wraps a SeamlessM4T feature extractor and a SeamlessM4T tokenizer into a single processor.

[SeamlessM4TProcessor] offers all the functionalities of [SeamlessM4TFeatureExtractor] and [SeamlessM4TTokenizerFast]. See the [~SeamlessM4TProcessor.__call__] and [~SeamlessM4TProcessor.decode] for more information.

PARAMETER	DESCRIPTION
feature_extractor	The audio processor is a required input. TYPE: [`SeamlessM4TFeatureExtractor`]
tokenizer	The tokenizer is a required input. TYPE: [`SeamlessM4TTokenizerFast`]

Source code in mindnlp\transformers\models\seamless_m4t\processing_seamless_m4t.py

class SeamlessM4TProcessor(ProcessorMixin):
    r"""
    Constructs a SeamlessM4T processor which wraps a SeamlessM4T feature extractor and a SeamlessM4T tokenizer into a
    single processor.

    [`SeamlessM4TProcessor`] offers all the functionalities of [`SeamlessM4TFeatureExtractor`] and
    [`SeamlessM4TTokenizerFast`]. See the [`~SeamlessM4TProcessor.__call__`] and [`~SeamlessM4TProcessor.decode`] for
    more information.

    Args:
        feature_extractor ([`SeamlessM4TFeatureExtractor`]):
            The audio processor is a required input.
        tokenizer ([`SeamlessM4TTokenizerFast`]):
            The tokenizer is a required input.
    """
    feature_extractor_class = "SeamlessM4TFeatureExtractor"
    tokenizer_class = ("SeamlessM4TTokenizer", "SeamlessM4TTokenizerFast")

    def __init__(self, feature_extractor, tokenizer):
        """
        Initializes a SeamlessM4TProcessor instance.

        Args:
            self (SeamlessM4TProcessor): The instance of the SeamlessM4TProcessor class.
            feature_extractor (object): The feature extractor object used for processing.
            tokenizer (object): The tokenizer object used for processing.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(feature_extractor, tokenizer)

    def __call__(self, text=None, audios=None, src_lang=None, tgt_lang=None, **kwargs):
        """
        Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
        and `kwargs` arguments to SeamlessM4TTokenizerFast's [`~SeamlessM4TTokenizerFast.__call__`] if `text` is not
        `None` to encode the text. To prepare the audio(s), this method forwards the `audios` and `kwrags` arguments to
        SeamlessM4TFeatureExtractor's [`~SeamlessM4TFeatureExtractor.__call__`] if `audios` is not `None`. Please refer
        to the doctsring of the above two methods for more information.

        Args:
            text (`str`, `List[str]`, `List[List[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            audios (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
                The audio or batch of audios to be prepared. Each audio can be NumPy array or PyTorch tensor. In case
                of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels,
                and T the sample length of the audio.
            src_lang (`str`, *optional*):
                The language code of the input texts/audios. If not specified, the last `src_lang` specified will be
                used.
            tgt_lang (`str`, *optional*):
                The code of the target language. If not specified, the last `tgt_lang` specified will be used.
            kwargs (*optional*):
                Remaining dictionary of keyword arguments that will be passed to the feature extractor and/or the
                tokenizer.

        Returns:
            [`BatchEncoding`]:
                A [`BatchEncoding`] with the following fields:

                - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
                - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
                `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
                `None`).
                - **input_features** -- Audio input features to be fed to a model. Returned when `audios` is not `None`.
        """
        sampling_rate = kwargs.pop("sampling_rate", None)

        if text is None and audios is None:
            raise ValueError("You have to specify either text or audios. Both cannot be none.")
        if text is not None and audios is not None:
            raise ValueError(
                "Text and audios are mututally exclusive when passed to `SeamlessM4T`. Specify one or another."
            )
        if text is not None:
            if tgt_lang is not None:
                self.tokenizer.tgt_lang = tgt_lang
            if src_lang is not None:
                self.tokenizer.src_lang = src_lang
            encoding = self.tokenizer(text, **kwargs)
        else:
            encoding = self.feature_extractor(audios, sampling_rate=sampling_rate, **kwargs)
        return encoding

    def batch_decode(self, *args, **kwargs):
        """
        This method forwards all its arguments to SeamlessM4TTokenizerFast's [`~PreTrainedTokenizer.batch_decode`].
        Please refer to the docstring of this method for more information.
        """
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        """
        This method forwards all its arguments to SeamlessM4TTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please
        refer to the docstring of this method for more information.
        """
        return self.tokenizer.decode(*args, **kwargs)

    @property
    def model_input_names(self):
        """
        Returns a list of unique model input names required by the SeamlessM4TProcessor.

        Args:
            self (SeamlessM4TProcessor): An instance of the SeamlessM4TProcessor class.

        Returns:
            None

        Raises:
            None

        This method retrieves the model input names from the tokenizer and feature extractor used by the
        SeamlessM4TProcessor. It then combines these names into a single list and removes any duplicates,
        returning the final list of model input names.
        """
        tokenizer_input_names = self.tokenizer.model_input_names
        feature_extractor_input_names = self.feature_extractor.model_input_names
        return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names))

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor.model_input_names property

Returns a list of unique model input names required by the SeamlessM4TProcessor.

PARAMETER	DESCRIPTION
self	An instance of the SeamlessM4TProcessor class. TYPE: SeamlessM4TProcessor

RETURNS	DESCRIPTION
	None

This method retrieves the model input names from the tokenizer and feature extractor used by the SeamlessM4TProcessor. It then combines these names into a single list and removes any duplicates, returning the final list of model input names.

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor.__call__(text=None, audios=None, src_lang=None, tgt_lang=None, **kwargs)

Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the text and kwargs arguments to SeamlessM4TTokenizerFast's [~SeamlessM4TTokenizerFast.__call__] if text is not None to encode the text. To prepare the audio(s), this method forwards the audios and kwrags arguments to SeamlessM4TFeatureExtractor's [~SeamlessM4TFeatureExtractor.__call__] if audios is not None. Please refer to the doctsring of the above two methods for more information.

PARAMETER	DESCRIPTION
text	The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set is_split_into_words=True (to lift the ambiguity with a batch of sequences). TYPE: `str`, `List[str]`, `List[List[str]]` DEFAULT: None
audios	The audio or batch of audios to be prepared. Each audio can be NumPy array or PyTorch tensor. In case of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels, and T the sample length of the audio. TYPE: `np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]` DEFAULT: None
src_lang	The language code of the input texts/audios. If not specified, the last src_lang specified will be used. TYPE: `str`, *optional* DEFAULT: None
tgt_lang	The code of the target language. If not specified, the last tgt_lang specified will be used. TYPE: `str`, *optional* DEFAULT: None
kwargs	Remaining dictionary of keyword arguments that will be passed to the feature extractor and/or the tokenizer. TYPE: *optional* DEFAULT: {}

RETURNS

DESCRIPTION

[BatchEncoding]: A [BatchEncoding] with the following fields: input_ids -- List of token ids to be fed to a model. Returned when text is not None. attention_mask -- List of indices specifying which tokens should be attended to by the model (when return_attention_mask=True or if "attention_mask" is in self.model_input_names and if text is not None). input_features -- Audio input features to be fed to a model. Returned when audios is not None.

Source code in mindnlp\transformers\models\seamless_m4t\processing_seamless_m4t.py

def __call__(self, text=None, audios=None, src_lang=None, tgt_lang=None, **kwargs):
    """
    Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
    and `kwargs` arguments to SeamlessM4TTokenizerFast's [`~SeamlessM4TTokenizerFast.__call__`] if `text` is not
    `None` to encode the text. To prepare the audio(s), this method forwards the `audios` and `kwrags` arguments to
    SeamlessM4TFeatureExtractor's [`~SeamlessM4TFeatureExtractor.__call__`] if `audios` is not `None`. Please refer
    to the doctsring of the above two methods for more information.

    Args:
        text (`str`, `List[str]`, `List[List[str]]`):
            The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
            (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
            `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
        audios (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
            The audio or batch of audios to be prepared. Each audio can be NumPy array or PyTorch tensor. In case
            of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels,
            and T the sample length of the audio.
        src_lang (`str`, *optional*):
            The language code of the input texts/audios. If not specified, the last `src_lang` specified will be
            used.
        tgt_lang (`str`, *optional*):
            The code of the target language. If not specified, the last `tgt_lang` specified will be used.
        kwargs (*optional*):
            Remaining dictionary of keyword arguments that will be passed to the feature extractor and/or the
            tokenizer.

    Returns:
        [`BatchEncoding`]:
            A [`BatchEncoding`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
            `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
            `None`).
            - **input_features** -- Audio input features to be fed to a model. Returned when `audios` is not `None`.
    """
    sampling_rate = kwargs.pop("sampling_rate", None)

    if text is None and audios is None:
        raise ValueError("You have to specify either text or audios. Both cannot be none.")
    if text is not None and audios is not None:
        raise ValueError(
            "Text and audios are mututally exclusive when passed to `SeamlessM4T`. Specify one or another."
        )
    if text is not None:
        if tgt_lang is not None:
            self.tokenizer.tgt_lang = tgt_lang
        if src_lang is not None:
            self.tokenizer.src_lang = src_lang
        encoding = self.tokenizer(text, **kwargs)
    else:
        encoding = self.feature_extractor(audios, sampling_rate=sampling_rate, **kwargs)
    return encoding

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor.__init__(feature_extractor, tokenizer)

Initializes a SeamlessM4TProcessor instance.

PARAMETER	DESCRIPTION
self	The instance of the SeamlessM4TProcessor class. TYPE: SeamlessM4TProcessor
feature_extractor	The feature extractor object used for processing. TYPE: object
tokenizer	The tokenizer object used for processing. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp\transformers\models\seamless_m4t\processing_seamless_m4t.py

def __init__(self, feature_extractor, tokenizer):
    """
    Initializes a SeamlessM4TProcessor instance.

    Args:
        self (SeamlessM4TProcessor): The instance of the SeamlessM4TProcessor class.
        feature_extractor (object): The feature extractor object used for processing.
        tokenizer (object): The tokenizer object used for processing.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(feature_extractor, tokenizer)

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor.batch_decode(*args, **kwargs)

This method forwards all its arguments to SeamlessM4TTokenizerFast's [~PreTrainedTokenizer.batch_decode]. Please refer to the docstring of this method for more information.

Source code in mindnlp\transformers\models\seamless_m4t\processing_seamless_m4t.py

def batch_decode(self, *args, **kwargs):
    """
    This method forwards all its arguments to SeamlessM4TTokenizerFast's [`~PreTrainedTokenizer.batch_decode`].
    Please refer to the docstring of this method for more information.
    """
    return self.tokenizer.batch_decode(*args, **kwargs)

mindnlp.transformers.models.seamless_m4t.processing_seamless_m4t.SeamlessM4TProcessor.decode(*args, **kwargs)

This method forwards all its arguments to SeamlessM4TTokenizerFast's [~PreTrainedTokenizer.decode]. Please refer to the docstring of this method for more information.

Source code in mindnlp\transformers\models\seamless_m4t\processing_seamless_m4t.py

def decode(self, *args, **kwargs):
    """
    This method forwards all its arguments to SeamlessM4TTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please
    refer to the docstring of this method for more information.
    """
    return self.tokenizer.decode(*args, **kwargs)