跳转至

ernie

mindnlp.transformers.models.ernie.modeling_ernie

MindSpore ERNIE model.

mindnlp.transformers.models.ernie.modeling_ernie.ErnieAttention

Bases: Module

This class represents the ErnieAttention module, which is a part of the ERNIE (Enhanced Representation through kNowledge Integration) model. The ErnieAttention module is used for self-attention mechanism and output processing. It includes methods for head pruning and attention forwardion. This class inherits from nn.Module and is designed to be used within the ERNIE model architecture for natural language processing tasks.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
class ErnieAttention(nn.Module):

    '''
    This class represents the ErnieAttention module, which is a part of the ERNIE
    (Enhanced Representation through kNowledge Integration) model.
    The ErnieAttention module is used for self-attention mechanism and output processing.
    It includes methods for head pruning and attention forwardion.
    This class inherits from nn.Module and is designed to be used within the ERNIE model architecture for
    natural language processing tasks.
    '''
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes an instance of the ErnieAttention class.

        Args:
            self (object): The instance of the class.
            config (object): The configuration object containing the model's settings and hyperparameters.
            position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.self = ErnieSelfAttention(config, position_embedding_type=position_embedding_type)
        self.output = ErnieSelfOutput(config)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        """
        This method 'prune_heads' is defined within the class 'ErnieAttention' and is responsible for
        pruning the attention heads based on the provided 'heads' parameter.

        Args:
            self (ErnieAttention): The instance of the ErnieAttention class.
                This parameter represents the instance of the ErnieAttention class which contains the attention heads to be pruned.

            heads (list): A list of integers representing the indices of attention heads to be pruned.
                This parameter specifies the indices of the attention heads that need to be pruned from the model.

        Returns:
            None: This method does not return any value.
                It operates by modifying the attributes of the ErnieAttention instance in-place.

        Raises:
            None.
        """
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
        )

        # Prune linear layers
        self.self.query = prune_linear_layer(self.self.query, index)
        self.self.key = prune_linear_layer(self.self.key, index)
        self.self.value = prune_linear_layer(self.self.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        This method forwards the attention mechanism for the Ernie model.

        Args:
            self (ErnieAttention): The instance of the ErnieAttention class.
            hidden_states (mindspore.Tensor): The input hidden states for the attention mechanism.
            attention_mask (Optional[mindspore.Tensor]): An optional mask tensor for the attention scores.
                Defaults to None.
            head_mask (Optional[mindspore.Tensor]): An optional mask tensor for controlling the attention heads.
                Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                An optional tensor containing the hidden states of the encoder. Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                An optional mask tensor for the encoder attention scores. Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                An optional tuple containing the past key and value tensors. Defaults to None.
            output_attentions (Optional[bool]): A flag indicating whether to output attentions. Defaults to False.

        Returns:
            Tuple[mindspore.Tensor]:
                A tuple containing the attention output tensor and any additional outputs from the attention mechanism.

        Raises:
            None
        """
        self_outputs = self.self(
            hidden_states,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.ernie.modeling_ernie.ErnieAttention.__init__(config, position_embedding_type=None)

Initializes an instance of the ErnieAttention class.

PARAMETER DESCRIPTION
self

The instance of the class.

TYPE: object

config

The configuration object containing the model's settings and hyperparameters.

TYPE: object

position_embedding_type

The type of position embedding to be used. Defaults to None.

TYPE: str DEFAULT: None

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
def __init__(self, config, position_embedding_type=None):
    """
    Initializes an instance of the ErnieAttention class.

    Args:
        self (object): The instance of the class.
        config (object): The configuration object containing the model's settings and hyperparameters.
        position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.self = ErnieSelfAttention(config, position_embedding_type=position_embedding_type)
    self.output = ErnieSelfOutput(config)
    self.pruned_heads = set()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards the attention mechanism for the Ernie model.

PARAMETER DESCRIPTION
self

The instance of the ErnieAttention class.

TYPE: ErnieAttention

hidden_states

The input hidden states for the attention mechanism.

TYPE: Tensor

attention_mask

An optional mask tensor for the attention scores. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

An optional mask tensor for controlling the attention heads. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

An optional tensor containing the hidden states of the encoder. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

An optional mask tensor for the encoder attention scores. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

An optional tuple containing the past key and value tensors. Defaults to None.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

A flag indicating whether to output attentions. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: A tuple containing the attention output tensor and any additional outputs from the attention mechanism.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    This method forwards the attention mechanism for the Ernie model.

    Args:
        self (ErnieAttention): The instance of the ErnieAttention class.
        hidden_states (mindspore.Tensor): The input hidden states for the attention mechanism.
        attention_mask (Optional[mindspore.Tensor]): An optional mask tensor for the attention scores.
            Defaults to None.
        head_mask (Optional[mindspore.Tensor]): An optional mask tensor for controlling the attention heads.
            Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            An optional tensor containing the hidden states of the encoder. Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            An optional mask tensor for the encoder attention scores. Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            An optional tuple containing the past key and value tensors. Defaults to None.
        output_attentions (Optional[bool]): A flag indicating whether to output attentions. Defaults to False.

    Returns:
        Tuple[mindspore.Tensor]:
            A tuple containing the attention output tensor and any additional outputs from the attention mechanism.

    Raises:
        None
    """
    self_outputs = self.self(
        hidden_states,
        attention_mask,
        head_mask,
        encoder_hidden_states,
        encoder_attention_mask,
        past_key_value,
        output_attentions,
    )
    attention_output = self.output(self_outputs[0], hidden_states)
    outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.ernie.modeling_ernie.ErnieAttention.prune_heads(heads)

This method 'prune_heads' is defined within the class 'ErnieAttention' and is responsible for pruning the attention heads based on the provided 'heads' parameter.

PARAMETER DESCRIPTION
self

The instance of the ErnieAttention class. This parameter represents the instance of the ErnieAttention class which contains the attention heads to be pruned.

TYPE: ErnieAttention

heads

A list of integers representing the indices of attention heads to be pruned. This parameter specifies the indices of the attention heads that need to be pruned from the model.

TYPE: list

RETURNS DESCRIPTION
None

This method does not return any value. It operates by modifying the attributes of the ErnieAttention instance in-place.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
def prune_heads(self, heads):
    """
    This method 'prune_heads' is defined within the class 'ErnieAttention' and is responsible for
    pruning the attention heads based on the provided 'heads' parameter.

    Args:
        self (ErnieAttention): The instance of the ErnieAttention class.
            This parameter represents the instance of the ErnieAttention class which contains the attention heads to be pruned.

        heads (list): A list of integers representing the indices of attention heads to be pruned.
            This parameter specifies the indices of the attention heads that need to be pruned from the model.

    Returns:
        None: This method does not return any value.
            It operates by modifying the attributes of the ErnieAttention instance in-place.

    Raises:
        None.
    """
    if len(heads) == 0:
        return
    heads, index = find_pruneable_heads_and_indices(
        heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
    )

    # Prune linear layers
    self.self.query = prune_linear_layer(self.self.query, index)
    self.self.key = prune_linear_layer(self.self.key, index)
    self.self.value = prune_linear_layer(self.self.value, index)
    self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

    # Update hyper params and store pruned heads
    self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
    self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
    self.pruned_heads = self.pruned_heads.union(heads)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEmbeddings

Bases: Module

Construct the embeddings from word, position and token_type embeddings.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
class ErnieEmbeddings(nn.Module):
    """Construct the embeddings from word, position and token_type embeddings."""
    def __init__(self, config):
        """
        Initializes an instance of the ErnieEmbeddings class.

        Args:
            self: The instance of the ErnieEmbeddings class.
            config: An object containing configuration parameters for the ErnieEmbeddings class.
                The config object should have the following attributes:

                - vocab_size (int): The size of the vocabulary.
                - hidden_size (int): The size of the hidden layers.
                - pad_token_id (int): The ID of the padding token.
                - max_position_embeddings (int): The maximum number of position embeddings.
                - type_vocab_size (int): The size of the token type vocabulary.
                - use_task_id (bool): Whether to use task IDs.
                - task_type_vocab_size (int): The size of the task type vocabulary.
                - layer_norm_eps (float): The epsilon value for layer normalization.
                - hidden_dropout_prob (float): The dropout probability for hidden layers.
                - position_embedding_type (str): The type of position embedding to use. Default is 'absolute'.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
        self.use_task_id = config.use_task_id
        if config.use_task_id:
            self.task_type_embeddings = nn.Embedding(config.task_type_vocab_size, config.hidden_size)

        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
        # any TensorFlow checkpoint file
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
        self.register_buffer(
            "position_ids", ops.arange(config.max_position_embeddings).broadcast_to((1, -1)), persistent=False
        )
        self.register_buffer(
            "token_type_ids", ops.zeros(self.position_ids.shape, dtype=mindspore.int64), persistent=False
        )

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        past_key_values_length: int = 0,
    ) -> mindspore.Tensor:
        """
        Constructs the embeddings for the ERNIE model.

        Args:
            self (ErnieEmbeddings): The instance of the ErnieEmbeddings class.
            input_ids (Optional[mindspore.Tensor]): The input tensor of shape [batch_size, sequence_length].
            token_type_ids (Optional[mindspore.Tensor]): The token type tensor of shape [batch_size, sequence_length].
            task_type_ids (Optional[mindspore.Tensor]): The task type tensor of shape [batch_size, sequence_length].
            position_ids (Optional[mindspore.Tensor]): The position ids tensor of shape [batch_size, sequence_length].
            inputs_embeds (Optional[mindspore.Tensor]): The input embeddings tensor of shape [batch_size, sequence_length, embedding_size].
            past_key_values_length (int): The length of past key values.

        Returns:
            mindspore.Tensor: The embeddings tensor of shape [batch_size, sequence_length, embedding_size].

        Raises:
            None
        """
        if input_ids is not None:
            input_shape = input_ids.shape
        else:
            input_shape = inputs_embeds.shape[:-1]

        seq_length = input_shape[1]

        if position_ids is None:
            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

        # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
        # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
        # issue #5664
        if token_type_ids is None:
            if hasattr(self, "token_type_ids"):
                buffered_token_type_ids = self.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((input_shape[0], seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)

        embeddings = inputs_embeds + token_type_embeddings
        if self.position_embedding_type == "absolute":
            position_embeddings = self.position_embeddings(position_ids)
            embeddings += position_embeddings

        # add `task_type_id` for ERNIE model
        if self.use_task_id:
            if task_type_ids is None:
                task_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)
            task_type_embeddings = self.task_type_embeddings(task_type_ids)
            embeddings += task_type_embeddings

        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEmbeddings.__init__(config)

Initializes an instance of the ErnieEmbeddings class.

PARAMETER DESCRIPTION
self

The instance of the ErnieEmbeddings class.
config

An object containing configuration parameters for the ErnieEmbeddings class. The config object should have the following attributes:

  • vocab_size (int): The size of the vocabulary.
  • hidden_size (int): The size of the hidden layers.
  • pad_token_id (int): The ID of the padding token.
  • max_position_embeddings (int): The maximum number of position embeddings.
  • type_vocab_size (int): The size of the token type vocabulary.
  • use_task_id (bool): Whether to use task IDs.
  • task_type_vocab_size (int): The size of the task type vocabulary.
  • layer_norm_eps (float): The epsilon value for layer normalization.
  • hidden_dropout_prob (float): The dropout probability for hidden layers.
  • position_embedding_type (str): The type of position embedding to use. Default is 'absolute'.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def __init__(self, config):
    """
    Initializes an instance of the ErnieEmbeddings class.

    Args:
        self: The instance of the ErnieEmbeddings class.
        config: An object containing configuration parameters for the ErnieEmbeddings class.
            The config object should have the following attributes:

            - vocab_size (int): The size of the vocabulary.
            - hidden_size (int): The size of the hidden layers.
            - pad_token_id (int): The ID of the padding token.
            - max_position_embeddings (int): The maximum number of position embeddings.
            - type_vocab_size (int): The size of the token type vocabulary.
            - use_task_id (bool): Whether to use task IDs.
            - task_type_vocab_size (int): The size of the task type vocabulary.
            - layer_norm_eps (float): The epsilon value for layer normalization.
            - hidden_dropout_prob (float): The dropout probability for hidden layers.
            - position_embedding_type (str): The type of position embedding to use. Default is 'absolute'.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
    self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
    self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
    self.use_task_id = config.use_task_id
    if config.use_task_id:
        self.task_type_embeddings = nn.Embedding(config.task_type_vocab_size, config.hidden_size)

    # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
    # any TensorFlow checkpoint file
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
    # position_ids (1, len position emb) is contiguous in memory and exported when serialized
    self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
    self.register_buffer(
        "position_ids", ops.arange(config.max_position_embeddings).broadcast_to((1, -1)), persistent=False
    )
    self.register_buffer(
        "token_type_ids", ops.zeros(self.position_ids.shape, dtype=mindspore.int64), persistent=False
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEmbeddings.forward(input_ids=None, token_type_ids=None, task_type_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0)

Constructs the embeddings for the ERNIE model.

PARAMETER DESCRIPTION
self

The instance of the ErnieEmbeddings class.

TYPE: ErnieEmbeddings

input_ids

The input tensor of shape [batch_size, sequence_length].

TYPE: Optional[Tensor] DEFAULT: None

token_type_ids

The token type tensor of shape [batch_size, sequence_length].

TYPE: Optional[Tensor] DEFAULT: None

task_type_ids

The task type tensor of shape [batch_size, sequence_length].

TYPE: Optional[Tensor] DEFAULT: None

position_ids

The position ids tensor of shape [batch_size, sequence_length].

TYPE: Optional[Tensor] DEFAULT: None

inputs_embeds

The input embeddings tensor of shape [batch_size, sequence_length, embedding_size].

TYPE: Optional[Tensor] DEFAULT: None

past_key_values_length

The length of past key values.

TYPE: int DEFAULT: 0

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The embeddings tensor of shape [batch_size, sequence_length, embedding_size].
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    past_key_values_length: int = 0,
) -> mindspore.Tensor:
    """
    Constructs the embeddings for the ERNIE model.

    Args:
        self (ErnieEmbeddings): The instance of the ErnieEmbeddings class.
        input_ids (Optional[mindspore.Tensor]): The input tensor of shape [batch_size, sequence_length].
        token_type_ids (Optional[mindspore.Tensor]): The token type tensor of shape [batch_size, sequence_length].
        task_type_ids (Optional[mindspore.Tensor]): The task type tensor of shape [batch_size, sequence_length].
        position_ids (Optional[mindspore.Tensor]): The position ids tensor of shape [batch_size, sequence_length].
        inputs_embeds (Optional[mindspore.Tensor]): The input embeddings tensor of shape [batch_size, sequence_length, embedding_size].
        past_key_values_length (int): The length of past key values.

    Returns:
        mindspore.Tensor: The embeddings tensor of shape [batch_size, sequence_length, embedding_size].

    Raises:
        None
    """
    if input_ids is not None:
        input_shape = input_ids.shape
    else:
        input_shape = inputs_embeds.shape[:-1]

    seq_length = input_shape[1]

    if position_ids is None:
        position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

    # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
    # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
    # issue #5664
    if token_type_ids is None:
        if hasattr(self, "token_type_ids"):
            buffered_token_type_ids = self.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((input_shape[0], seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

    if inputs_embeds is None:
        inputs_embeds = self.word_embeddings(input_ids)
    token_type_embeddings = self.token_type_embeddings(token_type_ids)

    embeddings = inputs_embeds + token_type_embeddings
    if self.position_embedding_type == "absolute":
        position_embeddings = self.position_embeddings(position_ids)
        embeddings += position_embeddings

    # add `task_type_id` for ERNIE model
    if self.use_task_id:
        if task_type_ids is None:
            task_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)
        task_type_embeddings = self.task_type_embeddings(task_type_ids)
        embeddings += task_type_embeddings

    embeddings = self.LayerNorm(embeddings)
    embeddings = self.dropout(embeddings)
    return embeddings

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEncoder

Bases: Module

The ErnieEncoder class represents a multi-layer Ernie (Enhanced Representation through kNowledge Integration) encoder module for processing sequential inputs. It inherits from the nn.Module class.

ATTRIBUTE DESCRIPTION
config

The configuration settings for the ErnieEncoder.
layer

A list of ErnieLayer instances representing the individual layers of the encoder.
gradient_checkpointing

A boolean indicating whether gradient checkpointing is enabled.
METHOD DESCRIPTION
__init__

Initializes the ErnieEncoder with the provided configuration.
forward

Constructs the ErnieEncoder module with the given inputs and returns the output either as a tuple of tensors or as a BaseModelOutputWithPastAndCrossAttentions object.
Notes
  • The forward method supports various optional input parameters and returns different types of outputs based on the provided arguments.
  • The class supports gradient checkpointing when enabled during training.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
class ErnieEncoder(nn.Module):

    """
    The ErnieEncoder class represents a multi-layer Ernie (Enhanced Representation through kNowledge Integration)
    encoder module for processing sequential inputs. It inherits from the nn.Module class.

    Attributes:
        config: The configuration settings for the ErnieEncoder.
        layer: A list of ErnieLayer instances representing the individual layers of the encoder.
        gradient_checkpointing: A boolean indicating whether gradient checkpointing is enabled.

    Methods:
        __init__: Initializes the ErnieEncoder with the provided configuration.
        forward:
            Constructs the ErnieEncoder module with the given inputs and returns the output either as a tuple of tensors
            or as a BaseModelOutputWithPastAndCrossAttentions object.

    Notes:
        - The forward method supports various optional input parameters and returns different types of outputs based
        on the provided arguments.
        - The class supports gradient checkpointing when enabled during training.
    """
    def __init__(self, config):
        """
        Initialize the ErnieEncoder class.

        Args:
            self (ErnieEncoder): The instance of the ErnieEncoder class.
            config (dict): A dictionary containing configuration parameters for the ErnieEncoder.
                It should include the following keys:

                - num_hidden_layers (int): The number of hidden layers in the ErnieEncoder.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList([ErnieLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = False,
        output_hidden_states: Optional[bool] = False,
        return_dict: Optional[bool] = True,
    ) -> Union[Tuple[mindspore.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
        '''
        Constructs the ErnieEncoder.

        Args:
            self (ErnieEncoder): The instance of the ErnieEncoder class.
            hidden_states (mindspore.Tensor): The input hidden states of the encoder.
            attention_mask (Optional[mindspore.Tensor]): The attention mask tensor. Defaults to None.
            head_mask (Optional[mindspore.Tensor]): The head mask tensor. Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder. Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]): The attention mask tensor for the encoder.
                Defaults to None.
            past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): The past key values. Defaults to None.
            use_cache (Optional[bool]): Whether to use cache. Defaults to None.
            output_attentions (Optional[bool]): Whether to output attentions. Defaults to False.
            output_hidden_states (Optional[bool]): Whether to output hidden states. Defaults to False.
            return_dict (Optional[bool]): Whether to return a dictionary. Defaults to True.

        Returns:
            Union[Tuple[mindspore.Tensor], BaseModelOutputWithPastAndCrossAttentions]: The output of the ErnieEncoder.

        Raises:
            None.
        '''
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None
        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                )
                use_cache = False

        next_decoder_cache = () if use_cache else None
        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None
            past_key_value = past_key_values[i] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    layer_module.__call__,
                    hidden_states,
                    attention_mask,
                    layer_head_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    past_key_value,
                    output_attentions,
                )
            else:
                layer_outputs = layer_module(
                    hidden_states,
                    attention_mask,
                    layer_head_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    past_key_value,
                    output_attentions,
                )

            hidden_states = layer_outputs[0]
            if use_cache:
                next_decoder_cache += (layer_outputs[-1],)
            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)
                if self.config.add_cross_attention:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    next_decoder_cache,
                    all_hidden_states,
                    all_self_attentions,
                    all_cross_attentions,
                ]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_decoder_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEncoder.__init__(config)

Initialize the ErnieEncoder class.

PARAMETER DESCRIPTION
self

The instance of the ErnieEncoder class.

TYPE: ErnieEncoder

config

A dictionary containing configuration parameters for the ErnieEncoder. It should include the following keys:

  • num_hidden_layers (int): The number of hidden layers in the ErnieEncoder.

TYPE: dict

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
def __init__(self, config):
    """
    Initialize the ErnieEncoder class.

    Args:
        self (ErnieEncoder): The instance of the ErnieEncoder class.
        config (dict): A dictionary containing configuration parameters for the ErnieEncoder.
            It should include the following keys:

            - num_hidden_layers (int): The number of hidden layers in the ErnieEncoder.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.config = config
    self.layer = nn.ModuleList([ErnieLayer(config) for _ in range(config.num_hidden_layers)])
    self.gradient_checkpointing = False

mindnlp.transformers.models.ernie.modeling_ernie.ErnieEncoder.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=False, output_hidden_states=False, return_dict=True)

Constructs the ErnieEncoder.

PARAMETER DESCRIPTION
self

The instance of the ErnieEncoder class.

TYPE: ErnieEncoder

hidden_states

The input hidden states of the encoder.

TYPE: Tensor

attention_mask

The attention mask tensor. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

The head mask tensor. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

The hidden states of the encoder. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

The attention mask tensor for the encoder. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_values

The past key values. Defaults to None.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

use_cache

Whether to use cache. Defaults to None.

TYPE: Optional[bool] DEFAULT: None

output_attentions

Whether to output attentions. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

output_hidden_states

Whether to output hidden states. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

return_dict

Whether to return a dictionary. Defaults to True.

TYPE: Optional[bool] DEFAULT: True

RETURNS DESCRIPTION
Union[Tuple[Tensor], BaseModelOutputWithPastAndCrossAttentions]

Union[Tuple[mindspore.Tensor], BaseModelOutputWithPastAndCrossAttentions]: The output of the ErnieEncoder.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = False,
    output_hidden_states: Optional[bool] = False,
    return_dict: Optional[bool] = True,
) -> Union[Tuple[mindspore.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
    '''
    Constructs the ErnieEncoder.

    Args:
        self (ErnieEncoder): The instance of the ErnieEncoder class.
        hidden_states (mindspore.Tensor): The input hidden states of the encoder.
        attention_mask (Optional[mindspore.Tensor]): The attention mask tensor. Defaults to None.
        head_mask (Optional[mindspore.Tensor]): The head mask tensor. Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder. Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]): The attention mask tensor for the encoder.
            Defaults to None.
        past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): The past key values. Defaults to None.
        use_cache (Optional[bool]): Whether to use cache. Defaults to None.
        output_attentions (Optional[bool]): Whether to output attentions. Defaults to False.
        output_hidden_states (Optional[bool]): Whether to output hidden states. Defaults to False.
        return_dict (Optional[bool]): Whether to return a dictionary. Defaults to True.

    Returns:
        Union[Tuple[mindspore.Tensor], BaseModelOutputWithPastAndCrossAttentions]: The output of the ErnieEncoder.

    Raises:
        None.
    '''
    all_hidden_states = () if output_hidden_states else None
    all_self_attentions = () if output_attentions else None
    all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
            )
            use_cache = False

    next_decoder_cache = () if use_cache else None
    for i, layer_module in enumerate(self.layer):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        layer_head_mask = head_mask[i] if head_mask is not None else None
        past_key_value = past_key_values[i] if past_key_values is not None else None

        if self.gradient_checkpointing and self.training:
            layer_outputs = self._gradient_checkpointing_func(
                layer_module.__call__,
                hidden_states,
                attention_mask,
                layer_head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                past_key_value,
                output_attentions,
            )
        else:
            layer_outputs = layer_module(
                hidden_states,
                attention_mask,
                layer_head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                past_key_value,
                output_attentions,
            )

        hidden_states = layer_outputs[0]
        if use_cache:
            next_decoder_cache += (layer_outputs[-1],)
        if output_attentions:
            all_self_attentions = all_self_attentions + (layer_outputs[1],)
            if self.config.add_cross_attention:
                all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    if not return_dict:
        return tuple(
            v
            for v in [
                hidden_states,
                next_decoder_cache,
                all_hidden_states,
                all_self_attentions,
                all_cross_attentions,
            ]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_decoder_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attentions,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM

Bases: ErniePreTrainedModel

This class represents a causal language modeling model based on the ERNIE (Enhanced Representation through kNowledge Integration) architecture. It is designed for generating text predictions based on input sequences, with a focus on predicting the next word in a sequence. The model includes functionality for forwarding the model, setting and getting output embeddings, preparing inputs for text generation, and reordering cache during generation.

The class includes methods for initializing the model, forwarding the model for inference or training, setting and getting output embeddings, preparing inputs for text generation, and reordering cache during generation.

The 'forward' method forwards the model for inference or training, taking various input tensors such as input ids, attention masks, token type ids, and more. It returns the model outputs including the language modeling loss and predictions.

The 'prepare_inputs_for_generation' method prepares input tensors for text generation, including handling past key values and attention masks. It returns a dictionary containing the input ids, attention mask, past key values, and use_cache flag.

The '_reorder_cache' method reorders the past key values during generation based on the beam index used for parallel decoding.

For more detailed information on each method's parameters and return values, refer to the method docstrings within the class code.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
class ErnieForCausalLM(ErniePreTrainedModel):

    """
    This class represents a causal language modeling model based on the ERNIE
    (Enhanced Representation through kNowledge Integration) architecture.
    It is designed for generating text predictions based on input sequences, with a focus on predicting the next word
    in a sequence.
    The model includes functionality for forwarding the model, setting and getting output embeddings, preparing inputs
    for text generation, and reordering cache during generation.

    The class includes methods for initializing the model, forwarding the model for inference or training, setting
    and getting output embeddings, preparing inputs for text generation, and reordering cache during generation.

    The 'forward' method forwards the model for inference or training, taking various input tensors such as
    input ids, attention masks, token type ids, and more. It returns the model outputs including the language modeling
    loss and predictions.

    The 'prepare_inputs_for_generation' method prepares input tensors for text generation, including handling past key
    values and attention masks. It returns a dictionary containing the input ids, attention  mask, past key values,
    and use_cache flag.

    The '_reorder_cache' method reorders the past key values during generation based on the beam index used for parallel
    decoding.

    For more detailed information on each method's parameters and return values, refer to the method docstrings within
    the class code.
    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.__init__ with BertLMHeadModel->ErnieForCausalLM,Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the `ErnieForCausalLM` class.

        Args:
            self: The instance of the class.
            config (object):
                The configuration object containing various settings for the model.

                - Type: object
                - Purpose: Specifies the configuration settings for the model.
                - Restrictions: None

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)

        if not config.is_decoder:
            logger.warning("If you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`")

        self.ernie = ErnieModel(config, add_pooling_layer=False)
        self.cls = ErnieOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.get_output_embeddings
    def get_output_embeddings(self):
        """
        Retrieve the output embeddings from the ErnieForCausalLM model.

        Args:
            self (ErnieForCausalLM): The instance of the ErnieForCausalLM class.

        Returns:
            decoder: This method returns the output embeddings from the model's predictions decoder layer.

        Raises:
            None.
        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the ErnieForCausalLM model.

        Args:
            self (ErnieForCausalLM): The instance of the ErnieForCausalLM class.
            new_embeddings: The new embeddings to be set as output embeddings.
                It should be of the same shape as the existing embeddings.

        Returns:
            None.

        Raises:
            None.

        Note:
            This method updates the output embeddings of the ErnieForCausalLM model to the provided new_embeddings.
            The new_embeddings should be of the same shape as the existing embeddings.

        Example:
            ```python
            >>> model = ErnieForCausalLM()
            >>> new_embeddings = torch.Tensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]])
            >>> model.set_output_embeddings(new_embeddings)
            ```
        """
        self.cls.predictions.decoder = new_embeddings
        self.cls.predictions.bias = new_embeddings.bias

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], CausalLMOutputWithCrossAttentions]:
        r"""
        Args:
            encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
                the model is configured as a decoder.
            encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
                the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
                `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
                ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
            past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4
                tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
                Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
                `past_key_values`).
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        if labels is not None:
            use_cache = False

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        lm_loss = None
        if labels is not None:
            # we are doing next-token prediction; shift prediction scores and input ids by one
            shifted_prediction_scores = prediction_scores[:, :-1, :]
            labels = labels[:, 1:]
            lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (prediction_scores,) + outputs[2:]
            return ((lm_loss,) + output) if lm_loss is not None else output

        return CausalLMOutputWithCrossAttentions(
            loss=lm_loss,
            logits=prediction_scores,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            cross_attentions=outputs.cross_attentions,
        )

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
    ):
        """
        Prepare inputs for generation.

        Args:
            self: The instance of the class.
            input_ids (torch.Tensor): The input tensor containing the input ids.
            past_key_values (tuple, optional): The tuple containing past key values. Defaults to None.
            attention_mask (torch.Tensor, optional): The attention mask tensor. Defaults to None.
            use_cache (bool, optional): Flag indicating whether to use cache. Defaults to True.

        Returns:
            dict: A dictionary containing the prepared input_ids, attention_mask, past_key_values, and use_cache.

        Raises:
            ValueError: If input_ids shape is incompatible with past_key_values.
        """
        input_shape = input_ids.shape
        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
        if attention_mask is None:
            attention_mask = input_ids.new_ones(input_shape)

        # cut decoder_input_ids if past_key_values is used
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "past_key_values": past_key_values,
            "use_cache": use_cache,
        }

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel._reorder_cache
    def _reorder_cache(self, past_key_values, beam_idx):
        """
        This method '_reorder_cache' reorders the past states based on the provided beam indices.

        Args:
            self (ErnieForCausalLM): The instance of the ErnieForCausalLM class.
            past_key_values (tuple): A tuple containing past states for each layer.
            beam_idx (Tensor): A tensor representing the beam indices used for reordering.

        Returns:
            None: This method does not return any value but updates the 'reordered_past' variable within the method.

        Raises:
            IndexError: If the provided beam indices are out of bounds.
            TypeError: If the input types are not as expected.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM.__init__(config)

Initializes an instance of the ErnieForCausalLM class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object containing various settings for the model.

  • Type: object
  • Purpose: Specifies the configuration settings for the model.
  • Restrictions: None

TYPE: object

RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
def __init__(self, config):
    """
    Initializes an instance of the `ErnieForCausalLM` class.

    Args:
        self: The instance of the class.
        config (object):
            The configuration object containing various settings for the model.

            - Type: object
            - Purpose: Specifies the configuration settings for the model.
            - Restrictions: None

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)

    if not config.is_decoder:
        logger.warning("If you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`")

    self.ernie = ErnieModel(config, add_pooling_layer=False)
    self.cls = ErnieOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
encoder_hidden_states

Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder.

TYPE: (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None

encoder_attention_mask

Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in [0, 1]:

  • 1 for tokens that are not masked,
  • 0 for tokens that are masked.

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

labels

Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels n [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

use_cache

If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values).

TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], CausalLMOutputWithCrossAttentions]:
    r"""
    Args:
        encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
        past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4
            tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    if labels is not None:
        use_cache = False

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    lm_loss = None
    if labels is not None:
        # we are doing next-token prediction; shift prediction scores and input ids by one
        shifted_prediction_scores = prediction_scores[:, :-1, :]
        labels = labels[:, 1:]
        lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (prediction_scores,) + outputs[2:]
        return ((lm_loss,) + output) if lm_loss is not None else output

    return CausalLMOutputWithCrossAttentions(
        loss=lm_loss,
        logits=prediction_scores,
        past_key_values=outputs.past_key_values,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
        cross_attentions=outputs.cross_attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM.get_output_embeddings()

Retrieve the output embeddings from the ErnieForCausalLM model.

PARAMETER DESCRIPTION
self

The instance of the ErnieForCausalLM class.

TYPE: ErnieForCausalLM

RETURNS DESCRIPTION
decoder

This method returns the output embeddings from the model's predictions decoder layer.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
def get_output_embeddings(self):
    """
    Retrieve the output embeddings from the ErnieForCausalLM model.

    Args:
        self (ErnieForCausalLM): The instance of the ErnieForCausalLM class.

    Returns:
        decoder: This method returns the output embeddings from the model's predictions decoder layer.

    Raises:
        None.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs)

Prepare inputs for generation.

PARAMETER DESCRIPTION
self

The instance of the class.
input_ids

The input tensor containing the input ids.

TYPE: Tensor

past_key_values

The tuple containing past key values. Defaults to None.

TYPE: tuple DEFAULT: None

attention_mask

The attention mask tensor. Defaults to None.

TYPE: Tensor DEFAULT: None

use_cache

Flag indicating whether to use cache. Defaults to True.

TYPE: bool DEFAULT: True

RETURNS DESCRIPTION
dict

A dictionary containing the prepared input_ids, attention_mask, past_key_values, and use_cache.
RAISES DESCRIPTION
ValueError

If input_ids shape is incompatible with past_key_values.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
def prepare_inputs_for_generation(
    self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
):
    """
    Prepare inputs for generation.

    Args:
        self: The instance of the class.
        input_ids (torch.Tensor): The input tensor containing the input ids.
        past_key_values (tuple, optional): The tuple containing past key values. Defaults to None.
        attention_mask (torch.Tensor, optional): The attention mask tensor. Defaults to None.
        use_cache (bool, optional): Flag indicating whether to use cache. Defaults to True.

    Returns:
        dict: A dictionary containing the prepared input_ids, attention_mask, past_key_values, and use_cache.

    Raises:
        ValueError: If input_ids shape is incompatible with past_key_values.
    """
    input_shape = input_ids.shape
    # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
    if attention_mask is None:
        attention_mask = input_ids.new_ones(input_shape)

    # cut decoder_input_ids if past_key_values is used
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "past_key_values": past_key_values,
        "use_cache": use_cache,
    }

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForCausalLM.set_output_embeddings(new_embeddings)

Sets the output embeddings for the ErnieForCausalLM model.

PARAMETER DESCRIPTION
self

The instance of the ErnieForCausalLM class.

TYPE: ErnieForCausalLM

new_embeddings

The new embeddings to be set as output embeddings. It should be of the same shape as the existing embeddings.
RETURNS DESCRIPTION

None.
Note

This method updates the output embeddings of the ErnieForCausalLM model to the provided new_embeddings. The new_embeddings should be of the same shape as the existing embeddings.

Example 
>>> model = ErnieForCausalLM()
>>> new_embeddings = torch.Tensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]])
>>> model.set_output_embeddings(new_embeddings)
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the ErnieForCausalLM model.

    Args:
        self (ErnieForCausalLM): The instance of the ErnieForCausalLM class.
        new_embeddings: The new embeddings to be set as output embeddings.
            It should be of the same shape as the existing embeddings.

    Returns:
        None.

    Raises:
        None.

    Note:
        This method updates the output embeddings of the ErnieForCausalLM model to the provided new_embeddings.
        The new_embeddings should be of the same shape as the existing embeddings.

    Example:
        ```python
        >>> model = ErnieForCausalLM()
        >>> new_embeddings = torch.Tensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]])
        >>> model.set_output_embeddings(new_embeddings)
        ```
    """
    self.cls.predictions.decoder = new_embeddings
    self.cls.predictions.bias = new_embeddings.bias

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM

Bases: ErniePreTrainedModel

This class represents a model for Masked Language Modeling using the ERNIE (Enhanced Representation through kNowledge Integration) architecture. It is designed for generating predictions for masked tokens within a sequence of text.

The class inherits from ErniePreTrainedModel and implements methods for initializing the model, getting and setting output embeddings, forwarding the model for training or inference, and preparing inputs for text generation.

METHOD DESCRIPTION
__init__

Initializes the ErnieForMaskedLM model with the given configuration.
get_output_embeddings

Retrieves the output embeddings from the model.
set_output_embeddings

Sets new output embeddings for the model.
forward

Constructs the model for training or inference, computing the masked language modeling loss and prediction scores.
prepare_inputs_for_generation

Prepares inputs for text generation, including handling padding and dummy tokens.
Note

This class assumes the existence of the ErnieModel and ErnieOnlyMLMHead classes for the ERNIE architecture.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
class ErnieForMaskedLM(ErniePreTrainedModel):

    """
    This class represents a model for Masked Language Modeling using the ERNIE
    (Enhanced Representation through kNowledge Integration) architecture.
    It is designed for generating predictions for masked tokens within a sequence of text.

    The class inherits from ErniePreTrainedModel and implements methods for initializing the model, getting and setting
    output embeddings, forwarding the model for training or inference, and preparing inputs for text generation.

    Methods:
        __init__: Initializes the ErnieForMaskedLM model with the given configuration.
        get_output_embeddings: Retrieves the output embeddings from the model.
        set_output_embeddings: Sets new output embeddings for the model.
        forward: Constructs the model for training or inference, computing the masked language modeling loss
            and prediction scores.
        prepare_inputs_for_generation: Prepares inputs for text generation, including handling padding and dummy tokens.

    Note:
        This class assumes the existence of the ErnieModel and ErnieOnlyMLMHead classes for the ERNIE architecture.
    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the 'ErnieForMaskedLM' class.

        Args:
            self: The current object instance.
            config: An instance of the 'Config' class containing the configuration settings for the model.

        Returns:
            None

        Raises:
            None

        Description:
            This method initializes the 'ErnieForMaskedLM' class by setting the configuration and initializing the
            'ErnieModel' and 'ErnieOnlyMLMHead' objects.

            The 'config' parameter is an instance of the 'Config' class, which contains various configuration settings for the model.
            This method also logs a warning if the 'is_decoder' flag in the 'config' parameter is set to True,
            indicating that the model is being used as a decoder.

            The 'ErnieModel' object is initialized with the given 'config' and the 'add_pooling_layer' flag set to False.

            The 'ErnieOnlyMLMHead' object is also initialized with the given 'config'.

            Finally, the 'post_init' method is called to perform any additional initialization steps.
        """
        super().__init__(config)

        if config.is_decoder:
            logger.warning(
                "If you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for "
                "bi-directional self-attention."
            )

        self.ernie = ErnieModel(config, add_pooling_layer=False)
        self.cls = ErnieOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.get_output_embeddings
    def get_output_embeddings(self):
        """
        Retrieve the output embeddings from the ErnieForMaskedLM model.

        Args:
            self (ErnieForMaskedLM): An instance of the ErnieForMaskedLM class.
                Represents the model object that contains the output embeddings.

        Returns:
            None: This method returns the output embeddings stored in the 'decoder' of the 'predictions' object
                within the ErnieForMaskedLM model.

        Raises:
            None.
        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the ErnieForMaskedLM model.

        Args:
            self (ErnieForMaskedLM): The instance of the ErnieForMaskedLM class.
            new_embeddings (object): The new embeddings to be set for the model's output.
                It can be any object that is compatible with the existing model's output embeddings.
                The new embeddings will replace the current embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings
        self.cls.predictions.bias = new_embeddings.bias

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], MaskedLMOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
                config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
                loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        masked_lm_loss = None
        if labels is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (prediction_scores,) + outputs[2:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return MaskedLMOutput(
            loss=masked_lm_loss,
            logits=prediction_scores,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.prepare_inputs_for_generation
    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
        """
        Prepare inputs for generation.

        This method prepares input data for generation in the ErnieForMaskedLM model.

        Args:
            self: The instance of the ErnieForMaskedLM class.
            input_ids (Tensor): The input token IDs. Shape (batch_size, sequence_length).
            attention_mask (Tensor, optional): The attention mask tensor. Shape (batch_size, sequence_length).
            **model_kwargs: Additional model-specific keyword arguments.

        Returns:
            dict: A dictionary containing the prepared input_ids and attention_mask.

        Raises:
            ValueError: If the PAD token is not defined for generation.
        """
        input_shape = input_ids.shape
        effective_batch_size = input_shape[0]

        #  add a dummy token
        if self.config.pad_token_id is None:
            raise ValueError("The PAD token should be defined for generation")

        attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
        dummy_token = ops.full(
            (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64
        )
        input_ids = ops.cat([input_ids, dummy_token], dim=1)

        return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM.__init__(config)

Initializes an instance of the 'ErnieForMaskedLM' class.

PARAMETER DESCRIPTION
self

The current object instance.
config

An instance of the 'Config' class containing the configuration settings for the model.
RETURNS DESCRIPTION

None
Description

This method initializes the 'ErnieForMaskedLM' class by setting the configuration and initializing the 'ErnieModel' and 'ErnieOnlyMLMHead' objects.

The 'config' parameter is an instance of the 'Config' class, which contains various configuration settings for the model. This method also logs a warning if the 'is_decoder' flag in the 'config' parameter is set to True, indicating that the model is being used as a decoder.

The 'ErnieModel' object is initialized with the given 'config' and the 'add_pooling_layer' flag set to False.

The 'ErnieOnlyMLMHead' object is also initialized with the given 'config'.

Finally, the 'post_init' method is called to perform any additional initialization steps.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
def __init__(self, config):
    """
    Initializes an instance of the 'ErnieForMaskedLM' class.

    Args:
        self: The current object instance.
        config: An instance of the 'Config' class containing the configuration settings for the model.

    Returns:
        None

    Raises:
        None

    Description:
        This method initializes the 'ErnieForMaskedLM' class by setting the configuration and initializing the
        'ErnieModel' and 'ErnieOnlyMLMHead' objects.

        The 'config' parameter is an instance of the 'Config' class, which contains various configuration settings for the model.
        This method also logs a warning if the 'is_decoder' flag in the 'config' parameter is set to True,
        indicating that the model is being used as a decoder.

        The 'ErnieModel' object is initialized with the given 'config' and the 'add_pooling_layer' flag set to False.

        The 'ErnieOnlyMLMHead' object is also initialized with the given 'config'.

        Finally, the 'post_init' method is called to perform any additional initialization steps.
    """
    super().__init__(config)

    if config.is_decoder:
        logger.warning(
            "If you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for "
            "bi-directional self-attention."
        )

    self.ernie = ErnieModel(config, add_pooling_layer=False)
    self.cls = ErnieOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
labels

Labels for computing the masked language modeling loss. Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], MaskedLMOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    masked_lm_loss = None
    if labels is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (prediction_scores,) + outputs[2:]
        return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

    return MaskedLMOutput(
        loss=masked_lm_loss,
        logits=prediction_scores,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM.get_output_embeddings()

Retrieve the output embeddings from the ErnieForMaskedLM model.

PARAMETER DESCRIPTION
self

An instance of the ErnieForMaskedLM class. Represents the model object that contains the output embeddings.

TYPE: ErnieForMaskedLM

RETURNS DESCRIPTION
None

This method returns the output embeddings stored in the 'decoder' of the 'predictions' object within the ErnieForMaskedLM model.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
def get_output_embeddings(self):
    """
    Retrieve the output embeddings from the ErnieForMaskedLM model.

    Args:
        self (ErnieForMaskedLM): An instance of the ErnieForMaskedLM class.
            Represents the model object that contains the output embeddings.

    Returns:
        None: This method returns the output embeddings stored in the 'decoder' of the 'predictions' object
            within the ErnieForMaskedLM model.

    Raises:
        None.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM.prepare_inputs_for_generation(input_ids, attention_mask=None, **model_kwargs)

Prepare inputs for generation.

This method prepares input data for generation in the ErnieForMaskedLM model.

PARAMETER DESCRIPTION
self

The instance of the ErnieForMaskedLM class.
input_ids

The input token IDs. Shape (batch_size, sequence_length).

TYPE: Tensor

attention_mask

The attention mask tensor. Shape (batch_size, sequence_length).

TYPE: Tensor DEFAULT: None

**model_kwargs

Additional model-specific keyword arguments.

DEFAULT: {}

RETURNS DESCRIPTION
dict

A dictionary containing the prepared input_ids and attention_mask.
RAISES DESCRIPTION
ValueError

If the PAD token is not defined for generation.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
    """
    Prepare inputs for generation.

    This method prepares input data for generation in the ErnieForMaskedLM model.

    Args:
        self: The instance of the ErnieForMaskedLM class.
        input_ids (Tensor): The input token IDs. Shape (batch_size, sequence_length).
        attention_mask (Tensor, optional): The attention mask tensor. Shape (batch_size, sequence_length).
        **model_kwargs: Additional model-specific keyword arguments.

    Returns:
        dict: A dictionary containing the prepared input_ids and attention_mask.

    Raises:
        ValueError: If the PAD token is not defined for generation.
    """
    input_shape = input_ids.shape
    effective_batch_size = input_shape[0]

    #  add a dummy token
    if self.config.pad_token_id is None:
        raise ValueError("The PAD token should be defined for generation")

    attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
    dummy_token = ops.full(
        (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64
    )
    input_ids = ops.cat([input_ids, dummy_token], dim=1)

    return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMaskedLM.set_output_embeddings(new_embeddings)

Sets the output embeddings for the ErnieForMaskedLM model.

PARAMETER DESCRIPTION
self

The instance of the ErnieForMaskedLM class.

TYPE: ErnieForMaskedLM

new_embeddings

The new embeddings to be set for the model's output. It can be any object that is compatible with the existing model's output embeddings. The new embeddings will replace the current embeddings.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the ErnieForMaskedLM model.

    Args:
        self (ErnieForMaskedLM): The instance of the ErnieForMaskedLM class.
        new_embeddings (object): The new embeddings to be set for the model's output.
            It can be any object that is compatible with the existing model's output embeddings.
            The new embeddings will replace the current embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings
    self.cls.predictions.bias = new_embeddings.bias

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMultipleChoice

Bases: ErniePreTrainedModel

This class represents an Ernie model for multiple choice tasks. It inherits from the ErniePreTrainedModel class.

The ErnieForMultipleChoice class initializes an Ernie model with the given configuration. It forwards the model by passing input tensors through the Ernie model layers and applies dropout and classification layers to generate the logits for multiple choice classification.

Example 
>>> model = ErnieForMultipleChoice(config)
>>> outputs = model.forward(input_ids, attention_mask, token_type_ids, task_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
METHOD DESCRIPTION
__init__

Initializes the ErnieForMultipleChoice class with the given configuration.
forward

Constructs the Ernie model for multiple choice tasks and returns the model outputs.
RETURNS DESCRIPTION

Union[Tuple[mindspore.Tensor], MultipleChoiceModelOutput]: The model outputs, which can include the loss, logits, hidden states, and attentions.
Note

The labels argument should be provided for computing the multiple choice classification loss. Indices in labels should be in the range [0, num_choices-1], where num_choices is the size of the second dimension of the input tensors (input_ids).

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
class ErnieForMultipleChoice(ErniePreTrainedModel):

    """
    This class represents an Ernie model for multiple choice tasks. It inherits from the ErniePreTrainedModel class.

    The ErnieForMultipleChoice class initializes an Ernie model with the given configuration.
    It forwards the model by passing input tensors through the Ernie model layers and applies dropout and
    classification layers to generate the logits for multiple choice classification.

    Example:
        ```python
        >>> model = ErnieForMultipleChoice(config)
        >>> outputs = model.forward(input_ids, attention_mask, token_type_ids, task_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
        ```
    Args:
        config (ErnieConfig): The configuration for the Ernie model.

    Methods:
        __init__:
            Initializes the ErnieForMultipleChoice class with the given configuration.

        forward:
            Constructs the Ernie model for multiple choice tasks and returns the model outputs.

    Returns:
        Union[Tuple[mindspore.Tensor], MultipleChoiceModelOutput]:
            The model outputs, which can include the loss, logits, hidden states, and attentions.

    Note:
        The labels argument should be provided for computing the multiple choice classification loss.
        Indices in labels should be in the range [0, num_choices-1], where num_choices is the size of the second
        dimension of the input tensors (input_ids).
    """
    # Copied from transformers.models.bert.modeling_bert.BertForMultipleChoice.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the ErnieForMultipleChoice class.

        Args:
            self (ErnieForMultipleChoice): The instance of the ErnieForMultipleChoice class.
            config: The configuration object containing various hyperparameters and settings for the model initialization.

        Returns:
            None.

        Raises:
            TypeError: If the input parameters are not of the expected types.
            ValueError: If the configuration object is missing required attributes.
            RuntimeError: If there are issues during model initialization or post-initialization steps.
        """
        super().__init__(config)

        self.ernie = ErnieModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, 1)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], MultipleChoiceModelOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
                num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
                `input_ids` above)
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

        input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
        attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
        token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
        position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
        inputs_embeds = (
            inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
            if inputs_embeds is not None
            else None
        )

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
        reshaped_logits = logits.view(-1, num_choices)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(reshaped_logits, labels)

        if not return_dict:
            output = (reshaped_logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return MultipleChoiceModelOutput(
            loss=loss,
            logits=reshaped_logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMultipleChoice.__init__(config)

Initializes an instance of the ErnieForMultipleChoice class.

PARAMETER DESCRIPTION
self

The instance of the ErnieForMultipleChoice class.

TYPE: ErnieForMultipleChoice

config

The configuration object containing various hyperparameters and settings for the model initialization.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the input parameters are not of the expected types.
ValueError

If the configuration object is missing required attributes.
RuntimeError

If there are issues during model initialization or post-initialization steps.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
def __init__(self, config):
    """
    Initializes an instance of the ErnieForMultipleChoice class.

    Args:
        self (ErnieForMultipleChoice): The instance of the ErnieForMultipleChoice class.
        config: The configuration object containing various hyperparameters and settings for the model initialization.

    Returns:
        None.

    Raises:
        TypeError: If the input parameters are not of the expected types.
        ValueError: If the configuration object is missing required attributes.
        RuntimeError: If there are issues during model initialization or post-initialization steps.
    """
    super().__init__(config)

    self.ernie = ErnieModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, 1)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForMultipleChoice.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
labels

Labels for computing the multiple choice classification loss. Indices should be in [0, ..., num_choices-1] where num_choices is the size of the second dimension of the input tensors. (See input_ids above)

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], MultipleChoiceModelOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

    input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
    attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
    token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
    position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
    inputs_embeds = (
        inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
        if inputs_embeds is not None
        else None
    )

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)
    reshaped_logits = logits.view(-1, num_choices)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(reshaped_logits, labels)

    if not return_dict:
        output = (reshaped_logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return MultipleChoiceModelOutput(
        loss=loss,
        logits=reshaped_logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForNextSentencePrediction

Bases: ErniePreTrainedModel

ErnieForNextSentencePrediction is a class that represents a model for next sentence prediction using the ERNIE (Enhanced Representation through kNowledge IntEgration) architecture. This class inherits from the ErniePreTrainedModel class.

The ERNIE model is designed for various natural language processing tasks, including next sentence prediction. It takes input sequences and predicts whether the second sequence follows the first sequence in a given pair.

The class's code initializes an instance of the ErnieForNextSentencePrediction class with the provided configuration. It creates an ERNIE model and a next sentence prediction head. The post_init() method is called to perform additional setup after the initialization.

The forward() method forwards the model using the provided input tensors and other optional arguments. It returns the predicted next sentence relationship scores. The method also supports computing the next sequence prediction loss if labels are provided.

The labels parameter is used to compute the next sequence prediction loss. It should be a tensor of shape (batch_size,) where each value indicates the relationship between the input sequences:

  • 0 indicates sequence B is a continuation of sequence A.
  • 1 indicates sequence B is a random sequence. The method returns a tuple of the next sentence prediction loss, the next sentence relationship scores, and other optional outputs such as hidden states and attentions.
Example 
>>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
...
>>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
class ErnieForNextSentencePrediction(ErniePreTrainedModel):

    """
    ErnieForNextSentencePrediction is a class that represents a model for next sentence prediction using the ERNIE
    (Enhanced Representation through kNowledge IntEgration) architecture.
    This class inherits from the ErniePreTrainedModel class.

    The ERNIE model is designed for various natural language processing tasks, including next sentence prediction.
    It takes input sequences and predicts whether the second sequence follows the first sequence in a given pair.

    The class's code initializes an instance of the ErnieForNextSentencePrediction class with the provided configuration.
    It creates an ERNIE model and a next sentence prediction head.
    The post_init() method is called to perform additional setup after the initialization.

    The forward() method forwards the model using the provided input tensors and other optional arguments.
    It returns the predicted next sentence relationship scores. The method also supports computing the next sequence
    prediction loss if labels are provided.

    The labels parameter is used to compute the next sequence prediction loss.
    It should be a tensor of shape (batch_size,) where each value indicates the relationship between the input sequences:

    - 0 indicates sequence B is a continuation of sequence A.
    - 1 indicates sequence B is a random sequence.
    The method returns a tuple of the next sentence prediction loss, the next sentence relationship scores,
    and other optional outputs such as hidden states and attentions.

    Example:
        ```python
        >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
        ...
        >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
        >>> logits = outputs.logits
        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
        ```
    """
    # Copied from transformers.models.bert.modeling_bert.BertForNextSentencePrediction.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of ErnieForNextSentencePrediction.

        Args:
            self (ErnieForNextSentencePrediction): The instance of the ErnieForNextSentencePrediction class.
            config (dict): The configuration dictionary containing parameters for initializing the model.
                It should include necessary settings for model configuration.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.ernie = ErnieModel(config)
        self.cls = ErnieOnlyNSPHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Tuple[mindspore.Tensor], NextSentencePredictorOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
                (see `input_ids` docstring). Indices should be in `[0, 1]`:

                - 0 indicates sequence B is a continuation of sequence A,
                - 1 indicates sequence B is a random sequence.

        Returns:
            Union[Tuple[mindspore.Tensor], NextSentencePredictorOutput]

        Example:
            ```python
            >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
            ...
            ...
            >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
            >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
            ...
            >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
            >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
            >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
            ...
            >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
            >>> logits = outputs.logits
            >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
            ```
        """
        if "next_sentence_label" in kwargs:
            warnings.warn(
                "The `next_sentence_label` argument is deprecated and will be removed in a future version, use"
                " `labels` instead.",
                FutureWarning,
            )
            labels = kwargs.pop("next_sentence_label")

        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        seq_relationship_scores = self.cls(pooled_output)

        next_sentence_loss = None
        if labels is not None:
            next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

        if not return_dict:
            output = (seq_relationship_scores,) + outputs[2:]
            return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

        return NextSentencePredictorOutput(
            loss=next_sentence_loss,
            logits=seq_relationship_scores,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForNextSentencePrediction.__init__(config)

Initializes an instance of ErnieForNextSentencePrediction.

PARAMETER DESCRIPTION
self

The instance of the ErnieForNextSentencePrediction class.

TYPE: ErnieForNextSentencePrediction

config

The configuration dictionary containing parameters for initializing the model. It should include necessary settings for model configuration.

TYPE: dict

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
def __init__(self, config):
    """
    Initializes an instance of ErnieForNextSentencePrediction.

    Args:
        self (ErnieForNextSentencePrediction): The instance of the ErnieForNextSentencePrediction class.
        config (dict): The configuration dictionary containing parameters for initializing the model.
            It should include necessary settings for model configuration.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.ernie = ErnieModel(config)
    self.cls = ErnieOnlyNSPHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForNextSentencePrediction.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None, **kwargs)

PARAMETER DESCRIPTION
labels

Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see input_ids docstring). Indices should be in [0, 1]:

  • 0 indicates sequence B is a continuation of sequence A,
  • 1 indicates sequence B is a random sequence.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS DESCRIPTION
Union[Tuple[Tensor], NextSentencePredictorOutput]

Union[Tuple[mindspore.Tensor], NextSentencePredictorOutput]
Example 
>>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
...
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
...
>>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
    **kwargs,
) -> Union[Tuple[mindspore.Tensor], NextSentencePredictorOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring). Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.

    Returns:
        Union[Tuple[mindspore.Tensor], NextSentencePredictorOutput]

    Example:
        ```python
        >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
        ...
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
        ...
        >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
        >>> logits = outputs.logits
        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
        ```
    """
    if "next_sentence_label" in kwargs:
        warnings.warn(
            "The `next_sentence_label` argument is deprecated and will be removed in a future version, use"
            " `labels` instead.",
            FutureWarning,
        )
        labels = kwargs.pop("next_sentence_label")

    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    seq_relationship_scores = self.cls(pooled_output)

    next_sentence_loss = None
    if labels is not None:
        next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

    if not return_dict:
        output = (seq_relationship_scores,) + outputs[2:]
        return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

    return NextSentencePredictorOutput(
        loss=next_sentence_loss,
        logits=seq_relationship_scores,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTraining

Bases: ErniePreTrainedModel

This class represents an Ernie model for pre-training tasks. It inherits from the ErniePreTrainedModel.

The class includes methods for initializing the model, getting and setting output embeddings, and forwarding the model for pre-training tasks. The forward method takes various input tensors and optional arguments, and returns the output of the model for pre-training. It also includes detailed information about the expected input parameters, optional arguments, and return values.

The class also provides an example of how to use the model for pre-training tasks using the AutoTokenizer and example inputs. The example demonstrates how to tokenize input text, generate model outputs, and access specific logits from the model.

For more details on the usage and functionality of the ErnieForPreTraining class, refer to the provided code and docstring examples.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
class ErnieForPreTraining(ErniePreTrainedModel):

    """
    This class represents an Ernie model for pre-training tasks. It inherits from the ErniePreTrainedModel.

    The class includes methods for initializing the model, getting and setting output embeddings, and forwarding the
    model for pre-training tasks. The `forward` method takes various input tensors and optional arguments, and returns
    the output of the model for pre-training. It also includes detailed information about the expected input parameters,
    optional arguments, and return values.

    The class also provides an example of how to use the model for pre-training tasks using the AutoTokenizer and
    example inputs. The example demonstrates how to tokenize input text, generate model outputs, and access specific
    logits from the model.

    For more details on the usage and functionality of the ErnieForPreTraining class, refer to the provided code and
    docstring examples.
    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """Initializes an instance of the ErnieForPreTraining class.

        Args:
            self (ErnieForPreTraining): An instance of the ErnieForPreTraining class.
            config (object): The configuration object for the ErnieForPreTraining class.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.ernie = ErnieModel(config)
        self.cls = ErniePreTrainingHeads(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.get_output_embeddings
    def get_output_embeddings(self):
        """
        Method to retrieve the output embeddings from the ErnieForPreTraining model.

        Args:
            self (ErnieForPreTraining): The instance of the ErnieForPreTraining class.

        Returns:
            None: This method does not return anything but directly accesses and returns the output embeddings
                from the model.

        Raises:
            None.
        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the ErnieForPreTraining model.

        Args:
            self (ErnieForPreTraining): An instance of the ErnieForPreTraining class.
            new_embeddings: The new embeddings to be set for the model predictions decoder. This can be of any type.

        Returns:
            None.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings
        self.cls.predictions.bias = new_embeddings.bias

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        next_sentence_label: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], ErnieForPreTrainingOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
                config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
                the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
            next_sentence_label (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
                pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

                - 0 indicates sequence B is a continuation of sequence A,
                - 1 indicates sequence B is a random sequence.
            kwargs (`Dict[str, any]`, optional, defaults to *{}*):
                Used to hide legacy arguments that have been deprecated.

        Returns:
            Union[Tuple[mindspore.Tensor], ErnieForPreTrainingOutput]

        Example:
            ```python
            >>> from transformers import AutoTokenizer, ErnieForPreTraining
            ...
            ...
            >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
            >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
            ...
            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
            >>> outputs = model(**inputs)
            ...
            >>> prediction_logits = outputs.prediction_logits
            >>> seq_relationship_logits = outputs.seq_relationship_logits
            ```
            """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output, pooled_output = outputs[:2]
        prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

        total_loss = None
        if labels is not None and next_sentence_label is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
            next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
            total_loss = masked_lm_loss + next_sentence_loss

        if not return_dict:
            output = (prediction_scores, seq_relationship_score) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return ErnieForPreTrainingOutput(
            loss=total_loss,
            prediction_logits=prediction_scores,
            seq_relationship_logits=seq_relationship_score,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTraining.__init__(config)

Initializes an instance of the ErnieForPreTraining class.

PARAMETER DESCRIPTION
self

An instance of the ErnieForPreTraining class.

TYPE: ErnieForPreTraining

config

The configuration object for the ErnieForPreTraining class.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
def __init__(self, config):
    """Initializes an instance of the ErnieForPreTraining class.

    Args:
        self (ErnieForPreTraining): An instance of the ErnieForPreTraining class.
        config (object): The configuration object for the ErnieForPreTraining class.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.ernie = ErnieModel(config)
    self.cls = ErniePreTrainingHeads(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTraining.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, next_sentence_label=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
labels

Labels for computing the masked language modeling loss. Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

next_sentence_label

Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see input_ids docstring) Indices should be in [0, 1]:

  • 0 indicates sequence B is a continuation of sequence A,
  • 1 indicates sequence B is a random sequence.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

kwargs

Used to hide legacy arguments that have been deprecated.

TYPE: `Dict[str, any]`, optional, defaults to *{}*

RETURNS DESCRIPTION
Union[Tuple[Tensor], ErnieForPreTrainingOutput]

Union[Tuple[mindspore.Tensor], ErnieForPreTrainingOutput]
Example 
>>> from transformers import AutoTokenizer, ErnieForPreTraining
...
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
>>> outputs = model(**inputs)
...
>>> prediction_logits = outputs.prediction_logits
>>> seq_relationship_logits = outputs.seq_relationship_logits
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    next_sentence_label: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], ErnieForPreTrainingOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
            the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        next_sentence_label (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
            pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.
        kwargs (`Dict[str, any]`, optional, defaults to *{}*):
            Used to hide legacy arguments that have been deprecated.

    Returns:
        Union[Tuple[mindspore.Tensor], ErnieForPreTrainingOutput]

    Example:
        ```python
        >>> from transformers import AutoTokenizer, ErnieForPreTraining
        ...
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
        >>> outputs = model(**inputs)
        ...
        >>> prediction_logits = outputs.prediction_logits
        >>> seq_relationship_logits = outputs.seq_relationship_logits
        ```
        """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output, pooled_output = outputs[:2]
    prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

    total_loss = None
    if labels is not None and next_sentence_label is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
        next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
        total_loss = masked_lm_loss + next_sentence_loss

    if not return_dict:
        output = (prediction_scores, seq_relationship_score) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return ErnieForPreTrainingOutput(
        loss=total_loss,
        prediction_logits=prediction_scores,
        seq_relationship_logits=seq_relationship_score,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTraining.get_output_embeddings()

Method to retrieve the output embeddings from the ErnieForPreTraining model.

PARAMETER DESCRIPTION
self

The instance of the ErnieForPreTraining class.

TYPE: ErnieForPreTraining

RETURNS DESCRIPTION
None

This method does not return anything but directly accesses and returns the output embeddings from the model.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
def get_output_embeddings(self):
    """
    Method to retrieve the output embeddings from the ErnieForPreTraining model.

    Args:
        self (ErnieForPreTraining): The instance of the ErnieForPreTraining class.

    Returns:
        None: This method does not return anything but directly accesses and returns the output embeddings
            from the model.

    Raises:
        None.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTraining.set_output_embeddings(new_embeddings)

Sets the output embeddings for the ErnieForPreTraining model.

PARAMETER DESCRIPTION
self

An instance of the ErnieForPreTraining class.

TYPE: ErnieForPreTraining

new_embeddings

The new embeddings to be set for the model predictions decoder. This can be of any type.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the ErnieForPreTraining model.

    Args:
        self (ErnieForPreTraining): An instance of the ErnieForPreTraining class.
        new_embeddings: The new embeddings to be set for the model predictions decoder. This can be of any type.

    Returns:
        None.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings
    self.cls.predictions.bias = new_embeddings.bias

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForPreTrainingOutput dataclass

Bases: ModelOutput

Output type of [ErnieForPreTraining].

PARAMETER DESCRIPTION
loss

Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss.

TYPE: *optional*, returned when `labels` is provided, `mindspore.Tensor` of shape `(1,)` DEFAULT: None

prediction_logits

Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)` DEFAULT: None

seq_relationship_logits

Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax).

TYPE: `mindspore.Tensor` of shape `(batch_size, 2)` DEFAULT: None

hidden_states

Tuple of mindspore.Tensor (one for the output of the embeddings + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size).

Hidden-states of the model at the output of each layer plus the initial embedding outputs.

TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None

attentions

Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length).

Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.

TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
@dataclass
# Copied from transformers.models.bert.modeling_bert.BertForPreTrainingOutput with Bert->Ernie
class ErnieForPreTrainingOutput(ModelOutput):
    """
    Output type of [`ErnieForPreTraining`].

    Args:
        loss (*optional*, returned when `labels` is provided, `mindspore.Tensor` of shape `(1,)`):
            Total loss as the sum of the masked language modeling loss and the next sequence prediction
            (classification) loss.
        prediction_logits (`mindspore.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        seq_relationship_logits (`mindspore.Tensor` of shape `(batch_size, 2)`):
            Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
            before SoftMax).
        hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    """
    loss: Optional[mindspore.Tensor] = None
    prediction_logits: mindspore.Tensor = None
    seq_relationship_logits: mindspore.Tensor = None
    hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    attentions: Optional[Tuple[mindspore.Tensor]] = None

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForQuestionAnswering

Bases: ErniePreTrainedModel

ErnieForQuestionAnswering is a class that represents a model for question answering tasks using the ERNIE (Enhanced Representation through kNowledge Integration) architecture. This class inherits from ErniePreTrainedModel and provides methods for forwarding the model and performing question answering inference.

The class forwardor initializes the model with the provided configuration. The model architecture includes an ERNIE model with the option to add a pooling layer. Additionally, it includes a dense layer for question answering outputs.

The forward method takes various input tensors and performs the question answering computation. It supports optional inputs for start and end positions, attention masks, token type IDs, task type IDs, position IDs, head masks, and input embeddings. The method returns the question-answering model output, which includes the start and end logits for the predicted answer spans.

The method also allows for customizing the return of outputs by specifying the return_dict parameter. If the return_dict parameter is not provided, the method uses the default value from the model's configuration.

Overall, the ErnieForQuestionAnswering class encapsulates the functionality for performing question answering tasks using the ERNIE model and provides a high-level interface for forwarding the model and performing inference.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
class ErnieForQuestionAnswering(ErniePreTrainedModel):

    """
    ErnieForQuestionAnswering is a class that represents a model for question answering tasks using the
    ERNIE (Enhanced Representation through kNowledge Integration) architecture.
    This class inherits from ErniePreTrainedModel and provides methods for forwarding the model and
    performing question answering inference.

    The class forwardor initializes the model with the provided configuration.
    The model architecture includes an ERNIE model with the option to add a pooling layer.
    Additionally, it includes a dense layer for question answering outputs.

    The forward method takes various input tensors and performs the question answering computation.
    It supports optional inputs for start and end positions, attention masks, token type IDs, task type IDs,
    position IDs, head masks, and input embeddings.
    The method returns the question-answering model output, which includes the start and end logits for the predicted
    answer spans.

    The method also allows for customizing the return of outputs by specifying the return_dict parameter.
    If the return_dict parameter is not provided, the method uses the default value from the model's configuration.

    Overall, the ErnieForQuestionAnswering class encapsulates the functionality for performing question answering tasks
    using the ERNIE model and provides a high-level interface for forwarding the model and
    performing inference.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the ErnieForQuestionAnswering class.

        Args:
            self (ErnieForQuestionAnswering): The object instance of the ErnieForQuestionAnswering class.
            config (object): An object containing configuration settings for the Ernie model.
                This parameter is required for initializing the ErnieForQuestionAnswering instance.
                It should include the following attributes:

                - num_labels (int): The number of labels for the classification task.
                - hidden_size (int): The size of the hidden layers in the model.
                - add_pooling_layer (bool): Flag indicating whether to add a pooling layer in the Ernie model.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected object type.
            ValueError: If the config object is missing any required attributes.
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.ernie = ErnieModel(config, add_pooling_layer=False)
        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], QuestionAnsweringModelOutput]:
        r"""
        Args:
            start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the start of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
                are not taken into account for computing the loss.
            end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the end of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
                are not taken into account for computing the loss.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]

        logits = self.qa_outputs(sequence_output)
        start_logits, end_logits = logits.split(1, axis=-1)
        start_logits = start_logits.squeeze(-1)
        end_logits = end_logits.squeeze(-1)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, split add a dimension
            if len(start_positions.shape) > 1:
                start_positions = start_positions.squeeze(-1)
            if len(end_positions.shape) > 1:
                end_positions = end_positions.squeeze(-1)
            # sometimes the start/end positions are outside our model inputs, we ignore these terms
            ignored_index = start_logits.shape[1]
            start_positions = start_positions.clamp(0, ignored_index)
            end_positions = end_positions.clamp(0, ignored_index)

            start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
            end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
            total_loss = (start_loss + end_loss) / 2

        if not return_dict:
            output = (start_logits, end_logits) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return QuestionAnsweringModelOutput(
            loss=total_loss,
            start_logits=start_logits,
            end_logits=end_logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForQuestionAnswering.__init__(config)

Initializes an instance of the ErnieForQuestionAnswering class.

PARAMETER DESCRIPTION
self

The object instance of the ErnieForQuestionAnswering class.

TYPE: ErnieForQuestionAnswering

config

An object containing configuration settings for the Ernie model. This parameter is required for initializing the ErnieForQuestionAnswering instance. It should include the following attributes:

  • num_labels (int): The number of labels for the classification task.
  • hidden_size (int): The size of the hidden layers in the model.
  • add_pooling_layer (bool): Flag indicating whether to add a pooling layer in the Ernie model.

TYPE: object

RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the config parameter is not of the expected object type.
ValueError

If the config object is missing any required attributes.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
def __init__(self, config):
    """
    Initializes an instance of the ErnieForQuestionAnswering class.

    Args:
        self (ErnieForQuestionAnswering): The object instance of the ErnieForQuestionAnswering class.
        config (object): An object containing configuration settings for the Ernie model.
            This parameter is required for initializing the ErnieForQuestionAnswering instance.
            It should include the following attributes:

            - num_labels (int): The number of labels for the classification task.
            - hidden_size (int): The size of the hidden layers in the model.
            - add_pooling_layer (bool): Flag indicating whether to add a pooling layer in the Ernie model.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected object type.
        ValueError: If the config object is missing any required attributes.
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.ernie = ErnieModel(config, add_pooling_layer=False)
    self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForQuestionAnswering.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
start_positions

Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

end_positions

Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], QuestionAnsweringModelOutput]:
    r"""
    Args:
        start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]

    logits = self.qa_outputs(sequence_output)
    start_logits, end_logits = logits.split(1, axis=-1)
    start_logits = start_logits.squeeze(-1)
    end_logits = end_logits.squeeze(-1)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        # If we are on multi-GPU, split add a dimension
        if len(start_positions.shape) > 1:
            start_positions = start_positions.squeeze(-1)
        if len(end_positions.shape) > 1:
            end_positions = end_positions.squeeze(-1)
        # sometimes the start/end positions are outside our model inputs, we ignore these terms
        ignored_index = start_logits.shape[1]
        start_positions = start_positions.clamp(0, ignored_index)
        end_positions = end_positions.clamp(0, ignored_index)

        start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
        end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
        total_loss = (start_loss + end_loss) / 2

    if not return_dict:
        output = (start_logits, end_logits) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return QuestionAnsweringModelOutput(
        loss=total_loss,
        start_logits=start_logits,
        end_logits=end_logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForSequenceClassification

Bases: ErniePreTrainedModel

This class represents an ERNIE model for sequence classification tasks. It is a subclass of the ErniePreTrainedModel class.

The ErnieForSequenceClassification class has an initialization method and a forward method. The initialization method initializes the ERNIE model and sets up the classifier layers. The forward method performs the forward pass of the model and returns the output.

ATTRIBUTE DESCRIPTION
num_labels

The number of labels for the sequence classification task.

TYPE: int

config

The configuration object for the ERNIE model.

TYPE: ErnieConfig

ernie

The ERNIE model instance.

TYPE: ErnieModel

dropout

Dropout layer for regularization.

TYPE: Dropout

classifier

Dense layer for classification.

TYPE: Linear

METHOD DESCRIPTION
__init__

Initializes the ErnieForSequenceClassification instance.
forward

Performs the forward pass of the ERNIE model and returns the output.
Example 
>>> # Initialize the model
>>> model = ErnieForSequenceClassification(config)
...
>>> # Perform forward pass
>>> output = model.forward(input_ids, attention_mask, token_type_ids, task_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
class ErnieForSequenceClassification(ErniePreTrainedModel):

    """
    This class represents an ERNIE model for sequence classification tasks.
    It is a subclass of the `ErniePreTrainedModel` class.

    The `ErnieForSequenceClassification` class has an initialization method and a `forward` method.
    The initialization method initializes the ERNIE model and sets up the classifier layers.
    The `forward` method performs the forward pass of the model and returns the output.

    Attributes:
        num_labels (int): The number of labels for the sequence classification task.
        config (ErnieConfig): The configuration object for the ERNIE model.
        ernie (ErnieModel): The ERNIE model instance.
        dropout (nn.Dropout): Dropout layer for regularization.
        classifier (nn.Linear): Dense layer for classification.

    Methods:
        __init__: Initializes the `ErnieForSequenceClassification` instance.
        forward: Performs the forward
            pass of the ERNIE model and returns the output.

    Example:
        ```python
        >>> # Initialize the model
        >>> model = ErnieForSequenceClassification(config)
        ...
        >>> # Perform forward pass
        >>> output = model.forward(input_ids, attention_mask, token_type_ids, task_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
        ```
    """
    # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the 'ErnieForSequenceClassification' class.

        Args:
            self: The instance of the class.
            config: An instance of 'Config' class containing the configuration parameters for the model.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config

        self.ernie = ErnieModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
                `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.num_labels == 1:
                    loss = F.mse_loss(logits.squeeze(), labels.squeeze())
                else:
                    loss = F.mse_loss(logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = F.binary_cross_entropy_with_logits(logits, labels)
        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForSequenceClassification.__init__(config)

Initializes an instance of the 'ErnieForSequenceClassification' class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An instance of 'Config' class containing the configuration parameters for the model.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
def __init__(self, config):
    """
    Initializes an instance of the 'ErnieForSequenceClassification' class.

    Args:
        self: The instance of the class.
        config: An instance of 'Config' class containing the configuration parameters for the model.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.config = config

    self.ernie = ErnieModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForSequenceClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
labels

Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If config.num_labels > 1 a classification loss is computed (Cross-Entropy).

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.num_labels == 1:
                loss = F.mse_loss(logits.squeeze(), labels.squeeze())
            else:
                loss = F.mse_loss(logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = F.binary_cross_entropy_with_logits(logits, labels)
    if not return_dict:
        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForTokenClassification

Bases: ErniePreTrainedModel

This class represents a token classification model based on the Ernie architecture. It is used for token-level classification tasks such as Named Entity Recognition (NER) and part-of-speech tagging. The model inherits from the ErniePreTrainedModel class and utilizes the ErnieModel for token embeddings and hidden representations. It includes methods for model initialization and forward propagation to compute token classification logits and loss.

The class's forwardor initializes the model with the provided configuration, sets the number of classification labels, and configures the ErnieModel with the specified parameters. Additionally, it sets up the dropout and classifier layers.

The forward method takes input tensors and optional arguments for token classification, and returns the token classification output. It also computes the token classification loss if labels are provided. The method supports various optional parameters for controlling the model's behavior during inference.

Note

The docstring is based on the provided information and does not include specific code signatures.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
class ErnieForTokenClassification(ErniePreTrainedModel):

    """
    This class represents a token classification model based on the Ernie architecture.
    It is used for token-level classification tasks such as Named Entity Recognition (NER) and part-of-speech tagging.
    The model inherits from the ErniePreTrainedModel class and utilizes the ErnieModel for token embeddings and
    hidden representations.
    It includes methods for model initialization and forward propagation to compute token classification logits and loss.

    The class's forwardor initializes the model with the provided configuration, sets the number of classification
    labels, and configures the ErnieModel with the specified parameters.
    Additionally, it sets up the dropout and classifier layers.

    The forward method takes input tensors and optional arguments for token classification, and returns the
    token classification output. It also computes the token classification loss if labels are provided.
    The method supports various optional parameters for controlling the model's behavior during inference.

    Note:
        The docstring is based on the provided information and does not include specific code signatures.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the ErnieForTokenClassification class.

        Args:
            self: The instance of the class.
            config (object): The configuration object containing the settings for the model.
                This object must have the following attributes:

                - num_labels (int): The number of labels for token classification.
                - classifier_dropout (float or None): The dropout rate for the classifier layer.
                If None, it defaults to the hidden dropout probability from the configuration.
                - hidden_dropout_prob (float): The dropout probability for the hidden layers.

        Returns:
            None.

        Raises:
            ValueError: If the config object is missing the num_labels attribute.
            TypeError: If the config object does not have the expected data types for the attributes.
            RuntimeError: If an error occurs during the initialization process.
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.ernie = ErnieModel(config, add_pooling_layer=False)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], TokenClassifierOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]

        sequence_output = self.dropout(sequence_output)
        logits = self.classifier(sequence_output)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return TokenClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForTokenClassification.__init__(config)

Initializes an instance of the ErnieForTokenClassification class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object containing the settings for the model. This object must have the following attributes:

  • num_labels (int): The number of labels for token classification.
  • classifier_dropout (float or None): The dropout rate for the classifier layer. If None, it defaults to the hidden dropout probability from the configuration.
  • hidden_dropout_prob (float): The dropout probability for the hidden layers.

TYPE: object

RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

If the config object is missing the num_labels attribute.
TypeError

If the config object does not have the expected data types for the attributes.
RuntimeError

If an error occurs during the initialization process.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
def __init__(self, config):
    """
    Initializes an instance of the ErnieForTokenClassification class.

    Args:
        self: The instance of the class.
        config (object): The configuration object containing the settings for the model.
            This object must have the following attributes:

            - num_labels (int): The number of labels for token classification.
            - classifier_dropout (float or None): The dropout rate for the classifier layer.
            If None, it defaults to the hidden dropout probability from the configuration.
            - hidden_dropout_prob (float): The dropout probability for the hidden layers.

    Returns:
        None.

    Raises:
        ValueError: If the config object is missing the num_labels attribute.
        TypeError: If the config object does not have the expected data types for the attributes.
        RuntimeError: If an error occurs during the initialization process.
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.ernie = ErnieModel(config, add_pooling_layer=False)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieForTokenClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
labels

Labels for computing the token classification loss. Indices should be in [0, ..., config.num_labels - 1].

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], TokenClassifierOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]

    sequence_output = self.dropout(sequence_output)
    logits = self.classifier(sequence_output)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

    if not return_dict:
        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return TokenClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieIntermediate

Bases: Module

Represents an intermediate layer for the ERNIE (Enhanced Representation through kNowledge Integration) model. This class provides methods to perform intermediate operations on input hidden states.

This class inherits from nn.Module and contains methods for initialization and forwarding the intermediate layer.

ATTRIBUTE DESCRIPTION
dense

A dense layer with the specified hidden size and intermediate size.

TYPE: Linear

intermediate_act_fn

The activation function applied to the intermediate hidden states.

TYPE: function

METHOD DESCRIPTION
__init__

Initializes the ERNIE intermediate layer with the provided configuration.
forward

Constructs the intermediate layer by applying dense and activation functions to the input hidden states.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
class ErnieIntermediate(nn.Module):

    '''
    Represents an intermediate layer for the ERNIE (Enhanced Representation through kNowledge Integration) model.
    This class provides methods to perform intermediate operations on input hidden states.

    This class inherits from nn.Module and contains methods for initialization and forwarding the intermediate layer.

    Attributes:
        dense (nn.Linear): A dense layer with the specified hidden size and intermediate size.
        intermediate_act_fn (function): The activation function applied to the intermediate hidden states.

    Methods:
        __init__: Initializes the ERNIE intermediate layer with the provided configuration.
        forward: Constructs the intermediate layer by applying dense and activation functions to the input hidden states.
    '''
    def __init__(self, config):
        """
        Initialize the ErnieIntermediate class with the provided configuration.

        Args:
            self (object): The instance of the ErnieIntermediate class.
            config (object):
                An object containing the configuration parameters.

                - hidden_size (int): The size of the hidden layer.
                - intermediate_size (int): The size of the intermediate layer.
                - hidden_act (str or function): The activation function for the hidden layer.
                If provided as a string, it should be a key in ACT2FN dictionary.

        Returns:
            None.

        Raises:
            ValueError: If the configuration parameters are invalid or missing.
            TypeError: If the provided hidden activation function is not a string or function.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the intermediate layer of the ERNIE model.

        Args:
            self (ErnieIntermediate): An instance of the ErnieIntermediate class.
            hidden_states (mindspore.Tensor): The input hidden states tensor of shape (batch_size, sequence_length, hidden_size).
                It represents the output from the previous layer of the ERNIE model.

        Returns:
            mindspore.Tensor: The tensor representing the intermediate hidden states of shape (batch_size, sequence_length, hidden_size).
                It is the result of applying the intermediate layer operations on the input hidden states.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieIntermediate.__init__(config)

Initialize the ErnieIntermediate class with the provided configuration.

PARAMETER DESCRIPTION
self

The instance of the ErnieIntermediate class.

TYPE: object

config

An object containing the configuration parameters.

  • hidden_size (int): The size of the hidden layer.
  • intermediate_size (int): The size of the intermediate layer.
  • hidden_act (str or function): The activation function for the hidden layer. If provided as a string, it should be a key in ACT2FN dictionary.

TYPE: object

RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

If the configuration parameters are invalid or missing.
TypeError

If the provided hidden activation function is not a string or function.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
def __init__(self, config):
    """
    Initialize the ErnieIntermediate class with the provided configuration.

    Args:
        self (object): The instance of the ErnieIntermediate class.
        config (object):
            An object containing the configuration parameters.

            - hidden_size (int): The size of the hidden layer.
            - intermediate_size (int): The size of the intermediate layer.
            - hidden_act (str or function): The activation function for the hidden layer.
            If provided as a string, it should be a key in ACT2FN dictionary.

    Returns:
        None.

    Raises:
        ValueError: If the configuration parameters are invalid or missing.
        TypeError: If the provided hidden activation function is not a string or function.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
    if isinstance(config.hidden_act, str):
        self.intermediate_act_fn = ACT2FN[config.hidden_act]
    else:
        self.intermediate_act_fn = config.hidden_act

mindnlp.transformers.models.ernie.modeling_ernie.ErnieIntermediate.forward(hidden_states)

Constructs the intermediate layer of the ERNIE model.

PARAMETER DESCRIPTION
self

An instance of the ErnieIntermediate class.

TYPE: ErnieIntermediate

hidden_states

The input hidden states tensor of shape (batch_size, sequence_length, hidden_size). It represents the output from the previous layer of the ERNIE model.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The tensor representing the intermediate hidden states of shape (batch_size, sequence_length, hidden_size). It is the result of applying the intermediate layer operations on the input hidden states.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the intermediate layer of the ERNIE model.

    Args:
        self (ErnieIntermediate): An instance of the ErnieIntermediate class.
        hidden_states (mindspore.Tensor): The input hidden states tensor of shape (batch_size, sequence_length, hidden_size).
            It represents the output from the previous layer of the ERNIE model.

    Returns:
        mindspore.Tensor: The tensor representing the intermediate hidden states of shape (batch_size, sequence_length, hidden_size).
            It is the result of applying the intermediate layer operations on the input hidden states.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.intermediate_act_fn(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLMPredictionHead

Bases: Module

Represents a prediction head for ERNIE Language Model that performs decoding and transformation operations on hidden states.

This class inherits from nn.Module and provides methods for initializing the prediction head and forwarding predictions based on the input hidden states.

ATTRIBUTE DESCRIPTION
transform

ErniePredictionHeadTransform object for transforming hidden states.
decoder

nn.Linear object for decoding hidden states into output predictions.
bias

Parameter object for bias initialization.
METHOD DESCRIPTION
__init__

Initializes the prediction head with the given configuration.
forward

Constructs predictions based on the input hidden states by applying transformation and decoding operations.
Example 
>>> config = get_config()
>>> prediction_head = ErnieLMPredictionHead(config)
>>> hidden_states = get_hidden_states()
>>> predictions = prediction_head.forward(hidden_states)
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
class ErnieLMPredictionHead(nn.Module):

    """
    Represents a prediction head for ERNIE Language Model that performs decoding and transformation operations
    on hidden states.

    This class inherits from nn.Module and provides methods for initializing the prediction head and forwarding
    predictions based on the input hidden states.

    Attributes:
        transform: ErniePredictionHeadTransform object for transforming hidden states.
        decoder: nn.Linear object for decoding hidden states into output predictions.
        bias: Parameter object for bias initialization.

    Methods:
        __init__: Initializes the prediction head with the given configuration.
        forward: Constructs predictions based on the input hidden states by applying
            transformation and decoding operations.

    Example:
        ```python
        >>> config = get_config()
        >>> prediction_head = ErnieLMPredictionHead(config)
        >>> hidden_states = get_hidden_states()
        >>> predictions = prediction_head.forward(hidden_states)
        ```
    """
    def __init__(self, config):
        """
        Initializes an instance of the ErnieLMPredictionHead class.

        Args:
            self: The instance of the ErnieLMPredictionHead class.
            config: An object that holds configuration settings for the ErnieLMPredictionHead.
                It is expected to contain properties like hidden_size, vocab_size, and any other relevant
                configuration parameters.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.transform = ErniePredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = Parameter(ops.zeros(config.vocab_size), 'bias')

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def _tie_weights(self):
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        """
        This method 'forward' is part of the class 'ErnieLMPredictionHead' and is responsible for forwarding
        the hidden states using transformation and decoding.

        Args:
            self: Represents the instance of the class. It is implicitly passed and does not need to be provided as
                an argument.

            hidden_states (Tensor): The input hidden states to be processed. It is expected to be a tensor containing
                the initial hidden states.

        Returns:
            Tensor: The processed hidden states after transformation and decoding.

        Raises:
            TypeError: If the input 'hidden_states' is not of type Tensor.
            ValueError: If the input 'hidden_states' is empty or invalid for transformation and decoding.
            RuntimeError: If there is an issue during the transformation or decoding process.
        """
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLMPredictionHead.__init__(config)

Initializes an instance of the ErnieLMPredictionHead class.

PARAMETER DESCRIPTION
self

The instance of the ErnieLMPredictionHead class.
config

An object that holds configuration settings for the ErnieLMPredictionHead. It is expected to contain properties like hidden_size, vocab_size, and any other relevant configuration parameters.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
def __init__(self, config):
    """
    Initializes an instance of the ErnieLMPredictionHead class.

    Args:
        self: The instance of the ErnieLMPredictionHead class.
        config: An object that holds configuration settings for the ErnieLMPredictionHead.
            It is expected to contain properties like hidden_size, vocab_size, and any other relevant
            configuration parameters.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.transform = ErniePredictionHeadTransform(config)

    # The output weights are the same as the input embeddings, but there is
    # an output-only bias for each token.
    self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

    self.bias = Parameter(ops.zeros(config.vocab_size), 'bias')

    # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
    self.decoder.bias = self.bias

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLMPredictionHead.forward(hidden_states)

This method 'forward' is part of the class 'ErnieLMPredictionHead' and is responsible for forwarding the hidden states using transformation and decoding.

PARAMETER DESCRIPTION
self

Represents the instance of the class. It is implicitly passed and does not need to be provided as an argument.
hidden_states

The input hidden states to be processed. It is expected to be a tensor containing the initial hidden states.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

The processed hidden states after transformation and decoding.
RAISES DESCRIPTION
TypeError

If the input 'hidden_states' is not of type Tensor.
ValueError

If the input 'hidden_states' is empty or invalid for transformation and decoding.
RuntimeError

If there is an issue during the transformation or decoding process.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
def forward(self, hidden_states):
    """
    This method 'forward' is part of the class 'ErnieLMPredictionHead' and is responsible for forwarding
    the hidden states using transformation and decoding.

    Args:
        self: Represents the instance of the class. It is implicitly passed and does not need to be provided as
            an argument.

        hidden_states (Tensor): The input hidden states to be processed. It is expected to be a tensor containing
            the initial hidden states.

    Returns:
        Tensor: The processed hidden states after transformation and decoding.

    Raises:
        TypeError: If the input 'hidden_states' is not of type Tensor.
        ValueError: If the input 'hidden_states' is empty or invalid for transformation and decoding.
        RuntimeError: If there is an issue during the transformation or decoding process.
    """
    hidden_states = self.transform(hidden_states)
    hidden_states = self.decoder(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLayer

Bases: Module

ErnieLayer is a class representing a layer in the Ernie model. This class inherits from nn.Module and contains methods for initializing the layer and forwarding the layer's feed forward chunk.

ATTRIBUTE DESCRIPTION
chunk_size_feed_forward

The chunk size for the feed forward operation.

TYPE: int

seq_len_dim

The dimension of the sequence length.

TYPE: int

attention

The attention mechanism used in the layer.

TYPE: ErnieAttention

is_decoder

Indicates whether the layer is a decoder model.

TYPE: bool

add_cross_attention

Indicates whether cross attention is added to the layer.

TYPE: bool

crossattention

The cross attention mechanism used in the layer.

TYPE: ErnieAttention

intermediate

The intermediate layer in the Ernie model.

TYPE: ErnieIntermediate

output

The output layer in the Ernie model.

TYPE: ErnieOutput

METHOD DESCRIPTION
__init__

Initializes the ErnieLayer with the provided configuration.
forward

Constructs the layer using the given input tensors and parameters.
feed_forward_chunk

Executes the feed forward operation on the attention output.
RAISES DESCRIPTION
ValueError

If the layer is not instantiated with cross-attention layers when encoder_hidden_states are passed.
RETURNS DESCRIPTION
Tuple

Outputs of the layer's forward method, including the layer output and present key value if the layer is a decoder model.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
class ErnieLayer(nn.Module):

    """ErnieLayer is a class representing a layer in the Ernie model.
    This class inherits from nn.Module and contains methods for initializing the layer and forwarding
    the layer's feed forward chunk.

    Attributes:
        chunk_size_feed_forward (int): The chunk size for the feed forward operation.
        seq_len_dim (int): The dimension of the sequence length.
        attention (ErnieAttention): The attention mechanism used in the layer.
        is_decoder (bool): Indicates whether the layer is a decoder model.
        add_cross_attention (bool): Indicates whether cross attention is added to the layer.
        crossattention (ErnieAttention): The cross attention mechanism used in the layer.
        intermediate (ErnieIntermediate): The intermediate layer in the Ernie model.
        output (ErnieOutput): The output layer in the Ernie model.

    Methods:
        __init__: Initializes the ErnieLayer with the provided configuration.
        forward: Constructs the layer using the given input tensors and parameters.
        feed_forward_chunk(attention_output): Executes the feed forward operation on the attention output.

    Raises:
        ValueError: If the layer is not instantiated with cross-attention layers when `encoder_hidden_states` are passed.

    Returns:
        Tuple: Outputs of the layer's forward method, including the layer output and present key value if the layer is a decoder model.
    """
    def __init__(self, config):
        """
        Initializes an instance of the ErnieLayer class.

        Args:
            self: The instance of the ErnieLayer class.
            config:
                A configuration object containing various settings for the ErnieLayer.

                - Type: object
                - Purpose: Configures the behavior of the ErnieLayer.
                - Restrictions: Must contain the following attributes:

                    - chunk_size_feed_forward: Chunk size for feed-forward operations.
                    - is_decoder: Boolean indicating whether the layer is used as a decoder model.
                    - add_cross_attention: Boolean indicating whether cross attention is added.
                    - position_embedding_type: Optional parameter specifying the position embedding type for cross
                    attention.

        Returns:
            None.

        Raises:
            ValueError: Raised if cross attention is added but the ErnieLayer is not used as a decoder model.
        """
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = ErnieAttention(config)
        self.is_decoder = config.is_decoder
        self.add_cross_attention = config.add_cross_attention
        if self.add_cross_attention:
            if not self.is_decoder:
                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
            self.crossattention = ErnieAttention(config, position_embedding_type="absolute")
        self.intermediate = ErnieIntermediate(config)
        self.output = ErnieOutput(config)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        Constructs an ERNIE (Enhanced Representation through kNowledge Integration) layer.

        Args:
            self: The object itself.
            hidden_states (mindspore.Tensor): The input hidden states for the layer.
            attention_mask (Optional[mindspore.Tensor]): Mask for the attention mechanism. Defaults to None.
            head_mask (Optional[mindspore.Tensor]): Mask for the attention heads. Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states from the encoder layer. Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]): Mask for the encoder attention mechanism.
                Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]): Cached key and value tensors for fast inference.
                Defaults to None.
            output_attentions (Optional[bool]): Whether to return attentions weights. Defaults to False.

        Returns:
            Tuple[mindspore.Tensor]: A tuple containing the layer output tensor.

        Raises:
            ValueError: If `encoder_hidden_states` are passed, and cross-attention layers are not instantiated
                by setting `config.add_cross_attention=True`.
        """
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        self_attention_outputs = self.attention(
            hidden_states,
            attention_mask,
            head_mask,
            output_attentions=output_attentions,
            past_key_value=self_attn_past_key_value,
        )
        attention_output = self_attention_outputs[0]

        # if decoder, the last output is tuple of self-attn cache
        if self.is_decoder:
            outputs = self_attention_outputs[1:-1]
            present_key_value = self_attention_outputs[-1]
        else:
            outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

        cross_attn_present_key_value = None
        if self.is_decoder and encoder_hidden_states is not None:
            if not hasattr(self, "crossattention"):
                raise ValueError(
                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                    " by setting `config.add_cross_attention=True`"
                )

            # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask,
                head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                cross_attn_past_key_value,
                output_attentions,
            )
            attention_output = cross_attention_outputs[0]
            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

            # add cross-attn cache to positions 3,4 of present_key_value tuple
            cross_attn_present_key_value = cross_attention_outputs[-1]
            present_key_value = present_key_value + cross_attn_present_key_value

        layer_output = apply_chunking_to_forward(
            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
        )
        outputs = (layer_output,) + outputs

        # if decoder, return the attn key/values as the last output
        if self.is_decoder:
            outputs = outputs + (present_key_value,)

        return outputs

    def feed_forward_chunk(self, attention_output):
        """
        This method calculates the feed-forward output for a chunk in the ErnieLayer.

        Args:
            self (object): The instance of the ErnieLayer class.
            attention_output (object): The attention output from the previous layer,
                expected to be a tensor representing the attention scores.

        Returns:
            None: This method does not return any value explicitly but updates the layer_output attribute of
                the ErnieLayer instance.

        Raises:
            None.
        """
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLayer.__init__(config)

Initializes an instance of the ErnieLayer class.

PARAMETER DESCRIPTION
self

The instance of the ErnieLayer class.
config

A configuration object containing various settings for the ErnieLayer.

  • Type: object
  • Purpose: Configures the behavior of the ErnieLayer.

  • Restrictions: Must contain the following attributes:

    • chunk_size_feed_forward: Chunk size for feed-forward operations.
    • is_decoder: Boolean indicating whether the layer is used as a decoder model.
    • add_cross_attention: Boolean indicating whether cross attention is added.
    • position_embedding_type: Optional parameter specifying the position embedding type for cross attention.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

Raised if cross attention is added but the ErnieLayer is not used as a decoder model.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
def __init__(self, config):
    """
    Initializes an instance of the ErnieLayer class.

    Args:
        self: The instance of the ErnieLayer class.
        config:
            A configuration object containing various settings for the ErnieLayer.

            - Type: object
            - Purpose: Configures the behavior of the ErnieLayer.
            - Restrictions: Must contain the following attributes:

                - chunk_size_feed_forward: Chunk size for feed-forward operations.
                - is_decoder: Boolean indicating whether the layer is used as a decoder model.
                - add_cross_attention: Boolean indicating whether cross attention is added.
                - position_embedding_type: Optional parameter specifying the position embedding type for cross
                attention.

    Returns:
        None.

    Raises:
        ValueError: Raised if cross attention is added but the ErnieLayer is not used as a decoder model.
    """
    super().__init__()
    self.chunk_size_feed_forward = config.chunk_size_feed_forward
    self.seq_len_dim = 1
    self.attention = ErnieAttention(config)
    self.is_decoder = config.is_decoder
    self.add_cross_attention = config.add_cross_attention
    if self.add_cross_attention:
        if not self.is_decoder:
            raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
        self.crossattention = ErnieAttention(config, position_embedding_type="absolute")
    self.intermediate = ErnieIntermediate(config)
    self.output = ErnieOutput(config)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLayer.feed_forward_chunk(attention_output)

This method calculates the feed-forward output for a chunk in the ErnieLayer.

PARAMETER DESCRIPTION
self

The instance of the ErnieLayer class.

TYPE: object

attention_output

The attention output from the previous layer, expected to be a tensor representing the attention scores.

TYPE: object

RETURNS DESCRIPTION
None

This method does not return any value explicitly but updates the layer_output attribute of the ErnieLayer instance.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
def feed_forward_chunk(self, attention_output):
    """
    This method calculates the feed-forward output for a chunk in the ErnieLayer.

    Args:
        self (object): The instance of the ErnieLayer class.
        attention_output (object): The attention output from the previous layer,
            expected to be a tensor representing the attention scores.

    Returns:
        None: This method does not return any value explicitly but updates the layer_output attribute of
            the ErnieLayer instance.

    Raises:
        None.
    """
    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)
    return layer_output

mindnlp.transformers.models.ernie.modeling_ernie.ErnieLayer.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

Constructs an ERNIE (Enhanced Representation through kNowledge Integration) layer.

PARAMETER DESCRIPTION
self

The object itself.
hidden_states

The input hidden states for the layer.

TYPE: Tensor

attention_mask

Mask for the attention mechanism. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

Mask for the attention heads. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

Hidden states from the encoder layer. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

Mask for the encoder attention mechanism. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

Cached key and value tensors for fast inference. Defaults to None.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

Whether to return attentions weights. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: A tuple containing the layer output tensor.
RAISES DESCRIPTION
ValueError

If encoder_hidden_states are passed, and cross-attention layers are not instantiated by setting config.add_cross_attention=True.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    Constructs an ERNIE (Enhanced Representation through kNowledge Integration) layer.

    Args:
        self: The object itself.
        hidden_states (mindspore.Tensor): The input hidden states for the layer.
        attention_mask (Optional[mindspore.Tensor]): Mask for the attention mechanism. Defaults to None.
        head_mask (Optional[mindspore.Tensor]): Mask for the attention heads. Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states from the encoder layer. Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]): Mask for the encoder attention mechanism.
            Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]): Cached key and value tensors for fast inference.
            Defaults to None.
        output_attentions (Optional[bool]): Whether to return attentions weights. Defaults to False.

    Returns:
        Tuple[mindspore.Tensor]: A tuple containing the layer output tensor.

    Raises:
        ValueError: If `encoder_hidden_states` are passed, and cross-attention layers are not instantiated
            by setting `config.add_cross_attention=True`.
    """
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    self_attention_outputs = self.attention(
        hidden_states,
        attention_mask,
        head_mask,
        output_attentions=output_attentions,
        past_key_value=self_attn_past_key_value,
    )
    attention_output = self_attention_outputs[0]

    # if decoder, the last output is tuple of self-attn cache
    if self.is_decoder:
        outputs = self_attention_outputs[1:-1]
        present_key_value = self_attention_outputs[-1]
    else:
        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

    cross_attn_present_key_value = None
    if self.is_decoder and encoder_hidden_states is not None:
        if not hasattr(self, "crossattention"):
            raise ValueError(
                f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                " by setting `config.add_cross_attention=True`"
            )

        # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        cross_attention_outputs = self.crossattention(
            attention_output,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            cross_attn_past_key_value,
            output_attentions,
        )
        attention_output = cross_attention_outputs[0]
        outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

        # add cross-attn cache to positions 3,4 of present_key_value tuple
        cross_attn_present_key_value = cross_attention_outputs[-1]
        present_key_value = present_key_value + cross_attn_present_key_value

    layer_output = apply_chunking_to_forward(
        self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
    )
    outputs = (layer_output,) + outputs

    # if decoder, return the attn key/values as the last output
    if self.is_decoder:
        outputs = outputs + (present_key_value,)

    return outputs

mindnlp.transformers.models.ernie.modeling_ernie.ErnieModel

Bases: ErniePreTrainedModel

The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in Attention is all you need by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

To behave as an decoder the model needs to be initialized with the is_decoder argument of the configuration set to True. To be used in a Seq2Seq model, the model needs to initialized with both is_decoder argument and add_cross_attention set to True; an encoder_hidden_states is then expected as an input to the forward pass.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
class ErnieModel(ErniePreTrainedModel):
    """
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    """
    # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Ernie
    def __init__(self, config, add_pooling_layer=True):
        """
        Initializes an instance of the ErnieModel class.

        Args:
            self: The instance of the class.
            config (object): The configuration object containing settings for the Ernie model.
            add_pooling_layer (bool): A flag indicating whether to add a pooling layer to the model. Default is True.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.config = config

        self.embeddings = ErnieEmbeddings(config)
        self.encoder = ErnieEncoder(config)

        self.pooler = ErniePooler(config) if add_pooling_layer else None

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertModel.get_input_embeddings
    def get_input_embeddings(self):
        """
        This method returns the input embeddings for the ErnieModel.

        Args:
            self: The instance of the ErnieModel class.

        Returns:
            The input embeddings for the ErnieModel.

        Raises:
            This method does not raise any exceptions.
        """
        return self.embeddings.word_embeddings

    # Copied from transformers.models.bert.modeling_bert.BertModel.set_input_embeddings
    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the ErnieModel.

        Args:
            self (ErnieModel): The instance of the ErnieModel class.
            value: The input embeddings to be set for the ErnieModel.
                It should be of type that is compatible with the embeddings.word_embeddings attribute.

        Returns:
            None.

        Raises:
            None.
        """
        self.embeddings.word_embeddings = value

    # Copied from transformers.models.bert.modeling_bert.BertModel._prune_heads
    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
        r"""
        encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4 tensors
            of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if self.config.is_decoder:
            use_cache = use_cache if use_cache is not None else self.config.use_cache
        else:
            use_cache = False

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.shape
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        batch_size, seq_length = input_shape

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if attention_mask is None:
            attention_mask = ops.ones(((batch_size, seq_length + past_key_values_length)))

        if token_type_ids is None:
            if hasattr(self.embeddings, "token_type_ids"):
                buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask: mindspore.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)

        # If a 2D or 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = ops.ones(encoder_hidden_shape)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            input_ids=input_ids,
            position_ids=position_ids,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            inputs_embeds=inputs_embeds,
            past_key_values_length=past_key_values_length,
        )
        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

        if not return_dict:
            return (sequence_output, pooled_output) + encoder_outputs[1:]

        return BaseModelOutputWithPoolingAndCrossAttentions(
            last_hidden_state=sequence_output,
            pooler_output=pooled_output,
            past_key_values=encoder_outputs.past_key_values,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
            cross_attentions=encoder_outputs.cross_attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieModel.__init__(config, add_pooling_layer=True)

Initializes an instance of the ErnieModel class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object containing settings for the Ernie model.

TYPE: object

add_pooling_layer

A flag indicating whether to add a pooling layer to the model. Default is True.

TYPE: bool DEFAULT: True

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
def __init__(self, config, add_pooling_layer=True):
    """
    Initializes an instance of the ErnieModel class.

    Args:
        self: The instance of the class.
        config (object): The configuration object containing settings for the Ernie model.
        add_pooling_layer (bool): A flag indicating whether to add a pooling layer to the model. Default is True.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.config = config

    self.embeddings = ErnieEmbeddings(config)
    self.encoder = ErnieEncoder(config)

    self.pooler = ErniePooler(config) if add_pooling_layer else None

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.ErnieModel.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

encoder_hidden_states (mindspore.Tensor of shape (batch_size, sequence_length, hidden_size), optional): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (mindspore.Tensor of shape (batch_size, sequence_length), optional): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in [0, 1]:

- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.

past_key_values (tuple(tuple(mindspore.Tensor)) of length config.n_layers with each tuple having 4 tensors of shape (batch_size, num_heads, sequence_length - 1, embed_size_per_head)): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If past_key_values are used, the user can optionally input only the last decoder_input_ids (those that don't have their past key value states given to this model) of shape (batch_size, 1) instead of all decoder_input_ids of shape (batch_size, sequence_length). use_cache (bool, optional): If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values).

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
    r"""
    encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
        the model is configured as a decoder.
    encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
        the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4 tensors
        of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
        Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
        If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
        don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
        `decoder_input_ids` of shape `(batch_size, sequence_length)`.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
        `past_key_values`).
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if self.config.is_decoder:
        use_cache = use_cache if use_cache is not None else self.config.use_cache
    else:
        use_cache = False

    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
        input_shape = input_ids.shape
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    batch_size, seq_length = input_shape

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if attention_mask is None:
        attention_mask = ops.ones(((batch_size, seq_length + past_key_values_length)))

    if token_type_ids is None:
        if hasattr(self.embeddings, "token_type_ids"):
            buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    extended_attention_mask: mindspore.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)

    # If a 2D or 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if self.config.is_decoder and encoder_hidden_states is not None:
        encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
        encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
            encoder_attention_mask = ops.ones(encoder_hidden_shape)
        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
    else:
        encoder_extended_attention_mask = None

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x n_heads x N x N
    # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
    # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
    head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

    embedding_output = self.embeddings(
        input_ids=input_ids,
        position_ids=position_ids,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        inputs_embeds=inputs_embeds,
        past_key_values_length=past_key_values_length,
    )
    encoder_outputs = self.encoder(
        embedding_output,
        attention_mask=extended_attention_mask,
        head_mask=head_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_extended_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    sequence_output = encoder_outputs[0]
    pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

    if not return_dict:
        return (sequence_output, pooled_output) + encoder_outputs[1:]

    return BaseModelOutputWithPoolingAndCrossAttentions(
        last_hidden_state=sequence_output,
        pooler_output=pooled_output,
        past_key_values=encoder_outputs.past_key_values,
        hidden_states=encoder_outputs.hidden_states,
        attentions=encoder_outputs.attentions,
        cross_attentions=encoder_outputs.cross_attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.ErnieModel.get_input_embeddings()

This method returns the input embeddings for the ErnieModel.

PARAMETER DESCRIPTION
self

The instance of the ErnieModel class.
RETURNS DESCRIPTION

The input embeddings for the ErnieModel.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
def get_input_embeddings(self):
    """
    This method returns the input embeddings for the ErnieModel.

    Args:
        self: The instance of the ErnieModel class.

    Returns:
        The input embeddings for the ErnieModel.

    Raises:
        This method does not raise any exceptions.
    """
    return self.embeddings.word_embeddings

mindnlp.transformers.models.ernie.modeling_ernie.ErnieModel.set_input_embeddings(value)

Sets the input embeddings for the ErnieModel.

PARAMETER DESCRIPTION
self

The instance of the ErnieModel class.

TYPE: ErnieModel

value

The input embeddings to be set for the ErnieModel. It should be of type that is compatible with the embeddings.word_embeddings attribute.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the ErnieModel.

    Args:
        self (ErnieModel): The instance of the ErnieModel class.
        value: The input embeddings to be set for the ErnieModel.
            It should be of type that is compatible with the embeddings.word_embeddings attribute.

    Returns:
        None.

    Raises:
        None.
    """
    self.embeddings.word_embeddings = value

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyMLMHead

Bases: Module

This class represents the implementation of the ErnieOnlyMLMHead, which is used for masked language model (MLM) prediction in Ernie language model. It inherits from the nn.Module class and contains methods for initializing the class and forwarding MLM predictions based on the input sequence output.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
class ErnieOnlyMLMHead(nn.Module):

    """
    This class represents the implementation of the ErnieOnlyMLMHead, which is used for masked language model (MLM)
    prediction in Ernie language model. It inherits from the nn.Module class and contains methods for initializing the
    class and forwarding MLM predictions based on the input sequence output.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the ErnieOnlyMLMHead class.

        Args:
            self: The instance of the ErnieOnlyMLMHead class.
            config: An object of the ErnieConfig class containing the configuration settings for the model.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.predictions = ErnieLMPredictionHead(config)

    def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the masked language model (MLM) head for the ERNIE model.

        Args:
            self (ErnieOnlyMLMHead): An instance of the ErnieOnlyMLMHead class.
            sequence_output (mindspore.Tensor): The output tensor of the ERNIE model's sequence encoder.
                This tensor represents the contextualized representations of input sequences.
                Shape: (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: The prediction scores generated by the MLM head.
                The prediction scores represent the likelihood of each token being masked and need to be compared with
                the corresponding ground truth labels during the training process.

                Shape: (batch_size, sequence_length, vocab_size).

        Raises:
            None.
        """
        prediction_scores = self.predictions(sequence_output)
        return prediction_scores

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyMLMHead.__init__(config)

Initializes a new instance of the ErnieOnlyMLMHead class.

PARAMETER DESCRIPTION
self

The instance of the ErnieOnlyMLMHead class.
config

An object of the ErnieConfig class containing the configuration settings for the model.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
def __init__(self, config):
    """
    Initializes a new instance of the ErnieOnlyMLMHead class.

    Args:
        self: The instance of the ErnieOnlyMLMHead class.
        config: An object of the ErnieConfig class containing the configuration settings for the model.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.predictions = ErnieLMPredictionHead(config)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyMLMHead.forward(sequence_output)

Constructs the masked language model (MLM) head for the ERNIE model.

PARAMETER DESCRIPTION
self

An instance of the ErnieOnlyMLMHead class.

TYPE: ErnieOnlyMLMHead

sequence_output

The output tensor of the ERNIE model's sequence encoder. This tensor represents the contextualized representations of input sequences. Shape: (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The prediction scores generated by the MLM head. The prediction scores represent the likelihood of each token being masked and need to be compared with the corresponding ground truth labels during the training process.

Shape: (batch_size, sequence_length, vocab_size).
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the masked language model (MLM) head for the ERNIE model.

    Args:
        self (ErnieOnlyMLMHead): An instance of the ErnieOnlyMLMHead class.
        sequence_output (mindspore.Tensor): The output tensor of the ERNIE model's sequence encoder.
            This tensor represents the contextualized representations of input sequences.
            Shape: (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: The prediction scores generated by the MLM head.
            The prediction scores represent the likelihood of each token being masked and need to be compared with
            the corresponding ground truth labels during the training process.

            Shape: (batch_size, sequence_length, vocab_size).

    Raises:
        None.
    """
    prediction_scores = self.predictions(sequence_output)
    return prediction_scores

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyNSPHead

Bases: Module

Represents a head for Next Sentence Prediction (NSP) task in the ERNIE model.

This class inherits from the nn.Module module and provides functionality to predict whether two input sequences are consecutive in the ERNIE model. It contains methods to initialize the head and forward the NSP score based on the pooled output of the model.

METHOD DESCRIPTION
__init__

Initializes the NSP head with a Dense layer for sequence relationship prediction.
forward

Constructs the NSP score by passing the pooled output through the Dense layer.
ATTRIBUTE DESCRIPTION
seq_relationship

A Dense layer with hidden_size neurons for predicting the relationship between sequences.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
class ErnieOnlyNSPHead(nn.Module):

    """
    Represents a head for Next Sentence Prediction (NSP) task in the ERNIE model.

    This class inherits from the nn.Module module and provides functionality to predict whether two input sequences
    are consecutive in the ERNIE model. It contains methods to initialize the head and forward the NSP score based
    on the pooled output of the model.

    Methods:
        __init__: Initializes the NSP head with a Dense layer for sequence relationship prediction.
        forward: Constructs the NSP score by passing the pooled output through the Dense layer.

    Attributes:
        seq_relationship: A Dense layer with hidden_size neurons for predicting the relationship between sequences.
    """
    def __init__(self, config):
        """
        Initializes an instance of the ErnieOnlyNSPHead class.

        Args:
            self: The instance of the class.
            config:
                An object containing configuration settings.

                - Type: Any
                - Purpose: Specifies the configuration settings for the head.
                - Restrictions: Must be compatible with the nn.Linear module.

        Returns:
            None. This method does not return any value.

        Raises:
            TypeError: If the 'config' parameter is not provided.
            ValueError: If the 'config.hidden_size' value is invalid or incompatible with nn.Linear.
        """
        super().__init__()
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, pooled_output):
        """
        Constructs the sequence relationship score based on the pooled output.

        Args:
            self: Instance of the ErnieOnlyNSPHead class.
            pooled_output:
                Tensor containing the pooled output from the model.

                - Type: Tensor
                - Purpose: Represents the output features obtained after pooling.
                - Restrictions: Must be a valid tensor object.

        Returns:
            seq_relationship_score:
                The calculated sequence relationship score based on the pooled output.

                - Type: None
                - Purpose: Represents the score indicating the relationship between two sequences.

        Raises:
            None
        """
        seq_relationship_score = self.seq_relationship(pooled_output)
        return seq_relationship_score

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyNSPHead.__init__(config)

Initializes an instance of the ErnieOnlyNSPHead class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An object containing configuration settings.

  • Type: Any
  • Purpose: Specifies the configuration settings for the head.
  • Restrictions: Must be compatible with the nn.Linear module.
RETURNS DESCRIPTION

None. This method does not return any value.
RAISES DESCRIPTION
TypeError

If the 'config' parameter is not provided.
ValueError

If the 'config.hidden_size' value is invalid or incompatible with nn.Linear.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
def __init__(self, config):
    """
    Initializes an instance of the ErnieOnlyNSPHead class.

    Args:
        self: The instance of the class.
        config:
            An object containing configuration settings.

            - Type: Any
            - Purpose: Specifies the configuration settings for the head.
            - Restrictions: Must be compatible with the nn.Linear module.

    Returns:
        None. This method does not return any value.

    Raises:
        TypeError: If the 'config' parameter is not provided.
        ValueError: If the 'config.hidden_size' value is invalid or incompatible with nn.Linear.
    """
    super().__init__()
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOnlyNSPHead.forward(pooled_output)

Constructs the sequence relationship score based on the pooled output.

PARAMETER DESCRIPTION
self

Instance of the ErnieOnlyNSPHead class.
pooled_output

Tensor containing the pooled output from the model.

  • Type: Tensor
  • Purpose: Represents the output features obtained after pooling.
  • Restrictions: Must be a valid tensor object.
RETURNS DESCRIPTION
seq_relationship_score

The calculated sequence relationship score based on the pooled output.

  • Type: None
  • Purpose: Represents the score indicating the relationship between two sequences.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
def forward(self, pooled_output):
    """
    Constructs the sequence relationship score based on the pooled output.

    Args:
        self: Instance of the ErnieOnlyNSPHead class.
        pooled_output:
            Tensor containing the pooled output from the model.

            - Type: Tensor
            - Purpose: Represents the output features obtained after pooling.
            - Restrictions: Must be a valid tensor object.

    Returns:
        seq_relationship_score:
            The calculated sequence relationship score based on the pooled output.

            - Type: None
            - Purpose: Represents the score indicating the relationship between two sequences.

    Raises:
        None
    """
    seq_relationship_score = self.seq_relationship(pooled_output)
    return seq_relationship_score

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOutput

Bases: Module

The ErnieOutput class represents a neural network cell for processing output in the ERNIE model. This class inherits from the nn.Module class and includes methods for initializing and forwarding the output layer for the model.

The init method initializes the ErnieOutput instance with the specified configuration. It initializes the dense layer, LayerNorm, and dropout for processing the output.

The forward method processes the hidden states and input tensor to generate the final output tensor using the initialized dense layer, dropout, and LayerNorm.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
class ErnieOutput(nn.Module):

    """
    The ErnieOutput class represents a neural network cell for processing output in the ERNIE model.
    This class inherits from the nn.Module class and includes methods for initializing and forwarding the output layer for the model.

    The __init__ method initializes the ErnieOutput instance with the specified configuration.
    It initializes the dense layer, LayerNorm, and dropout for processing the output.

    The forward method processes the hidden states and input tensor to generate the final output tensor using the
    initialized dense layer, dropout, and LayerNorm.

    """
    def __init__(self, config):
        """
        Initializes an instance of ErnieOutput.

        Args:
            self (ErnieOutput): The instance of the ErnieOutput class.
            config:
                An object containing configuration parameters.

                - Type: Any
                - Purpose: Configuration object specifying model settings.
                - Restrictions: Must be compatible with the specified configuration format.

        Returns:
            None.

        Raises:
            TypeError: If the provided config parameter is not of the expected type.
            ValueError: If the config parameter does not contain required attributes.
        """
        super().__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the output tensor for the Ernie model.

        Args:
            self (ErnieOutput): The instance of the ErnieOutput class.
            hidden_states (mindspore.Tensor): The hidden states tensor generated by the model.
                This tensor is processed through dense layers and normalization.
            input_tensor (mindspore.Tensor): The input tensor to be added to the processed hidden states.
                It serves as additional information for the final output.

        Returns:
            mindspore.Tensor: The forwarded output tensor that combines the processed hidden states
                with the input tensor to produce the final output of the Ernie model.

        Raises:
            None
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOutput.__init__(config)

Initializes an instance of ErnieOutput.

PARAMETER DESCRIPTION
self

The instance of the ErnieOutput class.

TYPE: ErnieOutput

config

An object containing configuration parameters.

  • Type: Any
  • Purpose: Configuration object specifying model settings.
  • Restrictions: Must be compatible with the specified configuration format.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the provided config parameter is not of the expected type.
ValueError

If the config parameter does not contain required attributes.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
def __init__(self, config):
    """
    Initializes an instance of ErnieOutput.

    Args:
        self (ErnieOutput): The instance of the ErnieOutput class.
        config:
            An object containing configuration parameters.

            - Type: Any
            - Purpose: Configuration object specifying model settings.
            - Restrictions: Must be compatible with the specified configuration format.

    Returns:
        None.

    Raises:
        TypeError: If the provided config parameter is not of the expected type.
        ValueError: If the config parameter does not contain required attributes.
    """
    super().__init__()
    self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieOutput.forward(hidden_states, input_tensor)

Constructs the output tensor for the Ernie model.

PARAMETER DESCRIPTION
self

The instance of the ErnieOutput class.

TYPE: ErnieOutput

hidden_states

The hidden states tensor generated by the model. This tensor is processed through dense layers and normalization.

TYPE: Tensor

input_tensor

The input tensor to be added to the processed hidden states. It serves as additional information for the final output.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The forwarded output tensor that combines the processed hidden states with the input tensor to produce the final output of the Ernie model.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the output tensor for the Ernie model.

    Args:
        self (ErnieOutput): The instance of the ErnieOutput class.
        hidden_states (mindspore.Tensor): The hidden states tensor generated by the model.
            This tensor is processed through dense layers and normalization.
        input_tensor (mindspore.Tensor): The input tensor to be added to the processed hidden states.
            It serves as additional information for the final output.

    Returns:
        mindspore.Tensor: The forwarded output tensor that combines the processed hidden states
            with the input tensor to produce the final output of the Ernie model.

    Raises:
        None
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErniePooler

Bases: Module

ErniePooler class represents a pooler layer for an ERNIE model.

This class inherits from nn.Module and implements a pooler layer that takes hidden states as input, processes the first token tensor through a dense layer and activation function, and returns the pooled output.

ATTRIBUTE DESCRIPTION
dense

A dense layer with the specified hidden size.

TYPE: Linear

activation

A hyperbolic tangent activation function.

TYPE: Tanh

METHOD DESCRIPTION
__init__

Initializes the ErniePooler object with the provided configuration.
forward

Constructs the pooled output from the hidden states input.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
class ErniePooler(nn.Module):

    """
    ErniePooler class represents a pooler layer for an ERNIE model.

    This class inherits from nn.Module and implements a pooler layer that takes hidden states as input,
    processes the first token tensor through a dense layer and activation function, and returns the pooled output.

    Attributes:
        dense (nn.Linear): A dense layer with the specified hidden size.
        activation (nn.Tanh): A hyperbolic tangent activation function.

    Methods:
        __init__: Initializes the ErniePooler object with the provided configuration.
        forward: Constructs the pooled output from the hidden states input.
    """
    def __init__(self, config):
        """
        Initializes an instance of the ErniePooler class.

        Args:
            self: The instance of the class.
            config:
                An object of type 'config' which contains the configuration parameters.

                - Type: Any valid object.
                - Purpose: Specifies the configuration parameters for the ErniePooler instance.
                - Restrictions: None.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs a pooled output tensor from the given hidden states.

        Args:
            self (ErniePooler): An instance of the ErniePooler class.
            hidden_states (mindspore.Tensor): A tensor containing the hidden states.
                Shape should be (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: A tensor representing the pooled output.
                Shape is (batch_size, hidden_size).

        Raises:
            None.
        """
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token.
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

mindnlp.transformers.models.ernie.modeling_ernie.ErniePooler.__init__(config)

Initializes an instance of the ErniePooler class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An object of type 'config' which contains the configuration parameters.

  • Type: Any valid object.
  • Purpose: Specifies the configuration parameters for the ErniePooler instance.
  • Restrictions: None.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
def __init__(self, config):
    """
    Initializes an instance of the ErniePooler class.

    Args:
        self: The instance of the class.
        config:
            An object of type 'config' which contains the configuration parameters.

            - Type: Any valid object.
            - Purpose: Specifies the configuration parameters for the ErniePooler instance.
            - Restrictions: None.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.activation = nn.Tanh()

mindnlp.transformers.models.ernie.modeling_ernie.ErniePooler.forward(hidden_states)

Constructs a pooled output tensor from the given hidden states.

PARAMETER DESCRIPTION
self

An instance of the ErniePooler class.

TYPE: ErniePooler

hidden_states

A tensor containing the hidden states. Shape should be (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: A tensor representing the pooled output. Shape is (batch_size, hidden_size).
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs a pooled output tensor from the given hidden states.

    Args:
        self (ErniePooler): An instance of the ErniePooler class.
        hidden_states (mindspore.Tensor): A tensor containing the hidden states.
            Shape should be (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: A tensor representing the pooled output.
            Shape is (batch_size, hidden_size).

    Raises:
        None.
    """
    # We "pool" the model by simply taking the hidden state corresponding
    # to the first token.
    first_token_tensor = hidden_states[:, 0]
    pooled_output = self.dense(first_token_tensor)
    pooled_output = self.activation(pooled_output)
    return pooled_output

mindnlp.transformers.models.ernie.modeling_ernie.ErniePreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
class ErniePreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """
    config_class = ErnieConfig
    base_model_prefix = "ernie"
    supports_gradient_checkpointing = True

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(initializer(Normal(self.config.initializer_range),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.bias is not None:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, self.config.initializer_range, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))
        elif isinstance(cell, nn.LayerNorm):
            cell.weight.set_data(initializer('ones', cell.weight.shape, cell.weight.dtype))
            cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))

mindnlp.transformers.models.ernie.modeling_ernie.ErniePreTrainingHeads

Bases: Module

The ErniePreTrainingHeads class represents the pre-training heads for ERNIE model, used for predicting masked tokens and sequence relationships. It inherits from nn.Module and provides methods for initializing the prediction heads and making predictions.

METHOD DESCRIPTION
__init__

Initializes the ErniePreTrainingHeads instance with the given configuration.
forward

Constructs the pre-training heads using the sequence output and pooled output, and returns the prediction scores and sequence relationship score.
ATTRIBUTE DESCRIPTION
predictions

Instance of ErnieLMPredictionHead for predicting masked tokens.
seq_relationship

Dense layer for predicting sequence relationships.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
class ErniePreTrainingHeads(nn.Module):

    """
    The ErniePreTrainingHeads class represents the pre-training heads for ERNIE model, used for predicting masked tokens
    and sequence relationships. It inherits from nn.Module and provides methods for initializing the prediction heads
    and making predictions.

    Methods:
        __init__: Initializes the ErniePreTrainingHeads instance with the given configuration.
        forward: Constructs the pre-training heads using the sequence output and pooled output, and returns the
            prediction scores and sequence relationship score.

    Attributes:
        predictions: Instance of ErnieLMPredictionHead for predicting masked tokens.
        seq_relationship: Dense layer for predicting sequence relationships.
    """
    def __init__(self, config):
        """
        Initialize the ErniePreTrainingHeads class.

        Args:
            self: The instance of the ErniePreTrainingHeads class.
            config: A configuration object containing the settings for the ErniePreTrainingHeads.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected type.
            ValueError: If the config parameter does not contain the required settings.
        """
        super().__init__()
        self.predictions = ErnieLMPredictionHead(config)
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, sequence_output, pooled_output):
        """
        Constructs the prediction scores and sequence relationship scores for the ErniePreTrainingHeads model.

        Args:
            self (ErniePreTrainingHeads): An instance of the ErniePreTrainingHeads class.
            sequence_output (Tensor): The output tensor from the sequence model.
                This tensor contains the contextualized representations for each token in the input sequence.
                Shape: (batch_size, sequence_length, hidden_size)
            pooled_output (Tensor): The output tensor from the pooling model.
                This tensor contains the pooled representation of the input sequence.
                Shape: (batch_size, hidden_size)

        Returns:
            Tuple[Tensor, Tensor]:
                A tuple of prediction scores and sequence relationship scores.

                - prediction_scores (Tensor): The prediction scores for each token in the input sequence.
                Each score represents the probability of the token being masked in pre-training.
                Shape: (batch_size, sequence_length, vocab_size)
                - seq_relationship_score (Tensor): The sequence relationship score.
                This score represents the probability of the input sequence being a continuation of another sequence.
                Shape: (batch_size, num_labels)

        Raises:
            None.
        """
        prediction_scores = self.predictions(sequence_output)
        seq_relationship_score = self.seq_relationship(pooled_output)
        return prediction_scores, seq_relationship_score

mindnlp.transformers.models.ernie.modeling_ernie.ErniePreTrainingHeads.__init__(config)

Initialize the ErniePreTrainingHeads class.

PARAMETER DESCRIPTION
self

The instance of the ErniePreTrainingHeads class.
config

A configuration object containing the settings for the ErniePreTrainingHeads.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the config parameter is not of the expected type.
ValueError

If the config parameter does not contain the required settings.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
def __init__(self, config):
    """
    Initialize the ErniePreTrainingHeads class.

    Args:
        self: The instance of the ErniePreTrainingHeads class.
        config: A configuration object containing the settings for the ErniePreTrainingHeads.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected type.
        ValueError: If the config parameter does not contain the required settings.
    """
    super().__init__()
    self.predictions = ErnieLMPredictionHead(config)
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.ernie.modeling_ernie.ErniePreTrainingHeads.forward(sequence_output, pooled_output)

Constructs the prediction scores and sequence relationship scores for the ErniePreTrainingHeads model.

PARAMETER DESCRIPTION
self

An instance of the ErniePreTrainingHeads class.

TYPE: ErniePreTrainingHeads

sequence_output

The output tensor from the sequence model. This tensor contains the contextualized representations for each token in the input sequence. Shape: (batch_size, sequence_length, hidden_size)

TYPE: Tensor

pooled_output

The output tensor from the pooling model. This tensor contains the pooled representation of the input sequence. Shape: (batch_size, hidden_size)

TYPE: Tensor

RETURNS DESCRIPTION

Tuple[Tensor, Tensor]: A tuple of prediction scores and sequence relationship scores.

  • prediction_scores (Tensor): The prediction scores for each token in the input sequence. Each score represents the probability of the token being masked in pre-training. Shape: (batch_size, sequence_length, vocab_size)
  • seq_relationship_score (Tensor): The sequence relationship score. This score represents the probability of the input sequence being a continuation of another sequence. Shape: (batch_size, num_labels)
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
def forward(self, sequence_output, pooled_output):
    """
    Constructs the prediction scores and sequence relationship scores for the ErniePreTrainingHeads model.

    Args:
        self (ErniePreTrainingHeads): An instance of the ErniePreTrainingHeads class.
        sequence_output (Tensor): The output tensor from the sequence model.
            This tensor contains the contextualized representations for each token in the input sequence.
            Shape: (batch_size, sequence_length, hidden_size)
        pooled_output (Tensor): The output tensor from the pooling model.
            This tensor contains the pooled representation of the input sequence.
            Shape: (batch_size, hidden_size)

    Returns:
        Tuple[Tensor, Tensor]:
            A tuple of prediction scores and sequence relationship scores.

            - prediction_scores (Tensor): The prediction scores for each token in the input sequence.
            Each score represents the probability of the token being masked in pre-training.
            Shape: (batch_size, sequence_length, vocab_size)
            - seq_relationship_score (Tensor): The sequence relationship score.
            This score represents the probability of the input sequence being a continuation of another sequence.
            Shape: (batch_size, num_labels)

    Raises:
        None.
    """
    prediction_scores = self.predictions(sequence_output)
    seq_relationship_score = self.seq_relationship(pooled_output)
    return prediction_scores, seq_relationship_score

mindnlp.transformers.models.ernie.modeling_ernie.ErniePredictionHeadTransform

Bases: Module

This class represents the transformation head for the ERNIE prediction model. It performs various operations such as dense transformation, activation function application, and layer normalization on the input hidden states.

Inherits from

nn.Module

ATTRIBUTE DESCRIPTION
dense

A dense layer used for transforming the input hidden states.

TYPE: Linear

transform_act_fn

The activation function applied to the transformed hidden states.

TYPE: function

LayerNorm

A layer normalization module applied to the hidden states.

TYPE: LayerNorm

METHOD DESCRIPTION
__init__

Initializes the class instance with the provided configuration.
forward

Applies the transformation operations on the input hidden states and returns the transformed states.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
class ErniePredictionHeadTransform(nn.Module):

    """
    This class represents the transformation head for the ERNIE prediction model.
    It performs various operations such as dense transformation, activation function application,
    and layer normalization on the input hidden states.

    Inherits from:
        nn.Module

    Attributes:
        dense (nn.Linear): A dense layer used for transforming the input hidden states.
        transform_act_fn (function): The activation function applied to the transformed hidden states.
        LayerNorm (nn.LayerNorm): A layer normalization module applied to the hidden states.

    Methods:
        __init__: Initializes the class instance with the provided configuration.
        forward: Applies the transformation operations on the input hidden states and returns the transformed states.

    """
    def __init__(self, config):
        """Initializes an instance of the ErniePredictionHeadTransform class.

        Args:
            self (ErniePredictionHeadTransform): An instance of the ErniePredictionHeadTransform class.
            config: The configuration object containing the settings for the ErniePredictionHeadTransform.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        if isinstance(config.hidden_act, str):
            self.transform_act_fn = ACT2FN[config.hidden_act]
        else:
            self.transform_act_fn = config.hidden_act
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the ErniePredictionHeadTransform.

        Args:
            self (ErniePredictionHeadTransform): An instance of the ErniePredictionHeadTransform class.
            hidden_states (mindspore.Tensor): The input hidden states to be transformed.
                It should have a shape of (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: The transformed hidden states. It has the same shape as the input hidden states.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.transform_act_fn(hidden_states)
        hidden_states = self.LayerNorm(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErniePredictionHeadTransform.__init__(config)

Initializes an instance of the ErniePredictionHeadTransform class.

PARAMETER DESCRIPTION
self

An instance of the ErniePredictionHeadTransform class.

TYPE: ErniePredictionHeadTransform

config

The configuration object containing the settings for the ErniePredictionHeadTransform.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
def __init__(self, config):
    """Initializes an instance of the ErniePredictionHeadTransform class.

    Args:
        self (ErniePredictionHeadTransform): An instance of the ErniePredictionHeadTransform class.
        config: The configuration object containing the settings for the ErniePredictionHeadTransform.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    if isinstance(config.hidden_act, str):
        self.transform_act_fn = ACT2FN[config.hidden_act]
    else:
        self.transform_act_fn = config.hidden_act
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)

mindnlp.transformers.models.ernie.modeling_ernie.ErniePredictionHeadTransform.forward(hidden_states)

Constructs the ErniePredictionHeadTransform.

PARAMETER DESCRIPTION
self

An instance of the ErniePredictionHeadTransform class.

TYPE: ErniePredictionHeadTransform

hidden_states

The input hidden states to be transformed. It should have a shape of (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The transformed hidden states. It has the same shape as the input hidden states.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the ErniePredictionHeadTransform.

    Args:
        self (ErniePredictionHeadTransform): An instance of the ErniePredictionHeadTransform class.
        hidden_states (mindspore.Tensor): The input hidden states to be transformed.
            It should have a shape of (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: The transformed hidden states. It has the same shape as the input hidden states.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.transform_act_fn(hidden_states)
    hidden_states = self.LayerNorm(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfAttention

Bases: Module

This class represents a self-attention mechanism for the ERNIE (Enhanced Representation through kNowledge Integration) model. It is used to compute attention scores and produce context layers during the processing of input data. The class inherits from nn.Module and includes methods for initializing the self-attention mechanism, transposing tensors for scoring calculations, and forwarding the attention mechanism outputs.

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
class ErnieSelfAttention(nn.Module):

    """
    This class represents a self-attention mechanism for the ERNIE (Enhanced Representation through kNowledge Integration) model.
    It is used to compute attention scores and produce context layers during the processing of input data.
    The class inherits from nn.Module and includes methods for initializing the self-attention mechanism,
    transposing tensors for scoring calculations, and forwarding the attention mechanism outputs.
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initialize the ErnieSelfAttention class.

        Args:
            self: The instance of the class.
            config:
                An instance of the configuration class containing the following attributes:

                - hidden_size (int): The size of the hidden layers.
                - num_attention_heads (int): The number of attention heads.
                - embedding_size (int, optional): The size of the embedding layer.
                If not provided, it is expected to be an attribute of the config.
                - attention_probs_dropout_prob (float): The dropout probability for attention probabilities.
                - position_embedding_type (str, optional): The type of position embedding.
                If not provided, it defaults to 'absolute'.
                - max_position_embeddings (int): The maximum number of position embeddings.
                - is_decoder (bool): Indicates if the model is a decoder.

            position_embedding_type (str, optional): The type of position embedding. Defaults to None.

        Returns:
            None.

        Raises:
            ValueError: If the hidden size is not a multiple of the number of attention heads and the config does not
                have an 'embedding_size' attribute.
        """
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
                f"heads ({config.num_attention_heads})"
            )

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        self.key = nn.Linear(config.hidden_size, self.all_head_size)
        self.value = nn.Linear(config.hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
        self.position_embedding_type = position_embedding_type or getattr(
            config, "position_embedding_type", "absolute"
        )
        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            self.max_position_embeddings = config.max_position_embeddings
            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

        self.is_decoder = config.is_decoder

    def transpose_for_scores(self, x: mindspore.Tensor) -> mindspore.Tensor:
        """
        Transpose the input tensor for calculating attention scores.

        Args:
            self (ErnieSelfAttention): The instance of the ErnieSelfAttention class.
            x (mindspore.Tensor): The input tensor with shape (batch_size, seq_length, hidden_size).

        Returns:
            mindspore.Tensor:
                The transposed tensor with shape (batch_size, num_attention_heads, seq_length, attention_head_size).

        Raises:
            TypeError: If the input tensor is not of type mindspore.Tensor.
            ValueError: If the input tensor shape is not compatible with the expected shape for transposition.
        """
        new_x_shape = x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        Constructs the self-attention mechanism for the ERNIE model.

        Args:
            self (ErnieSelfAttention): The instance of the ErnieSelfAttention class.
            hidden_states (mindspore.Tensor): The input hidden states of the model.
                Shape: (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]): The attention mask tensor.
                It is a binary tensor of shape (batch_size, sequence_length) where 1 indicates a valid token
                and 0 indicates a padded token. Defaults to None.
            head_mask (Optional[mindspore.Tensor]): The head mask tensor.
                It is a binary tensor of shape (num_attention_heads,) indicating which heads to mask. Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                The hidden states of the encoder. Shape: (batch_size, encoder_sequence_length, hidden_size). Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                The attention mask tensor for the encoder. Shape: (batch_size, encoder_sequence_length). Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                The cached key-value pairs from previous attention computations. Defaults to None.
            output_attentions (Optional[bool]): Whether to output attention probabilities. Defaults to False.

        Returns:
            Tuple[mindspore.Tensor]:
                A tuple containing the context layer tensor and optionally the attention probabilities tensor.
                The context layer tensor has shape (batch_size, sequence_length, hidden_size)
                and represents the output of the self-attention mechanism.

        Raises:
            None: This method does not raise any exceptions.

        """
        mixed_query_layer = self.query(hidden_states)

        # If this is instantiated as a cross-attention module, the keys
        # and values come from an encoder; the attention mask needs to be
        # such that the encoder's padding tokens are not attended to.
        is_cross_attention = encoder_hidden_states is not None

        if is_cross_attention and past_key_value is not None:
            # reuse k,v, cross_attentions
            key_layer = past_key_value[0]
            value_layer = past_key_value[1]
            attention_mask = encoder_attention_mask
        elif is_cross_attention:
            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
            attention_mask = encoder_attention_mask
        elif past_key_value is not None:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))
            key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
            value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
        else:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))

        query_layer = self.transpose_for_scores(mixed_query_layer)

        use_cache = past_key_value is not None
        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_layer, value_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            query_length, key_length = query_layer.shape[2], key_layer.shape[2]
            if use_cache:
                position_ids_l = mindspore.Tensor(key_length - 1, dtype=mindspore.int64).view(
                    -1, 1
                )
            else:
                position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
            position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
            distance = position_ids_l - position_ids_r

            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

            if self.position_embedding_type == "relative_key":
                relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores
            elif self.position_embedding_type == "relative_key_query":
                relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in ErnieModel forward() function)
            attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = ops.softmax(attention_scores, dim=-1)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = ops.matmul(attention_probs, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3)
        new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(new_context_layer_shape)

        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

        if self.is_decoder:
            outputs = outputs + (past_key_value,)
        return outputs

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfAttention.__init__(config, position_embedding_type=None)

Initialize the ErnieSelfAttention class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An instance of the configuration class containing the following attributes:

  • hidden_size (int): The size of the hidden layers.
  • num_attention_heads (int): The number of attention heads.
  • embedding_size (int, optional): The size of the embedding layer. If not provided, it is expected to be an attribute of the config.
  • attention_probs_dropout_prob (float): The dropout probability for attention probabilities.
  • position_embedding_type (str, optional): The type of position embedding. If not provided, it defaults to 'absolute'.
  • max_position_embeddings (int): The maximum number of position embeddings.
  • is_decoder (bool): Indicates if the model is a decoder.
position_embedding_type

The type of position embedding. Defaults to None.

TYPE: str DEFAULT: None

RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

If the hidden size is not a multiple of the number of attention heads and the config does not have an 'embedding_size' attribute.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def __init__(self, config, position_embedding_type=None):
    """
    Initialize the ErnieSelfAttention class.

    Args:
        self: The instance of the class.
        config:
            An instance of the configuration class containing the following attributes:

            - hidden_size (int): The size of the hidden layers.
            - num_attention_heads (int): The number of attention heads.
            - embedding_size (int, optional): The size of the embedding layer.
            If not provided, it is expected to be an attribute of the config.
            - attention_probs_dropout_prob (float): The dropout probability for attention probabilities.
            - position_embedding_type (str, optional): The type of position embedding.
            If not provided, it defaults to 'absolute'.
            - max_position_embeddings (int): The maximum number of position embeddings.
            - is_decoder (bool): Indicates if the model is a decoder.

        position_embedding_type (str, optional): The type of position embedding. Defaults to None.

    Returns:
        None.

    Raises:
        ValueError: If the hidden size is not a multiple of the number of attention heads and the config does not
            have an 'embedding_size' attribute.
    """
    super().__init__()
    if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
        raise ValueError(
            f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
            f"heads ({config.num_attention_heads})"
        )

    self.num_attention_heads = config.num_attention_heads
    self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
    self.all_head_size = self.num_attention_heads * self.attention_head_size

    self.query = nn.Linear(config.hidden_size, self.all_head_size)
    self.key = nn.Linear(config.hidden_size, self.all_head_size)
    self.value = nn.Linear(config.hidden_size, self.all_head_size)

    self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
    self.position_embedding_type = position_embedding_type or getattr(
        config, "position_embedding_type", "absolute"
    )
    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        self.max_position_embeddings = config.max_position_embeddings
        self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

    self.is_decoder = config.is_decoder

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

Constructs the self-attention mechanism for the ERNIE model.

PARAMETER DESCRIPTION
self

The instance of the ErnieSelfAttention class.

TYPE: ErnieSelfAttention

hidden_states

The input hidden states of the model. Shape: (batch_size, sequence_length, hidden_size).

TYPE: Tensor

attention_mask

The attention mask tensor. It is a binary tensor of shape (batch_size, sequence_length) where 1 indicates a valid token and 0 indicates a padded token. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

The head mask tensor. It is a binary tensor of shape (num_attention_heads,) indicating which heads to mask. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

The hidden states of the encoder. Shape: (batch_size, encoder_sequence_length, hidden_size). Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

The attention mask tensor for the encoder. Shape: (batch_size, encoder_sequence_length). Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

The cached key-value pairs from previous attention computations. Defaults to None.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

Whether to output attention probabilities. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: A tuple containing the context layer tensor and optionally the attention probabilities tensor. The context layer tensor has shape (batch_size, sequence_length, hidden_size) and represents the output of the self-attention mechanism.
RAISES DESCRIPTION
None

This method does not raise any exceptions.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    Constructs the self-attention mechanism for the ERNIE model.

    Args:
        self (ErnieSelfAttention): The instance of the ErnieSelfAttention class.
        hidden_states (mindspore.Tensor): The input hidden states of the model.
            Shape: (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]): The attention mask tensor.
            It is a binary tensor of shape (batch_size, sequence_length) where 1 indicates a valid token
            and 0 indicates a padded token. Defaults to None.
        head_mask (Optional[mindspore.Tensor]): The head mask tensor.
            It is a binary tensor of shape (num_attention_heads,) indicating which heads to mask. Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            The hidden states of the encoder. Shape: (batch_size, encoder_sequence_length, hidden_size). Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            The attention mask tensor for the encoder. Shape: (batch_size, encoder_sequence_length). Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            The cached key-value pairs from previous attention computations. Defaults to None.
        output_attentions (Optional[bool]): Whether to output attention probabilities. Defaults to False.

    Returns:
        Tuple[mindspore.Tensor]:
            A tuple containing the context layer tensor and optionally the attention probabilities tensor.
            The context layer tensor has shape (batch_size, sequence_length, hidden_size)
            and represents the output of the self-attention mechanism.

    Raises:
        None: This method does not raise any exceptions.

    """
    mixed_query_layer = self.query(hidden_states)

    # If this is instantiated as a cross-attention module, the keys
    # and values come from an encoder; the attention mask needs to be
    # such that the encoder's padding tokens are not attended to.
    is_cross_attention = encoder_hidden_states is not None

    if is_cross_attention and past_key_value is not None:
        # reuse k,v, cross_attentions
        key_layer = past_key_value[0]
        value_layer = past_key_value[1]
        attention_mask = encoder_attention_mask
    elif is_cross_attention:
        key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
        value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
        attention_mask = encoder_attention_mask
    elif past_key_value is not None:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))
        key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
        value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
    else:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))

    query_layer = self.transpose_for_scores(mixed_query_layer)

    use_cache = past_key_value is not None
    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_layer, value_layer)

    # Take the dot product between "query" and "key" to get the raw attention scores.
    attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        query_length, key_length = query_layer.shape[2], key_layer.shape[2]
        if use_cache:
            position_ids_l = mindspore.Tensor(key_length - 1, dtype=mindspore.int64).view(
                -1, 1
            )
        else:
            position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
        position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
        distance = position_ids_l - position_ids_r

        positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
        positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

        if self.position_embedding_type == "relative_key":
            relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
            attention_scores = attention_scores + relative_position_scores
        elif self.position_embedding_type == "relative_key_query":
            relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
            relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
            attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

    attention_scores = attention_scores / math.sqrt(self.attention_head_size)
    if attention_mask is not None:
        # Apply the attention mask is (precomputed for all layers in ErnieModel forward() function)
        attention_scores = attention_scores + attention_mask

    # Normalize the attention scores to probabilities.
    attention_probs = ops.softmax(attention_scores, dim=-1)

    # This is actually dropping out entire tokens to attend to, which might
    # seem a bit unusual, but is taken from the original Transformer paper.
    attention_probs = self.dropout(attention_probs)

    # Mask heads if we want to
    if head_mask is not None:
        attention_probs = attention_probs * head_mask

    context_layer = ops.matmul(attention_probs, value_layer)

    context_layer = context_layer.permute(0, 2, 1, 3)
    new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
    context_layer = context_layer.view(new_context_layer_shape)

    outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

    if self.is_decoder:
        outputs = outputs + (past_key_value,)
    return outputs

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfAttention.transpose_for_scores(x)

Transpose the input tensor for calculating attention scores.

PARAMETER DESCRIPTION
self

The instance of the ErnieSelfAttention class.

TYPE: ErnieSelfAttention

x

The input tensor with shape (batch_size, seq_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The transposed tensor with shape (batch_size, num_attention_heads, seq_length, attention_head_size).
RAISES DESCRIPTION
TypeError

If the input tensor is not of type mindspore.Tensor.
ValueError

If the input tensor shape is not compatible with the expected shape for transposition.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
def transpose_for_scores(self, x: mindspore.Tensor) -> mindspore.Tensor:
    """
    Transpose the input tensor for calculating attention scores.

    Args:
        self (ErnieSelfAttention): The instance of the ErnieSelfAttention class.
        x (mindspore.Tensor): The input tensor with shape (batch_size, seq_length, hidden_size).

    Returns:
        mindspore.Tensor:
            The transposed tensor with shape (batch_size, num_attention_heads, seq_length, attention_head_size).

    Raises:
        TypeError: If the input tensor is not of type mindspore.Tensor.
        ValueError: If the input tensor shape is not compatible with the expected shape for transposition.
    """
    new_x_shape = x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
    x = x.view(new_x_shape)
    return x.permute(0, 2, 1, 3)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfOutput

Bases: Module

The ErnieSelfOutput class represents a module for self-attention mechanism in ERNIE (Enhanced Representation through kNowledge Integration) model. This class inherits from nn.Module and contains methods to apply dense, layer normalization, and dropout operations to the input tensor.

ATTRIBUTE DESCRIPTION
dense

A dense layer to transform the input tensor's hidden states.

TYPE: Linear

LayerNorm

A layer normalization module to normalize the hidden states.

TYPE: LayerNorm

dropout

A dropout module to apply dropout to the hidden states.

TYPE: Dropout

METHOD DESCRIPTION
forward

Applies dense, dropout, and layer normalization operations to the input tensor's hidden states and returns the output tensor.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
class ErnieSelfOutput(nn.Module):

    """
    The ErnieSelfOutput class represents a module for self-attention mechanism in ERNIE (Enhanced Representation
    through kNowledge Integration) model.
    This class inherits from nn.Module and contains methods to apply dense, layer normalization, and dropout operations
    to the input tensor.

    Attributes:
        dense (nn.Linear): A dense layer to transform the input tensor's hidden states.
        LayerNorm (nn.LayerNorm): A layer normalization module to normalize the hidden states.
        dropout (nn.Dropout): A dropout module to apply dropout to the hidden states.

    Methods:
        forward:
            Applies dense, dropout, and layer normalization operations to the input tensor's hidden states
            and returns the output tensor.
    """
    def __init__(self, config):
        """
        Initializes an instance of the ErnieSelfOutput class.

        Args:
            self (ErnieSelfOutput): The instance of the ErnieSelfOutput class.
            config (object): An object containing configuration parameters for the ErnieSelfOutput instance.

        Returns:
            None.

        Raises:
            ValueError: If the configuration parameters are invalid or inconsistent.
            TypeError: If the configuration object is not of the expected type.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the output of the ERNIE self-attention layer.

        Args:
            self (ErnieSelfOutput): An instance of the ErnieSelfOutput class.
            hidden_states (mindspore.Tensor): The hidden states tensor.
                It should have a shape of (batch_size, sequence_length, hidden_size).
            input_tensor (mindspore.Tensor): The input tensor.
                It should have the same shape as the hidden_states tensor.

        Returns:
            mindspore.Tensor: The output tensor of the ERNIE self-attention layer.
                It has the same shape as the input_tensor.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfOutput.__init__(config)

Initializes an instance of the ErnieSelfOutput class.

PARAMETER DESCRIPTION
self

The instance of the ErnieSelfOutput class.

TYPE: ErnieSelfOutput

config

An object containing configuration parameters for the ErnieSelfOutput instance.

TYPE: object

RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

If the configuration parameters are invalid or inconsistent.
TypeError

If the configuration object is not of the expected type.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
def __init__(self, config):
    """
    Initializes an instance of the ErnieSelfOutput class.

    Args:
        self (ErnieSelfOutput): The instance of the ErnieSelfOutput class.
        config (object): An object containing configuration parameters for the ErnieSelfOutput instance.

    Returns:
        None.

    Raises:
        ValueError: If the configuration parameters are invalid or inconsistent.
        TypeError: If the configuration object is not of the expected type.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.ernie.modeling_ernie.ErnieSelfOutput.forward(hidden_states, input_tensor)

Constructs the output of the ERNIE self-attention layer.

PARAMETER DESCRIPTION
self

An instance of the ErnieSelfOutput class.

TYPE: ErnieSelfOutput

hidden_states

The hidden states tensor. It should have a shape of (batch_size, sequence_length, hidden_size).

TYPE: Tensor

input_tensor

The input tensor. It should have the same shape as the hidden_states tensor.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The output tensor of the ERNIE self-attention layer. It has the same shape as the input_tensor.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the output of the ERNIE self-attention layer.

    Args:
        self (ErnieSelfOutput): An instance of the ErnieSelfOutput class.
        hidden_states (mindspore.Tensor): The hidden states tensor.
            It should have a shape of (batch_size, sequence_length, hidden_size).
        input_tensor (mindspore.Tensor): The input tensor.
            It should have the same shape as the hidden_states tensor.

    Returns:
        mindspore.Tensor: The output tensor of the ERNIE self-attention layer.
            It has the same shape as the input_tensor.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_ernie.UIE

Bases: ErniePreTrainedModel

Ernie Model with two linear layer on top of the hidden-states output to compute start_prob and end_prob, designed for Universal Information Extraction. Args: config (:class:ErnieConfig): An instance of ErnieConfig used to forward UIE

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
class UIE(ErniePreTrainedModel):
    """
    Ernie Model with two linear layer on top of the hidden-states
    output to compute `start_prob` and `end_prob`,
    designed for Universal Information Extraction.
    Args:
        config (:class:`ErnieConfig`):
            An instance of ErnieConfig used to forward UIE
    """
    def __init__(self, config: ErnieConfig):
        """
        Initializes an instance of the UIE class.

        Args:
            self (UIE): The instance of the UIE class.
            config (ErnieConfig): An instance of ErnieConfig containing the configuration parameters for the UIE model.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.ernie = ErnieModel(config)
        self.linear_start = nn.Linear(config.hidden_size, 1)
        self.linear_end = nn.Linear(config.hidden_size, 1)
        self.sigmoid = nn.Sigmoid()

        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None
    ):
        r"""
        Args:
            input_ids (Tensor):
                See :class:`ErnieModel`.
            token_type_ids (Tensor, optional):
                See :class:`ErnieModel`.
            position_ids (Tensor, optional):
                See :class:`ErnieModel`.
            attention_mask (Tensor, optional):
                See :class:`ErnieModel`.

        Example:
            ```python
            >>> import paddle
            >>> from paddlenlp.transformers import UIE, ErnieTokenizer
            >>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
            >>> model = UIE.from_pretrained('uie-base')
            >>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
            >>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
            >>> start_prob, end_prob = model(**inputs)
            ```
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        outputs = self.ernie(
            input_ids=input_ids,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            attention_mask=attention_mask,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict
        )

        sequence_output = outputs[0]

        start_logits = self.linear_start(sequence_output)
        start_logits = ops.squeeze(start_logits, -1)
        start_prob = self.sigmoid(start_logits)
        end_logits = self.linear_end(sequence_output)
        end_logits = ops.squeeze(end_logits, -1)
        end_prob = self.sigmoid(end_logits)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            start_loss = F.binary_cross_entropy_with_logits(start_prob, start_positions)
            end_loss = F.binary_cross_entropy_with_logits(end_prob, end_positions)
            total_loss = (start_loss + end_loss) / 2.0

        if not return_dict:
            output = (start_prob, end_prob) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return UIEModelOutput(
            loss=total_loss,
            start_prob=start_prob,
            end_prob=end_prob,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.ernie.modeling_ernie.UIE.__init__(config)

Initializes an instance of the UIE class.

PARAMETER DESCRIPTION
self

The instance of the UIE class.

TYPE: UIE

config

An instance of ErnieConfig containing the configuration parameters for the UIE model.

TYPE: ErnieConfig

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
def __init__(self, config: ErnieConfig):
    """
    Initializes an instance of the UIE class.

    Args:
        self (UIE): The instance of the UIE class.
        config (ErnieConfig): An instance of ErnieConfig containing the configuration parameters for the UIE model.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.ernie = ErnieModel(config)
    self.linear_start = nn.Linear(config.hidden_size, 1)
    self.linear_end = nn.Linear(config.hidden_size, 1)
    self.sigmoid = nn.Sigmoid()

    self.post_init()

mindnlp.transformers.models.ernie.modeling_ernie.UIE.forward(input_ids=None, token_type_ids=None, position_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER DESCRIPTION
input_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

token_type_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

position_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

attention_mask

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

Example 
>>> import paddle
>>> from paddlenlp.transformers import UIE, ErnieTokenizer
>>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
>>> model = UIE.from_pretrained('uie-base')
>>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
>>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
>>> start_prob, end_prob = model(**inputs)
Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None
):
    r"""
    Args:
        input_ids (Tensor):
            See :class:`ErnieModel`.
        token_type_ids (Tensor, optional):
            See :class:`ErnieModel`.
        position_ids (Tensor, optional):
            See :class:`ErnieModel`.
        attention_mask (Tensor, optional):
            See :class:`ErnieModel`.

    Example:
        ```python
        >>> import paddle
        >>> from paddlenlp.transformers import UIE, ErnieTokenizer
        >>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
        >>> model = UIE.from_pretrained('uie-base')
        >>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
        >>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
        >>> start_prob, end_prob = model(**inputs)
        ```
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    outputs = self.ernie(
        input_ids=input_ids,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        attention_mask=attention_mask,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict
    )

    sequence_output = outputs[0]

    start_logits = self.linear_start(sequence_output)
    start_logits = ops.squeeze(start_logits, -1)
    start_prob = self.sigmoid(start_logits)
    end_logits = self.linear_end(sequence_output)
    end_logits = ops.squeeze(end_logits, -1)
    end_prob = self.sigmoid(end_logits)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        start_loss = F.binary_cross_entropy_with_logits(start_prob, start_positions)
        end_loss = F.binary_cross_entropy_with_logits(end_prob, end_positions)
        total_loss = (start_loss + end_loss) / 2.0

    if not return_dict:
        output = (start_prob, end_prob) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return UIEModelOutput(
        loss=total_loss,
        start_prob=start_prob,
        end_prob=end_prob,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.ernie.modeling_ernie.UIEModelOutput dataclass

Bases: ModelOutput

Output class for outputs of UIE.

PARAMETER DESCRIPTION
loss

Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.

TYPE: `mindspore.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided DEFAULT: None

start_prob

Span-start scores (after Sigmoid).

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None

end_prob

Span-end scores (after Sigmoid).

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None

hidden_states

Tuple of mindspore.Tensor (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size). Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.

TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None

attentions

Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length). Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.

TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_ernie.py 
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
@dataclass
class UIEModelOutput(ModelOutput):
    """
    Output class for outputs of UIE.

    Args:
        loss (`mindspore.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
        start_prob (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Span-start scores (after Sigmoid).
        end_prob (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Span-end scores (after Sigmoid).
        hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    """
    loss: Optional[mindspore.Tensor] = None
    start_prob: mindspore.Tensor = None
    end_prob: mindspore.Tensor = None
    hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    attentions: Optional[Tuple[mindspore.Tensor]] = None

mindnlp.transformers.models.ernie.modeling_graph_ernie

MindSpore ERNIE model.

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieAttention

Bases: Module

This class represents the attention mechanism used in the MSErnie model. It is responsible for calculating the attention scores between the input sequence and itself or encoder hidden states. The attention scores are then used to weigh the importance of different parts of the input sequence during the model's computation.

This class inherits from the nn.Module class.

METHOD DESCRIPTION
__init__

Initializes the MSErnieAttention instance.
prune_heads

Prunes the specified attention heads from the model.
forward

Constructs the attention mechanism by calculating attention scores and applying them to the input sequence.
ATTRIBUTE DESCRIPTION
self

An instance of MSErnieSelfAttention, representing the self-attention mechanism.
self_attn

An instance of MSErnieSelfAttention, representing the self-attention mechanism (used in older versions of MindSpore).
output

An instance of MSErnieSelfOutput, representing the output layer of the attention mechanism.
pruned_heads

A set that stores the indices of the pruned attention heads.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
class MSErnieAttention(nn.Module):

    """
    This class represents the attention mechanism used in the MSErnie model.
    It is responsible for calculating the attention scores between the input sequence and itself or encoder hidden states.
    The attention scores are then used to weigh the importance of different parts of the input sequence during the
    model's computation.

    This class inherits from the nn.Module class.

    Methods:
        __init__: Initializes the MSErnieAttention instance.
        prune_heads: Prunes the specified attention heads from the model.
        forward: Constructs the attention mechanism by calculating attention scores and applying them to the
            input sequence.

    Attributes:
        self: An instance of MSErnieSelfAttention, representing the self-attention mechanism.
        self_attn: An instance of MSErnieSelfAttention,
            representing the self-attention mechanism (used in older versions of MindSpore).
        output: An instance of MSErnieSelfOutput, representing the output layer of the attention mechanism.
        pruned_heads: A set that stores the indices of the pruned attention heads.

    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes an instance of the MSErnieAttention class.

        Args:
            self: The object itself.
            config: An object of type 'config' containing configuration settings.
            position_embedding_type: (Optional) A string specifying the type of position embedding. Default is None.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.self = MSErnieSelfAttention(config, position_embedding_type=position_embedding_type)
        self.output = MSErnieSelfOutput(config)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        """
        Prunes the specified attention heads from the MSErnieAttention layer.

        Args:
            self (MSErnieAttention): The instance of the MSErnieAttention class.
            heads (List[int]): A list of attention head indices to be pruned.

        Returns:
            None

        Raises:
            None
        """
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
        )

        # Prune linear layers
        self.self.query = prune_linear_layer(self.self.query, index)
        self.self.key = prune_linear_layer(self.self.key, index)
        self.self.value = prune_linear_layer(self.self.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        This method forwards the MSErnieAttention module.

        Args:
            self: The instance of the class.
            hidden_states (mindspore.Tensor):
                The input tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states.
            attention_mask (Optional[mindspore.Tensor]):
                An optional input tensor of shape (batch_size, sequence_length) representing the attention mask for
                the input sequence. Defaults to None.
            head_mask (Optional[mindspore.Tensor]):
                An optional tensor of shape (num_heads,) representing the mask for the attention heads.
                Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                An optional tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of
                the encoder. Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                An optional tensor of shape (batch_size, sequence_length) representing the attention mask for the
                encoder hidden states. Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                An optional tuple containing the past key and value tensors for fast decoding. Defaults to None.
            output_attentions (Optional[bool]): A boolean flag indicating whether to output attentions. Defaults to False.

        Returns:
            Tuple[mindspore.Tensor]: A tuple containing the attention output tensor of shape
                (batch_size, sequence_length, hidden_size).

        Raises:
            No specific exceptions are raised by this method.
        """
        self_outputs = self.self(
            hidden_states,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieAttention.__init__(config, position_embedding_type=None)

Initializes an instance of the MSErnieAttention class.

PARAMETER DESCRIPTION
self

The object itself.
config

An object of type 'config' containing configuration settings.
position_embedding_type

(Optional) A string specifying the type of position embedding. Default is None.

DEFAULT: None

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
def __init__(self, config, position_embedding_type=None):
    """
    Initializes an instance of the MSErnieAttention class.

    Args:
        self: The object itself.
        config: An object of type 'config' containing configuration settings.
        position_embedding_type: (Optional) A string specifying the type of position embedding. Default is None.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.self = MSErnieSelfAttention(config, position_embedding_type=position_embedding_type)
    self.output = MSErnieSelfOutput(config)
    self.pruned_heads = set()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards the MSErnieAttention module.

PARAMETER DESCRIPTION
self

The instance of the class.
hidden_states

The input tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states.

TYPE: Tensor

attention_mask

An optional input tensor of shape (batch_size, sequence_length) representing the attention mask for the input sequence. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

An optional tensor of shape (num_heads,) representing the mask for the attention heads. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

An optional tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

An optional tensor of shape (batch_size, sequence_length) representing the attention mask for the encoder hidden states. Defaults to None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

An optional tuple containing the past key and value tensors for fast decoding. Defaults to None.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

A boolean flag indicating whether to output attentions. Defaults to False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: A tuple containing the attention output tensor of shape (batch_size, sequence_length, hidden_size).
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    This method forwards the MSErnieAttention module.

    Args:
        self: The instance of the class.
        hidden_states (mindspore.Tensor):
            The input tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states.
        attention_mask (Optional[mindspore.Tensor]):
            An optional input tensor of shape (batch_size, sequence_length) representing the attention mask for
            the input sequence. Defaults to None.
        head_mask (Optional[mindspore.Tensor]):
            An optional tensor of shape (num_heads,) representing the mask for the attention heads.
            Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            An optional tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of
            the encoder. Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            An optional tensor of shape (batch_size, sequence_length) representing the attention mask for the
            encoder hidden states. Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            An optional tuple containing the past key and value tensors for fast decoding. Defaults to None.
        output_attentions (Optional[bool]): A boolean flag indicating whether to output attentions. Defaults to False.

    Returns:
        Tuple[mindspore.Tensor]: A tuple containing the attention output tensor of shape
            (batch_size, sequence_length, hidden_size).

    Raises:
        No specific exceptions are raised by this method.
    """
    self_outputs = self.self(
        hidden_states,
        attention_mask,
        head_mask,
        encoder_hidden_states,
        encoder_attention_mask,
        past_key_value,
        output_attentions,
    )
    attention_output = self.output(self_outputs[0], hidden_states)
    outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieAttention.prune_heads(heads)

Prunes the specified attention heads from the MSErnieAttention layer.

PARAMETER DESCRIPTION
self

The instance of the MSErnieAttention class.

TYPE: MSErnieAttention

heads

A list of attention head indices to be pruned.

TYPE: List[int]

RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
def prune_heads(self, heads):
    """
    Prunes the specified attention heads from the MSErnieAttention layer.

    Args:
        self (MSErnieAttention): The instance of the MSErnieAttention class.
        heads (List[int]): A list of attention head indices to be pruned.

    Returns:
        None

    Raises:
        None
    """
    if len(heads) == 0:
        return
    heads, index = find_pruneable_heads_and_indices(
        heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
    )

    # Prune linear layers
    self.self.query = prune_linear_layer(self.self.query, index)
    self.self.key = prune_linear_layer(self.self.key, index)
    self.self.value = prune_linear_layer(self.self.value, index)
    self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

    # Update hyper params and store pruned heads
    self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
    self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
    self.pruned_heads = self.pruned_heads.union(heads)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEmbeddings

Bases: Module

Construct the embeddings from word, position and token_type embeddings.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
class MSErnieEmbeddings(nn.Module):
    """Construct the embeddings from word, position and token_type embeddings."""
    def __init__(self, config):
        """
        Initializes an instance of the `MSErnieEmbeddings` class.

        Args:
            self: The object itself.
            config: An instance of the `Config` class containing the configuration parameters for the embeddings.

        Returns:
            None

        Raises:
            None

        Description:
            This method initializes the `MSErnieEmbeddings` object by setting up the necessary embedding layers and other
            attributes. It takes in the `config` object which contains the configuration parameters for the embeddings.

            - `word_embeddings`: A `nn.Embedding` layer that maps word indices to word embeddings. It has dimensions
            (config.vocab_size, config.hidden_size) and uses the `config.pad_token_id` as the padding index.
            - `position_embeddings`: A `nn.Embedding` layer that maps position indices to position embeddings.
            It has dimensions (config.max_position_embeddings, config.hidden_size).
            - `token_type_embeddings`: A `nn.Embedding` layer that maps token type indices to token type embeddings.
            It has dimensions (config.type_vocab_size, config.hidden_size).
            - `use_task_id`: A boolean indicating whether to use task type embeddings. If `True`, an additional
            `task_type_embeddings` layer is created with dimensions (config.task_type_vocab_size, config.hidden_size).
            - `LayerNorm`: A `nn.LayerNorm` layer that applies layer normalization to the embeddings.
            It has dimensions [config.hidden_size] and uses `config.layer_norm_eps` as epsilon.
            - `dropout`:
                A `nn.Dropout` layer that applies dropout to the embeddings with probability `config.hidden_dropout_prob`.
            - `position_embedding_type`:
                A string indicating the type of position embeddings to use. It defaults to 'absolute'.
            - `position_ids`: A tensor containing the position indices.
                It is created using `ops.arange` and has dimensions (1, config.max_position_embeddings).
            - `token_type_ids`: A tensor containing the token type indices. It is created using `ops.zeros` with the same
            dimensions as `position_ids`.
        """
        super().__init__()
        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
        self.use_task_id = config.use_task_id
        if config.use_task_id:
            self.task_type_embeddings = nn.Embedding(config.task_type_vocab_size, config.hidden_size)

        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
        # any TensorFlow checkpoint file
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
        self.position_ids = ops.arange(config.max_position_embeddings).broadcast_to((1, -1))
        self.token_type_ids = ops.zeros(self.position_ids.shape, dtype=mindspore.int64)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        past_key_values_length: int = 0,
    ) -> mindspore.Tensor:
        """
        Constructs the MSErnie embeddings for the given input.

        Args:
            self (MSErnieEmbeddings): The instance of the MSErnieEmbeddings class.
            input_ids (Optional[mindspore.Tensor]): The input tensor containing the token ids. Default is None.
            token_type_ids (Optional[mindspore.Tensor]): The tensor containing the token type ids. Default is None.
            task_type_ids (Optional[mindspore.Tensor]): The tensor containing the task type ids. Default is None.
            position_ids (Optional[mindspore.Tensor]): The tensor containing the position ids. Default is None.
            inputs_embeds (Optional[mindspore.Tensor]): The tensor containing the input embeddings. Default is None.
            past_key_values_length (int): The length of past key values. Default is 0.

        Returns:
            mindspore.Tensor: The tensor representing the forwarded embeddings.

        Raises:
            None
        """
        if input_ids is not None:
            input_shape = input_ids.shape
        else:
            input_shape = inputs_embeds.shape[:-1]

        seq_length = input_shape[1]

        if position_ids is None:
            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

        # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
        # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
        # issue #5664
        if token_type_ids is None:
            if hasattr(self, "token_type_ids"):
                buffered_token_type_ids = self.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((input_shape[0], seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)

        embeddings = inputs_embeds + token_type_embeddings
        if self.position_embedding_type == "absolute":
            position_embeddings = self.position_embeddings(position_ids)
            embeddings += position_embeddings

        # add `task_type_id` for ERNIE model
        if self.use_task_id:
            if task_type_ids is None:
                task_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)
            task_type_embeddings = self.task_type_embeddings(task_type_ids)
            embeddings += task_type_embeddings

        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEmbeddings.__init__(config)

Initializes an instance of the MSErnieEmbeddings class.

PARAMETER DESCRIPTION
self

The object itself.
config

An instance of the Config class containing the configuration parameters for the embeddings.
RETURNS DESCRIPTION

None
Description

This method initializes the MSErnieEmbeddings object by setting up the necessary embedding layers and other attributes. It takes in the config object which contains the configuration parameters for the embeddings.

  • word_embeddings: A nn.Embedding layer that maps word indices to word embeddings. It has dimensions (config.vocab_size, config.hidden_size) and uses the config.pad_token_id as the padding index.
  • position_embeddings: A nn.Embedding layer that maps position indices to position embeddings. It has dimensions (config.max_position_embeddings, config.hidden_size).
  • token_type_embeddings: A nn.Embedding layer that maps token type indices to token type embeddings. It has dimensions (config.type_vocab_size, config.hidden_size).
  • use_task_id: A boolean indicating whether to use task type embeddings. If True, an additional task_type_embeddings layer is created with dimensions (config.task_type_vocab_size, config.hidden_size).
  • LayerNorm: A nn.LayerNorm layer that applies layer normalization to the embeddings. It has dimensions [config.hidden_size] and uses config.layer_norm_eps as epsilon.
  • dropout: A nn.Dropout layer that applies dropout to the embeddings with probability config.hidden_dropout_prob.
  • position_embedding_type: A string indicating the type of position embeddings to use. It defaults to 'absolute'.
  • position_ids: A tensor containing the position indices. It is created using ops.arange and has dimensions (1, config.max_position_embeddings).
  • token_type_ids: A tensor containing the token type indices. It is created using ops.zeros with the same dimensions as position_ids.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def __init__(self, config):
    """
    Initializes an instance of the `MSErnieEmbeddings` class.

    Args:
        self: The object itself.
        config: An instance of the `Config` class containing the configuration parameters for the embeddings.

    Returns:
        None

    Raises:
        None

    Description:
        This method initializes the `MSErnieEmbeddings` object by setting up the necessary embedding layers and other
        attributes. It takes in the `config` object which contains the configuration parameters for the embeddings.

        - `word_embeddings`: A `nn.Embedding` layer that maps word indices to word embeddings. It has dimensions
        (config.vocab_size, config.hidden_size) and uses the `config.pad_token_id` as the padding index.
        - `position_embeddings`: A `nn.Embedding` layer that maps position indices to position embeddings.
        It has dimensions (config.max_position_embeddings, config.hidden_size).
        - `token_type_embeddings`: A `nn.Embedding` layer that maps token type indices to token type embeddings.
        It has dimensions (config.type_vocab_size, config.hidden_size).
        - `use_task_id`: A boolean indicating whether to use task type embeddings. If `True`, an additional
        `task_type_embeddings` layer is created with dimensions (config.task_type_vocab_size, config.hidden_size).
        - `LayerNorm`: A `nn.LayerNorm` layer that applies layer normalization to the embeddings.
        It has dimensions [config.hidden_size] and uses `config.layer_norm_eps` as epsilon.
        - `dropout`:
            A `nn.Dropout` layer that applies dropout to the embeddings with probability `config.hidden_dropout_prob`.
        - `position_embedding_type`:
            A string indicating the type of position embeddings to use. It defaults to 'absolute'.
        - `position_ids`: A tensor containing the position indices.
            It is created using `ops.arange` and has dimensions (1, config.max_position_embeddings).
        - `token_type_ids`: A tensor containing the token type indices. It is created using `ops.zeros` with the same
        dimensions as `position_ids`.
    """
    super().__init__()
    self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
    self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
    self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
    self.use_task_id = config.use_task_id
    if config.use_task_id:
        self.task_type_embeddings = nn.Embedding(config.task_type_vocab_size, config.hidden_size)

    # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
    # any TensorFlow checkpoint file
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
    # position_ids (1, len position emb) is contiguous in memory and exported when serialized
    self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
    self.position_ids = ops.arange(config.max_position_embeddings).broadcast_to((1, -1))
    self.token_type_ids = ops.zeros(self.position_ids.shape, dtype=mindspore.int64)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEmbeddings.forward(input_ids=None, token_type_ids=None, task_type_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0)

Constructs the MSErnie embeddings for the given input.

PARAMETER DESCRIPTION
self

The instance of the MSErnieEmbeddings class.

TYPE: MSErnieEmbeddings

input_ids

The input tensor containing the token ids. Default is None.

TYPE: Optional[Tensor] DEFAULT: None

token_type_ids

The tensor containing the token type ids. Default is None.

TYPE: Optional[Tensor] DEFAULT: None

task_type_ids

The tensor containing the task type ids. Default is None.

TYPE: Optional[Tensor] DEFAULT: None

position_ids

The tensor containing the position ids. Default is None.

TYPE: Optional[Tensor] DEFAULT: None

inputs_embeds

The tensor containing the input embeddings. Default is None.

TYPE: Optional[Tensor] DEFAULT: None

past_key_values_length

The length of past key values. Default is 0.

TYPE: int DEFAULT: 0

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The tensor representing the forwarded embeddings.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    past_key_values_length: int = 0,
) -> mindspore.Tensor:
    """
    Constructs the MSErnie embeddings for the given input.

    Args:
        self (MSErnieEmbeddings): The instance of the MSErnieEmbeddings class.
        input_ids (Optional[mindspore.Tensor]): The input tensor containing the token ids. Default is None.
        token_type_ids (Optional[mindspore.Tensor]): The tensor containing the token type ids. Default is None.
        task_type_ids (Optional[mindspore.Tensor]): The tensor containing the task type ids. Default is None.
        position_ids (Optional[mindspore.Tensor]): The tensor containing the position ids. Default is None.
        inputs_embeds (Optional[mindspore.Tensor]): The tensor containing the input embeddings. Default is None.
        past_key_values_length (int): The length of past key values. Default is 0.

    Returns:
        mindspore.Tensor: The tensor representing the forwarded embeddings.

    Raises:
        None
    """
    if input_ids is not None:
        input_shape = input_ids.shape
    else:
        input_shape = inputs_embeds.shape[:-1]

    seq_length = input_shape[1]

    if position_ids is None:
        position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

    # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
    # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
    # issue #5664
    if token_type_ids is None:
        if hasattr(self, "token_type_ids"):
            buffered_token_type_ids = self.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((input_shape[0], seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

    if inputs_embeds is None:
        inputs_embeds = self.word_embeddings(input_ids)
    token_type_embeddings = self.token_type_embeddings(token_type_ids)

    embeddings = inputs_embeds + token_type_embeddings
    if self.position_embedding_type == "absolute":
        position_embeddings = self.position_embeddings(position_ids)
        embeddings += position_embeddings

    # add `task_type_id` for ERNIE model
    if self.use_task_id:
        if task_type_ids is None:
            task_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)
        task_type_embeddings = self.task_type_embeddings(task_type_ids)
        embeddings += task_type_embeddings

    embeddings = self.LayerNorm(embeddings)
    embeddings = self.dropout(embeddings)
    return embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEncoder

Bases: Module

MSErnieEncoder represents a customized encoder for the MSErnie model that inherits from nn.Module.

ATTRIBUTE DESCRIPTION
config

A dictionary containing configuration parameters for the encoder.
layer

A CellList containing MSErnieLayer instances for each hidden layer in the encoder.
gradient_checkpointing

A boolean indicating whether gradient checkpointing is enabled in the encoder.
METHOD DESCRIPTION
__init__

Initializes the MSErnieEncoder with the given configuration.
forward

Constructs the forward
RETURNS DESCRIPTION

Union[Tuple[mindspore.Tensor], dict]: A tuple containing relevant output tensors or a dictionary with optional outputs based on the method parameters.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
class MSErnieEncoder(nn.Module):

    """
    MSErnieEncoder represents a customized encoder for the MSErnie model that inherits from nn.Module.

    Attributes:
        config: A dictionary containing configuration parameters for the encoder.
        layer: A CellList containing MSErnieLayer instances for each hidden layer in the encoder.
        gradient_checkpointing: A boolean indicating whether gradient checkpointing is enabled in the encoder.

    Methods:
        __init__: Initializes the MSErnieEncoder with the given configuration.
        forward: Constructs the forward
        pass of the encoder with optional outputs based on the input parameters.

    Returns:
        Union[Tuple[mindspore.Tensor], dict]:
            A tuple containing relevant output tensors or a dictionary with optional outputs based on the method parameters.
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieEncoder class.

        Args:
            self (object): The instance of the MSErnieEncoder class.
            config (object): Configuration object containing parameters for the MSErnieEncoder.
                This object should include the following attributes:

                - num_hidden_layers (int): Number of hidden layers for the encoder.
                - Other configuration parameters specific to the MSErnieEncoder.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList([MSErnieLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = False,
        output_hidden_states: Optional[bool] = False,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        """
        This method forwards the MSErnie encoder with the provided input parameters and returns the
        output hidden states, decoder cache, all hidden states, self attentions, and cross attentions.

        Args:
            self: The instance of the MSErnieEncoder class.
            hidden_states (mindspore.Tensor): The input hidden states to the encoder.
            attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens.
            head_mask (Optional[mindspore.Tensor]): Mask for masked multi-head attention.
            encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder.
            encoder_attention_mask (Optional[mindspore.Tensor]): Mask for encoder attention.
            past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Tuple of past key values for fast decoding.
            use_cache (Optional[bool]): Flag to use the cache for decoding.
            output_attentions (Optional[bool]): Flag to output attentions.
            output_hidden_states (Optional[bool]): Flag to output hidden states.

        Returns:
            Union[Tuple[mindspore.Tensor], dict]: Depending on the output flags, returns a tuple containing hidden states,
                next decoder cache, all hidden states, self attentions, and cross attentions. If any of these values are
                None, they are excluded from the tuple.

        Raises:
            None

        """
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None
        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

        next_decoder_cache = () if use_cache else None
        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None
            past_key_value = past_key_values[i] if past_key_values is not None else None

            layer_outputs = layer_module(
                hidden_states,
                attention_mask,
                layer_head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                past_key_value,
                output_attentions,
            )

            hidden_states = layer_outputs[0]
            if use_cache:
                next_decoder_cache += (layer_outputs[-1],)
            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)
                if self.config.add_cross_attention:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        return tuple(
            v
            for v in [
                hidden_states,
                next_decoder_cache,
                all_hidden_states,
                all_self_attentions,
                all_cross_attentions,
            ]
            if v is not None
        )

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEncoder.__init__(config)

Initializes an instance of the MSErnieEncoder class.

PARAMETER DESCRIPTION
self

The instance of the MSErnieEncoder class.

TYPE: object

config

Configuration object containing parameters for the MSErnieEncoder. This object should include the following attributes:

  • num_hidden_layers (int): Number of hidden layers for the encoder.
  • Other configuration parameters specific to the MSErnieEncoder.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
def __init__(self, config):
    """
    Initializes an instance of the MSErnieEncoder class.

    Args:
        self (object): The instance of the MSErnieEncoder class.
        config (object): Configuration object containing parameters for the MSErnieEncoder.
            This object should include the following attributes:

            - num_hidden_layers (int): Number of hidden layers for the encoder.
            - Other configuration parameters specific to the MSErnieEncoder.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.config = config
    self.layer = nn.ModuleList([MSErnieLayer(config) for _ in range(config.num_hidden_layers)])
    self.gradient_checkpointing = False

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieEncoder.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=False, output_hidden_states=False)

This method forwards the MSErnie encoder with the provided input parameters and returns the output hidden states, decoder cache, all hidden states, self attentions, and cross attentions.

PARAMETER DESCRIPTION
self

The instance of the MSErnieEncoder class.
hidden_states

The input hidden states to the encoder.

TYPE: Tensor

attention_mask

Mask to avoid attention on padding tokens.

TYPE: Optional[Tensor] DEFAULT: None

head_mask

Mask for masked multi-head attention.

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

The hidden states of the encoder.

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

Mask for encoder attention.

TYPE: Optional[Tensor] DEFAULT: None

past_key_values

Tuple of past key values for fast decoding.

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

use_cache

Flag to use the cache for decoding.

TYPE: Optional[bool] DEFAULT: None

output_attentions

Flag to output attentions.

TYPE: Optional[bool] DEFAULT: False

output_hidden_states

Flag to output hidden states.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Union[Tuple[Tensor], dict]

Union[Tuple[mindspore.Tensor], dict]: Depending on the output flags, returns a tuple containing hidden states, next decoder cache, all hidden states, self attentions, and cross attentions. If any of these values are None, they are excluded from the tuple.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = False,
    output_hidden_states: Optional[bool] = False,
) -> Union[Tuple[mindspore.Tensor], dict]:
    """
    This method forwards the MSErnie encoder with the provided input parameters and returns the
    output hidden states, decoder cache, all hidden states, self attentions, and cross attentions.

    Args:
        self: The instance of the MSErnieEncoder class.
        hidden_states (mindspore.Tensor): The input hidden states to the encoder.
        attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens.
        head_mask (Optional[mindspore.Tensor]): Mask for masked multi-head attention.
        encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder.
        encoder_attention_mask (Optional[mindspore.Tensor]): Mask for encoder attention.
        past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Tuple of past key values for fast decoding.
        use_cache (Optional[bool]): Flag to use the cache for decoding.
        output_attentions (Optional[bool]): Flag to output attentions.
        output_hidden_states (Optional[bool]): Flag to output hidden states.

    Returns:
        Union[Tuple[mindspore.Tensor], dict]: Depending on the output flags, returns a tuple containing hidden states,
            next decoder cache, all hidden states, self attentions, and cross attentions. If any of these values are
            None, they are excluded from the tuple.

    Raises:
        None

    """
    all_hidden_states = () if output_hidden_states else None
    all_self_attentions = () if output_attentions else None
    all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

    next_decoder_cache = () if use_cache else None
    for i, layer_module in enumerate(self.layer):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        layer_head_mask = head_mask[i] if head_mask is not None else None
        past_key_value = past_key_values[i] if past_key_values is not None else None

        layer_outputs = layer_module(
            hidden_states,
            attention_mask,
            layer_head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )

        hidden_states = layer_outputs[0]
        if use_cache:
            next_decoder_cache += (layer_outputs[-1],)
        if output_attentions:
            all_self_attentions = all_self_attentions + (layer_outputs[1],)
            if self.config.add_cross_attention:
                all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    return tuple(
        v
        for v in [
            hidden_states,
            next_decoder_cache,
            all_hidden_states,
            all_self_attentions,
            all_cross_attentions,
        ]
        if v is not None
    )

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM

Bases: MSErniePreTrainedModel

MSErnieForCausalLM

This class is an implementation of the MSErnie model for causal language modeling (LM). It inherits from the MSErniePreTrainedModel class.

ATTRIBUTE DESCRIPTION
ernie

The main MSErnie model.

TYPE: MSErnieModel

cls

The MLM head for generating predictions.

TYPE: MSErnieOnlyMLMHead

METHOD DESCRIPTION
__init__

Initializes the MSErnieForCausalLM class.
get_output_embeddings

Retrieves the output embeddings of the model.
set_output_embeddings

Sets the output embeddings of the model.
forward

Constructs the MSErnie model for causal language modeling.
prepare_inputs_for_generation

Prepares the inputs for text generation.
_reorder_cache

Reorders the cache for beam search decoding.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
class MSErnieForCausalLM(MSErniePreTrainedModel):

    """
    MSErnieForCausalLM
    ------------------

    This class is an implementation of the MSErnie model for causal language modeling (LM).
    It inherits from the MSErniePreTrainedModel class.

    Attributes:
        ernie (MSErnieModel): The main MSErnie model.
        cls (MSErnieOnlyMLMHead): The MLM head for generating predictions.

    Methods:
        __init__: Initializes the MSErnieForCausalLM class.
        get_output_embeddings: Retrieves the output embeddings of the model.
        set_output_embeddings: Sets the output embeddings of the model.
        forward: Constructs the MSErnie model for causal language modeling.
        prepare_inputs_for_generation: Prepares the inputs for text generation.
        _reorder_cache: Reorders the cache for beam search decoding.
    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.__init__ with BertLMHeadModel->ErnieForCausalLM,Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of MSErnieForCausalLM class.

        Args:
            self (object): The instance of the class.
            config (object):
                An object containing configuration settings for the model.

                - Type: Config
                - Purpose: Specifies the model configuration parameters.
                - Restrictions: Must be provided to initialize the model.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        if not config.is_decoder:
            logger.warning("If you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`")

        self.ernie = MSErnieModel(config, add_pooling_layer=False)
        self.cls = MSErnieOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.get_output_embeddings
    def get_output_embeddings(self):
        """
        Returns the output embeddings of the MSErnieForCausalLM model.

        Args:
            self: The instance of the MSErnieForCausalLM class.

        Returns:
            decoder: The method returns the output embeddings of the model which are of type None.
                These embeddings represent the learned representation of the input data.

        Raises:
            None.

        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """
        Sets new output embeddings for the model.

        Args:
            self (MSErnieForCausalLM): The instance of the MSErnieForCausalLM class.
            new_embeddings (Any): The new embeddings to be set for the output layer.

        Returns:
            None.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
                the model is configured as a decoder.
            encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
                the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
                `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
                ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
            past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having
                4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
                Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
                `past_key_values`).
        """
        if labels is not None:
            use_cache = False

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        lm_loss = None
        if labels is not None:
            # we are doing next-token prediction; shift prediction scores and input ids by one
            shifted_prediction_scores = prediction_scores[:, :-1, :]
            labels = labels[:, 1:]
            lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        output = (prediction_scores,) + outputs[2:]
        return ((lm_loss,) + output) if lm_loss is not None else output

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel.prepare_inputs_for_generation
    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
    ):
        """
        Prepare inputs for generation.

        This method prepares the input tensors for generating text using the MSErnie model for causal language modeling.

        Args:
            self (MSErnieForCausalLM): The instance of the MSErnieForCausalLM class.
            input_ids (torch.Tensor): The input tensor containing the tokenized input text.
                Shape: [batch_size, sequence_length].
            past_key_values (Tuple[torch.Tensor]): The past key-value pairs used for fast decoding.
                Each tuple element contains past key-value tensors.
                Shape: [(batch_size, num_heads, sequence_length, hidden_size // num_heads)] * num_layers.
                Default: None.
            attention_mask (torch.Tensor): The attention mask tensor to avoid attending to padding tokens.
                Shape: [batch_size, sequence_length].
                Default: None.
            use_cache (bool): Whether to use the past key-value cache for fast decoding.
                Default: True.
            **model_kwargs: Additional model-specific keyword arguments.

        Returns:
            dict:
                A dictionary containing the prepared input tensors.

                - 'input_ids' (torch.Tensor): The modified input tensor.
                Shape: [batch_size, modified_sequence_length].
                - 'attention_mask' (torch.Tensor): The attention mask tensor.
                Shape: [batch_size, modified_sequence_length].
                - 'past_key_values' (Tuple[torch.Tensor]): The past key-value pairs.
                Each tuple element contains past key-value tensors.
                Shape: [(batch_size, num_heads, modified_sequence_length, hidden_size // num_heads)] * num_layers.
                - 'use_cache' (bool): The flag indicating whether to use the past key-value cache.

        Raises:
            None
        """
        input_shape = input_ids.shape
        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
        if attention_mask is None:
            attention_mask = input_ids.new_ones(input_shape)

        # cut decoder_input_ids if past_key_values is used
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "past_key_values": past_key_values,
            "use_cache": use_cache,
        }

    # Copied from transformers.models.bert.modeling_bert.BertLMHeadModel._reorder_cache
    def _reorder_cache(self, past_key_values, beam_idx):
        """
        Reorders the cache items based on the provided beam index.

        Args:
            self (MSErnieForCausalLM): The instance of the MSErnieForCausalLM class.
            past_key_values (tuple): A tuple containing the past key-value states for each layer.
            beam_idx (torch.Tensor): An index tensor representing the beam index.

        Returns:
            tuple: A tuple containing the reordered past key-value states.

        Raises:
            None.

        Description:
            This method takes in the past key-value states and reorders them based on the provided beam index.
            It returns a tuple containing the reordered past key-value states.

            The 'self' parameter refers to the instance of the MSErnieForCausalLM class in which this method is called.

            The 'past_key_values' parameter is a tuple containing the past key-value states for each layer.
            These states are used to preserve information over time steps during generation.

            The 'beam_idx' parameter is a tensor representing the beam index.
            It is used to determine the order in which the past key-value states should be reordered.

            The method does not raise any exceptions.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM.__init__(config)

Initializes an instance of MSErnieForCausalLM class.

PARAMETER DESCRIPTION
self

The instance of the class.

TYPE: object

config

An object containing configuration settings for the model.

  • Type: Config
  • Purpose: Specifies the model configuration parameters.
  • Restrictions: Must be provided to initialize the model.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
def __init__(self, config):
    """
    Initializes an instance of MSErnieForCausalLM class.

    Args:
        self (object): The instance of the class.
        config (object):
            An object containing configuration settings for the model.

            - Type: Config
            - Purpose: Specifies the model configuration parameters.
            - Restrictions: Must be provided to initialize the model.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    if not config.is_decoder:
        logger.warning("If you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`")

    self.ernie = MSErnieModel(config, add_pooling_layer=False)
    self.cls = MSErnieOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
encoder_hidden_states

Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder.

TYPE: (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None

encoder_attention_mask

Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in [0, 1]:

  • 1 for tokens that are not masked,
  • 0 for tokens that are masked.

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

labels

Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels n [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

use_cache

If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values).

TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
        past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having
            4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
    """
    if labels is not None:
        use_cache = False

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    lm_loss = None
    if labels is not None:
        # we are doing next-token prediction; shift prediction scores and input ids by one
        shifted_prediction_scores = prediction_scores[:, :-1, :]
        labels = labels[:, 1:]
        lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    output = (prediction_scores,) + outputs[2:]
    return ((lm_loss,) + output) if lm_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM.get_output_embeddings()

Returns the output embeddings of the MSErnieForCausalLM model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForCausalLM class.
RETURNS DESCRIPTION
decoder

The method returns the output embeddings of the model which are of type None. These embeddings represent the learned representation of the input data.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
def get_output_embeddings(self):
    """
    Returns the output embeddings of the MSErnieForCausalLM model.

    Args:
        self: The instance of the MSErnieForCausalLM class.

    Returns:
        decoder: The method returns the output embeddings of the model which are of type None.
            These embeddings represent the learned representation of the input data.

    Raises:
        None.

    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs)

Prepare inputs for generation.

This method prepares the input tensors for generating text using the MSErnie model for causal language modeling.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForCausalLM class.

TYPE: MSErnieForCausalLM

input_ids

The input tensor containing the tokenized input text. Shape: [batch_size, sequence_length].

TYPE: Tensor

past_key_values

The past key-value pairs used for fast decoding. Each tuple element contains past key-value tensors. Shape: [(batch_size, num_heads, sequence_length, hidden_size // num_heads)] * num_layers. Default: None.

TYPE: Tuple[Tensor] DEFAULT: None

attention_mask

The attention mask tensor to avoid attending to padding tokens. Shape: [batch_size, sequence_length]. Default: None.

TYPE: Tensor DEFAULT: None

use_cache

Whether to use the past key-value cache for fast decoding. Default: True.

TYPE: bool DEFAULT: True

**model_kwargs

Additional model-specific keyword arguments.

DEFAULT: {}

RETURNS DESCRIPTION
dict

A dictionary containing the prepared input tensors.

  • 'input_ids' (torch.Tensor): The modified input tensor. Shape: [batch_size, modified_sequence_length].
  • 'attention_mask' (torch.Tensor): The attention mask tensor. Shape: [batch_size, modified_sequence_length].
  • 'past_key_values' (Tuple[torch.Tensor]): The past key-value pairs. Each tuple element contains past key-value tensors. Shape: [(batch_size, num_heads, modified_sequence_length, hidden_size // num_heads)] * num_layers.
  • 'use_cache' (bool): The flag indicating whether to use the past key-value cache.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
def prepare_inputs_for_generation(
    self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
):
    """
    Prepare inputs for generation.

    This method prepares the input tensors for generating text using the MSErnie model for causal language modeling.

    Args:
        self (MSErnieForCausalLM): The instance of the MSErnieForCausalLM class.
        input_ids (torch.Tensor): The input tensor containing the tokenized input text.
            Shape: [batch_size, sequence_length].
        past_key_values (Tuple[torch.Tensor]): The past key-value pairs used for fast decoding.
            Each tuple element contains past key-value tensors.
            Shape: [(batch_size, num_heads, sequence_length, hidden_size // num_heads)] * num_layers.
            Default: None.
        attention_mask (torch.Tensor): The attention mask tensor to avoid attending to padding tokens.
            Shape: [batch_size, sequence_length].
            Default: None.
        use_cache (bool): Whether to use the past key-value cache for fast decoding.
            Default: True.
        **model_kwargs: Additional model-specific keyword arguments.

    Returns:
        dict:
            A dictionary containing the prepared input tensors.

            - 'input_ids' (torch.Tensor): The modified input tensor.
            Shape: [batch_size, modified_sequence_length].
            - 'attention_mask' (torch.Tensor): The attention mask tensor.
            Shape: [batch_size, modified_sequence_length].
            - 'past_key_values' (Tuple[torch.Tensor]): The past key-value pairs.
            Each tuple element contains past key-value tensors.
            Shape: [(batch_size, num_heads, modified_sequence_length, hidden_size // num_heads)] * num_layers.
            - 'use_cache' (bool): The flag indicating whether to use the past key-value cache.

    Raises:
        None
    """
    input_shape = input_ids.shape
    # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
    if attention_mask is None:
        attention_mask = input_ids.new_ones(input_shape)

    # cut decoder_input_ids if past_key_values is used
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "past_key_values": past_key_values,
        "use_cache": use_cache,
    }

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForCausalLM.set_output_embeddings(new_embeddings)

Sets new output embeddings for the model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForCausalLM class.

TYPE: MSErnieForCausalLM

new_embeddings

The new embeddings to be set for the output layer.

TYPE: Any

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
def set_output_embeddings(self, new_embeddings):
    """
    Sets new output embeddings for the model.

    Args:
        self (MSErnieForCausalLM): The instance of the MSErnieForCausalLM class.
        new_embeddings (Any): The new embeddings to be set for the output layer.

    Returns:
        None.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM

Bases: MSErniePreTrainedModel

The MSErnieForMaskedLM class is a Python class that represents a model for masked language modeling (MLM) using the MSErnie architecture. It is designed to generate predictions for masked tokens in a given input sequence.

This class inherits from the MSErniePreTrainedModel class, which provides the basic functionality and configuration for the MSErnie model.

The MSErnieForMaskedLM class contains the following methods:

  • __init__(self, config): Initializes the MSErnieForMaskedLM instance with a given configuration. It creates the MSErnie model and MLM head, and performs additional initialization steps.
  • get_output_embeddings: Returns the decoder layer of the MLM head.
  • set_output_embeddings: Sets the decoder layer of the MLM head to the given embeddings.
  • forward: Constructs the MSErnie model and performs the forward pass. It takes various input tensors and returns the masked language modeling loss and other outputs.
  • prepare_inputs_for_generation: Prepares the inputs for generation by adding a dummy token for each input sequence and adjusting the attention mask accordingly.

Please note that the detailed docstring provided here omits method signatures and any other code. Refer to the actual implementation for complete details on the method signatures and any additional code.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
class MSErnieForMaskedLM(MSErniePreTrainedModel):

    """
    The `MSErnieForMaskedLM` class is a Python class that represents a model for masked language modeling (MLM) using
    the MSErnie architecture. It is designed to generate predictions for masked tokens in a given input sequence.

    This class inherits from the `MSErniePreTrainedModel` class, which provides the basic functionality and configuration
    for the MSErnie model.

    The `MSErnieForMaskedLM` class contains the following methods:

    - `__init__(self, config)`: Initializes the `MSErnieForMaskedLM` instance with a given configuration.
    It creates the MSErnie model and MLM head, and performs additional initialization steps.
    - `get_output_embeddings`: Returns the decoder layer of the MLM head.
    - `set_output_embeddings`: Sets the decoder layer of the MLM head to the given embeddings.
    - `forward`: Constructs the MSErnie model and performs the forward pass.
    It takes various input tensors and returns the masked language modeling loss and other outputs.
    - `prepare_inputs_for_generation`:
    Prepares the inputs for generation by adding a dummy token for each input sequence and adjusting the
    attention mask accordingly.

    Please note that the detailed docstring provided here omits method signatures and any other code.
    Refer to the actual implementation for complete details on the method signatures and any additional code.

    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the `MSErnieForMaskedLM` class.

        Args:
            self: The instance of the class.
            config: An object of type `Config` containing the configuration parameters.

        Returns:
            None

        Raises:
            None

        This method initializes the `MSErnieForMaskedLM` instance by setting up the configuration and the model components.
        It takes in the `config` parameter, which is an object of type `Config` and contains
        the necessary configuration parameters for the model.
        The `self` parameter refers to the instance of the class itself.
        If the `config.is_decoder` attribute is True, a warning message is logged to ensure that the `config.is_decoder`
        attribute is set to False for bi-directional self-attention.
        The method then initializes the `ernie` attribute by creating an instance of the `MSErnieModel` class,
        passing the `config` object and setting the `add_pooling_layer` attribute to False.
        The `cls` attribute is initialized with an instance of the `MSErnieOnlyMLMHead` class, using the `config` object.
        Finally, the `post_init` method is called to perform any additional initialization tasks.
        """
        super().__init__(config)

        if config.is_decoder:
            logger.warning(
                "If you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for "
                "bi-directional self-attention."
            )

        self.ernie = MSErnieModel(config, add_pooling_layer=False)
        self.cls = MSErnieOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.get_output_embeddings
    def get_output_embeddings(self):
        """
        Returns the output embeddings for the MSErnieForMaskedLM model.

        Args:
            self: An instance of the MSErnieForMaskedLM class.

        Returns:
            None: The method returns a value of type 'None'.

        Raises:
            None.

        This method retrieves the output embeddings of the MSErnieForMaskedLM model.
        The output embeddings represent the predicted decoder values for the given input.

        Note:
            The output embeddings are obtained using the `predictions.decoder` attribute of the `self.cls` object.

        Example:
            ```python
            >>> model = MSErnieForMaskedLM()
            >>> embeddings = model.get_output_embeddings()
            ```
        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """"
        Sets the output embeddings for the MSErnieForMaskedLM model.

        Args:
            self (object): The instance of the MSErnieForMaskedLM class.
            new_embeddings (object): The new embeddings to be set as the output embeddings for the model.
                Should be of the same type as the current output embeddings.

        Returns:
            None: This method does not return any value.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
                config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
                loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        """
        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        masked_lm_loss = None
        if labels is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        output = (prediction_scores,) + outputs[2:]
        return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

    # Copied from transformers.models.bert.modeling_bert.BertForMaskedLM.prepare_inputs_for_generation
    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
        """
        Prepare inputs for generation.

        This method takes three parameters: self, input_ids, and attention_mask.
        It prepares the input data for generation by modifying the input_ids and attention_mask tensors.

        Args:
            self (MSErnieForMaskedLM): The instance of the MSErnieForMaskedLM class.
            input_ids (Tensor): The input tensor of shape [batch_size, sequence_length].
                It contains the input token IDs.
            attention_mask (Tensor, optional): The attention mask tensor of shape [batch_size, sequence_length].
                It is used to mask out the padding tokens. Defaults to None.

        Returns:
            dict:
                A dictionary containing the modified input_ids and attention_mask tensors.

                - 'input_ids' (Tensor): The modified input tensor of shape [batch_size, sequence_length+1].
                It contains the input token IDs with an additional dummy token appended.
                - 'attention_mask' (Tensor): The modified attention mask tensor of shape [batch_size, sequence_length+1].
                It is used to mask out the padding tokens with an additional padding token appended.

        Raises:
            ValueError: If the PAD token is not defined for generation.
        """
        input_shape = input_ids.shape
        effective_batch_size = input_shape[0]

        #  add a dummy token
        if self.config.pad_token_id is None:
            raise ValueError("The PAD token should be defined for generation")

        attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
        dummy_token = ops.full(
            (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64
        )
        input_ids = ops.cat([input_ids, dummy_token], dim=1)

        return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM.__init__(config)

Initializes an instance of the MSErnieForMaskedLM class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An object of type Config containing the configuration parameters.
RETURNS DESCRIPTION

None

This method initializes the MSErnieForMaskedLM instance by setting up the configuration and the model components. It takes in the config parameter, which is an object of type Config and contains the necessary configuration parameters for the model. The self parameter refers to the instance of the class itself. If the config.is_decoder attribute is True, a warning message is logged to ensure that the config.is_decoder attribute is set to False for bi-directional self-attention. The method then initializes the ernie attribute by creating an instance of the MSErnieModel class, passing the config object and setting the add_pooling_layer attribute to False. The cls attribute is initialized with an instance of the MSErnieOnlyMLMHead class, using the config object. Finally, the post_init method is called to perform any additional initialization tasks.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
def __init__(self, config):
    """
    Initializes an instance of the `MSErnieForMaskedLM` class.

    Args:
        self: The instance of the class.
        config: An object of type `Config` containing the configuration parameters.

    Returns:
        None

    Raises:
        None

    This method initializes the `MSErnieForMaskedLM` instance by setting up the configuration and the model components.
    It takes in the `config` parameter, which is an object of type `Config` and contains
    the necessary configuration parameters for the model.
    The `self` parameter refers to the instance of the class itself.
    If the `config.is_decoder` attribute is True, a warning message is logged to ensure that the `config.is_decoder`
    attribute is set to False for bi-directional self-attention.
    The method then initializes the `ernie` attribute by creating an instance of the `MSErnieModel` class,
    passing the `config` object and setting the `add_pooling_layer` attribute to False.
    The `cls` attribute is initialized with an instance of the `MSErnieOnlyMLMHead` class, using the `config` object.
    Finally, the `post_init` method is called to perform any additional initialization tasks.
    """
    super().__init__(config)

    if config.is_decoder:
        logger.warning(
            "If you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for "
            "bi-directional self-attention."
        )

    self.ernie = MSErnieModel(config, add_pooling_layer=False)
    self.cls = MSErnieOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
labels

Labels for computing the masked language modeling loss. Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
    """
    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    masked_lm_loss = None
    if labels is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    output = (prediction_scores,) + outputs[2:]
    return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM.get_output_embeddings()

Returns the output embeddings for the MSErnieForMaskedLM model.

PARAMETER DESCRIPTION
self

An instance of the MSErnieForMaskedLM class.
RETURNS DESCRIPTION
None

The method returns a value of type 'None'.

This method retrieves the output embeddings of the MSErnieForMaskedLM model. The output embeddings represent the predicted decoder values for the given input.

Note

The output embeddings are obtained using the predictions.decoder attribute of the self.cls object.

Example 
>>> model = MSErnieForMaskedLM()
>>> embeddings = model.get_output_embeddings()
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
def get_output_embeddings(self):
    """
    Returns the output embeddings for the MSErnieForMaskedLM model.

    Args:
        self: An instance of the MSErnieForMaskedLM class.

    Returns:
        None: The method returns a value of type 'None'.

    Raises:
        None.

    This method retrieves the output embeddings of the MSErnieForMaskedLM model.
    The output embeddings represent the predicted decoder values for the given input.

    Note:
        The output embeddings are obtained using the `predictions.decoder` attribute of the `self.cls` object.

    Example:
        ```python
        >>> model = MSErnieForMaskedLM()
        >>> embeddings = model.get_output_embeddings()
        ```
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM.prepare_inputs_for_generation(input_ids, attention_mask=None, **model_kwargs)

Prepare inputs for generation.

This method takes three parameters: self, input_ids, and attention_mask. It prepares the input data for generation by modifying the input_ids and attention_mask tensors.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForMaskedLM class.

TYPE: MSErnieForMaskedLM

input_ids

The input tensor of shape [batch_size, sequence_length]. It contains the input token IDs.

TYPE: Tensor

attention_mask

The attention mask tensor of shape [batch_size, sequence_length]. It is used to mask out the padding tokens. Defaults to None.

TYPE: Tensor DEFAULT: None

RETURNS DESCRIPTION
dict

A dictionary containing the modified input_ids and attention_mask tensors.

  • 'input_ids' (Tensor): The modified input tensor of shape [batch_size, sequence_length+1]. It contains the input token IDs with an additional dummy token appended.
  • 'attention_mask' (Tensor): The modified attention mask tensor of shape [batch_size, sequence_length+1]. It is used to mask out the padding tokens with an additional padding token appended.
RAISES DESCRIPTION
ValueError

If the PAD token is not defined for generation.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
    """
    Prepare inputs for generation.

    This method takes three parameters: self, input_ids, and attention_mask.
    It prepares the input data for generation by modifying the input_ids and attention_mask tensors.

    Args:
        self (MSErnieForMaskedLM): The instance of the MSErnieForMaskedLM class.
        input_ids (Tensor): The input tensor of shape [batch_size, sequence_length].
            It contains the input token IDs.
        attention_mask (Tensor, optional): The attention mask tensor of shape [batch_size, sequence_length].
            It is used to mask out the padding tokens. Defaults to None.

    Returns:
        dict:
            A dictionary containing the modified input_ids and attention_mask tensors.

            - 'input_ids' (Tensor): The modified input tensor of shape [batch_size, sequence_length+1].
            It contains the input token IDs with an additional dummy token appended.
            - 'attention_mask' (Tensor): The modified attention mask tensor of shape [batch_size, sequence_length+1].
            It is used to mask out the padding tokens with an additional padding token appended.

    Raises:
        ValueError: If the PAD token is not defined for generation.
    """
    input_shape = input_ids.shape
    effective_batch_size = input_shape[0]

    #  add a dummy token
    if self.config.pad_token_id is None:
        raise ValueError("The PAD token should be defined for generation")

    attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
    dummy_token = ops.full(
        (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64
    )
    input_ids = ops.cat([input_ids, dummy_token], dim=1)

    return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMaskedLM.set_output_embeddings(new_embeddings)

" Sets the output embeddings for the MSErnieForMaskedLM model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForMaskedLM class.

TYPE: object

new_embeddings

The new embeddings to be set as the output embeddings for the model. Should be of the same type as the current output embeddings.

TYPE: object

RETURNS DESCRIPTION
None

This method does not return any value.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
def set_output_embeddings(self, new_embeddings):
    """"
    Sets the output embeddings for the MSErnieForMaskedLM model.

    Args:
        self (object): The instance of the MSErnieForMaskedLM class.
        new_embeddings (object): The new embeddings to be set as the output embeddings for the model.
            Should be of the same type as the current output embeddings.

    Returns:
        None: This method does not return any value.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMultipleChoice

Bases: MSErniePreTrainedModel

MSErnieForMultipleChoice represents a multiple choice question answering model based on the ERNIE (Enhanced Representation through kNowledge Integration) architecture. This class extends MSErniePreTrainedModel and provides methods for forwarding the model, including processing input data, computing logits, and calculating loss for training. The model utilizes an ERNIE model for encoding input sequences and a classifier for predicting the correct choice among multiple options. The forward method takes various input tensors such as input_ids, attention_mask, token_type_ids, and labels, and returns the loss and reshaped logits for the multiple choice classification task. Additionally, the class includes functionality for handling dropout during training and post-initialization tasks.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
class MSErnieForMultipleChoice(MSErniePreTrainedModel):

    """
    MSErnieForMultipleChoice represents a multiple choice question answering model based on the ERNIE
    (Enhanced Representation through kNowledge Integration) architecture.
    This class extends MSErniePreTrainedModel and provides methods for forwarding the model,
    including processing input data, computing logits, and calculating loss for training.
    The model utilizes an ERNIE model for encoding input sequences and a classifier for predicting the correct choice
    among multiple options.
    The forward method takes various input tensors such as input_ids, attention_mask, token_type_ids, and labels,
    and returns the loss and reshaped logits for the multiple choice classification task.
    Additionally, the class includes functionality for handling dropout during training and post-initialization tasks.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForMultipleChoice.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieForMultipleChoice class.

        Args:
            self (MSErnieForMultipleChoice): The current instance of the MSErnieForMultipleChoice class.
            config:
                An object containing configuration settings for the model.

                - Type: Dict
                - Purpose: Specifies the configuration parameters for the model.
                - Restrictions: Must be a valid dictionary object.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.ernie = MSErnieModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, 1)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
                num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
                `input_ids` above)
        """
        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

        input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
        attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
        token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
        position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
        inputs_embeds = (
            inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
            if inputs_embeds is not None
            else None
        )

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
        reshaped_logits = logits.view(-1, num_choices)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(reshaped_logits, labels)

        output = (reshaped_logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMultipleChoice.__init__(config)

Initializes an instance of the MSErnieForMultipleChoice class.

PARAMETER DESCRIPTION
self

The current instance of the MSErnieForMultipleChoice class.

TYPE: MSErnieForMultipleChoice

config

An object containing configuration settings for the model.

  • Type: Dict
  • Purpose: Specifies the configuration parameters for the model.
  • Restrictions: Must be a valid dictionary object.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
def __init__(self, config):
    """
    Initializes an instance of the MSErnieForMultipleChoice class.

    Args:
        self (MSErnieForMultipleChoice): The current instance of the MSErnieForMultipleChoice class.
        config:
            An object containing configuration settings for the model.

            - Type: Dict
            - Purpose: Specifies the configuration parameters for the model.
            - Restrictions: Must be a valid dictionary object.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.ernie = MSErnieModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, 1)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForMultipleChoice.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
labels

Labels for computing the multiple choice classification loss. Indices should be in [0, ..., num_choices-1] where num_choices is the size of the second dimension of the input tensors. (See input_ids above)

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
    """
    num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

    input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
    attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
    token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
    position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
    inputs_embeds = (
        inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
        if inputs_embeds is not None
        else None
    )

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)
    reshaped_logits = logits.view(-1, num_choices)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(reshaped_logits, labels)

    output = (reshaped_logits,) + outputs[2:]
    return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForNextSentencePrediction

Bases: MSErniePreTrainedModel

This class represents a model for next sentence prediction using MSErnie, a pre-trained model for natural language understanding. It inherits from the MSErniePreTrainedModel class.

The class has an initializer method that takes a configuration object as input. It initializes an instance of the MSErnieModel class and the MSErnieOnlyNSPHead class, and then calls the post_init method.

The forward method is used to perform the next sentence prediction task. It takes several input tensors, such as input_ids, attention_mask, token_type_ids, and labels. It returns a tuple containing the next sentence prediction loss, the sequence relationship scores, and additional outputs.

The labels parameter is optional and is used for computing the next sequence prediction loss. The labels should be a tensor of shape (batch_size,) containing indices in the range [0, 1]. A label of 0 indicates that sequence B is a continuation of sequence A, while a label of 1 indicates that sequence B is a random sequence.

Example 
>>> from transformers import AutoTokenizer, MSErnieForNextSentencePrediction
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = MSErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
...
>>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
Note

The 'next_sentence_label' argument in the forward method is deprecated. Use the 'labels' argument instead.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
class MSErnieForNextSentencePrediction(MSErniePreTrainedModel):

    """
    This class represents a model for next sentence prediction using MSErnie, a pre-trained model for natural language
    understanding. It inherits from the MSErniePreTrainedModel class.

    The class has an initializer method that takes a configuration object as input.
    It initializes an instance of the MSErnieModel class and the MSErnieOnlyNSPHead class, and then calls the post_init method.

    The forward method is used to perform the next sentence prediction task. It takes several input tensors,
    such as input_ids, attention_mask, token_type_ids, and labels.
    It returns a tuple containing the next sentence prediction loss, the sequence relationship scores, and additional outputs.

    The labels parameter is optional and is used for computing the next sequence prediction loss.
    The labels should be a tensor of shape (batch_size,) containing indices in the range [0, 1]. A label of 0 indicates
    that sequence B is a continuation of sequence A, while a label of 1 indicates that sequence B is a random sequence.

    Example:
        ```python
        >>> from transformers import AutoTokenizer, MSErnieForNextSentencePrediction
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = MSErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
        ...
        >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
        >>> logits = outputs.logits
        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
        ```

    Note:
        The 'next_sentence_label' argument in the forward method is deprecated.
        Use the 'labels' argument instead.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForNextSentencePrediction.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of MSErnieForNextSentencePrediction.

        Args:
            self: The instance of the class.
            config: The configuration object containing settings for the model.

        Returns:
            None.

        Raises:
            TypeError: If the provided 'config' parameter is not of the expected type.
            ValueError: If any required attribute in the 'config' object is missing.
        """
        super().__init__(config)

        self.ernie = MSErnieModel(config)
        self.cls = MSErnieOnlyNSPHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        **kwargs,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
                (see `input_ids` docstring). Indices should be in `[0, 1]`:

                - 0 indicates sequence B is a continuation of sequence A,
                - 1 indicates sequence B is a random sequence.

        Returns:
            Union[Tuple[mindspore.Tensor], dict]

        Example:
            ```python
            >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
            ...
            ...
            >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
            >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
            ...
            >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
            >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
            >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
            ...
            >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
            >>> logits = outputs.logits
            >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
            ```
        """
        if "next_sentence_label" in kwargs:
            warnings.warn(
                "The `next_sentence_label` argument is deprecated and will be removed in a future version, use"
                " `labels` instead.",
                FutureWarning,
            )
            labels = kwargs.pop("next_sentence_label")

        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        pooled_output = outputs[1]

        seq_relationship_scores = self.cls(pooled_output)

        next_sentence_loss = None
        if labels is not None:
            next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

        output = (seq_relationship_scores,) + outputs[2:]
        return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForNextSentencePrediction.__init__(config)

Initializes an instance of MSErnieForNextSentencePrediction.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object containing settings for the model.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the provided 'config' parameter is not of the expected type.
ValueError

If any required attribute in the 'config' object is missing.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
def __init__(self, config):
    """
    Initializes an instance of MSErnieForNextSentencePrediction.

    Args:
        self: The instance of the class.
        config: The configuration object containing settings for the model.

    Returns:
        None.

    Raises:
        TypeError: If the provided 'config' parameter is not of the expected type.
        ValueError: If any required attribute in the 'config' object is missing.
    """
    super().__init__(config)

    self.ernie = MSErnieModel(config)
    self.cls = MSErnieOnlyNSPHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForNextSentencePrediction.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, **kwargs)

PARAMETER DESCRIPTION
labels

Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see input_ids docstring). Indices should be in [0, 1]:

  • 0 indicates sequence B is a continuation of sequence A,
  • 1 indicates sequence B is a random sequence.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS DESCRIPTION
Union[Tuple[Tensor], dict]

Union[Tuple[mindspore.Tensor], dict]
Example 
>>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
...
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
...
>>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    **kwargs,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring). Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.

    Returns:
        Union[Tuple[mindspore.Tensor], dict]

    Example:
        ```python
        >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
        ...
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="ms")
        ...
        >>> outputs = model(**encoding, labels=mindspore.Tensor([1]))
        >>> logits = outputs.logits
        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
        ```
    """
    if "next_sentence_label" in kwargs:
        warnings.warn(
            "The `next_sentence_label` argument is deprecated and will be removed in a future version, use"
            " `labels` instead.",
            FutureWarning,
        )
        labels = kwargs.pop("next_sentence_label")

    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    pooled_output = outputs[1]

    seq_relationship_scores = self.cls(pooled_output)

    next_sentence_loss = None
    if labels is not None:
        next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

    output = (seq_relationship_scores,) + outputs[2:]
    return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForPreTraining

Bases: MSErniePreTrainedModel

MSErnieForPreTraining is a class that extends MSErniePreTrainedModel and is designed for pre-training the Ernie model for masked language modeling and next sentence prediction tasks.

The class includes methods for initializing the model with configuration, getting and setting output embeddings, and forwarding the model for training. The 'forward' method takes various input tensors such as input_ids, attention_mask, token_type_ids, etc., and computes the total loss for masked language modeling and next sentence prediction. The method returns the total loss, prediction scores, sequence relationship scores, and additional outputs if specified.

Example usage of the MSErnieForPreTraining class involves initializing a tokenizer and the model, processing inputs using the tokenizer, and obtaining prediction and sequence relationship logits from the model's outputs.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
class MSErnieForPreTraining(MSErniePreTrainedModel):

    """
    MSErnieForPreTraining is a class that extends MSErniePreTrainedModel and is designed for pre-training the
    Ernie model for masked language modeling and next sentence prediction tasks.

    The class includes methods for initializing the model with configuration, getting and setting output embeddings,
    and forwarding the model for training. The 'forward' method takes various input tensors
    such as input_ids, attention_mask, token_type_ids, etc., and computes the total loss for masked language modeling
    and next sentence prediction. The method returns the total loss, prediction scores, sequence relationship scores,
    and additional outputs if specified.

    Example usage of the MSErnieForPreTraining class involves initializing a tokenizer and the model, processing inputs
    using the tokenizer, and obtaining prediction and sequence relationship logits from the model's outputs.
    """
    _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"]

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes an instance of MSErnieForPreTraining.

        Args:
            self (object): The instance of the class.
            config (object): Configuration object containing settings for the model.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.ernie = MSErnieModel(config)
        self.cls = MSErniePreTrainingHeads(config)

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.get_output_embeddings
    def get_output_embeddings(self):
        """
        Returns the output embeddings from the MSErnieForPreTraining model.

        Args:
            self: An instance of the MSErnieForPreTraining class.

        Returns:
            The output embeddings from the model.

        Raises:
            None.

        Example:
            ```python
            >>> model = MSErnieForPreTraining()
            >>> embeddings = model.get_output_embeddings()
            ```
        """
        return self.cls.predictions.decoder

    # Copied from transformers.models.bert.modeling_bert.BertForPreTraining.set_output_embeddings
    def set_output_embeddings(self, new_embeddings):
        """
        Method to set new output embeddings in the MSErnieForPreTraining model.

        Args:
            self (MSErnieForPreTraining): The instance of the MSErnieForPreTraining class.
                This parameter represents the current instance of the class.
            new_embeddings (object): The new output embeddings to be set in the model.
                This parameter should be of the desired type for output embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        next_sentence_label: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
                config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
                the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
            next_sentence_label (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
                pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

                - 0 indicates sequence B is a continuation of sequence A,
                - 1 indicates sequence B is a random sequence.
            kwargs (`Dict[str, any]`, optional, defaults to *{}*):
                Used to hide legacy arguments that have been deprecated.

        Returns:
            Union[Tuple[mindspore.Tensor], dict]

        Example:
            ```python
            >>> from transformers import AutoTokenizer, ErnieForPreTraining
            ...
            ...
            >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
            >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
            ...
            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
            >>> outputs = model(**inputs)
            ...
            >>> prediction_logits = outputs.prediction_logits
            >>> seq_relationship_logits = outputs.seq_relationship_logits
            ```
        """
        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output, pooled_output = outputs[:2]
        prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

        total_loss = None
        if labels is not None and next_sentence_label is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
            next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
            total_loss = masked_lm_loss + next_sentence_loss

        output = (prediction_scores, seq_relationship_score) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForPreTraining.__init__(config)

Initializes an instance of MSErnieForPreTraining.

PARAMETER DESCRIPTION
self

The instance of the class.

TYPE: object

config

Configuration object containing settings for the model.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
def __init__(self, config):
    """
    Initializes an instance of MSErnieForPreTraining.

    Args:
        self (object): The instance of the class.
        config (object): Configuration object containing settings for the model.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.ernie = MSErnieModel(config)
    self.cls = MSErniePreTrainingHeads(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForPreTraining.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, next_sentence_label=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
labels

Labels for computing the masked language modeling loss. Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

next_sentence_label

Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see input_ids docstring) Indices should be in [0, 1]:

  • 0 indicates sequence B is a continuation of sequence A,
  • 1 indicates sequence B is a random sequence.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

kwargs

Used to hide legacy arguments that have been deprecated.

TYPE: `Dict[str, any]`, optional, defaults to *{}*

RETURNS DESCRIPTION
Union[Tuple[Tensor], dict]

Union[Tuple[mindspore.Tensor], dict]
Example 
>>> from transformers import AutoTokenizer, ErnieForPreTraining
...
...
>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
...
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
>>> outputs = model(**inputs)
...
>>> prediction_logits = outputs.prediction_logits
>>> seq_relationship_logits = outputs.seq_relationship_logits
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    next_sentence_label: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
            the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        next_sentence_label (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
            pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.
        kwargs (`Dict[str, any]`, optional, defaults to *{}*):
            Used to hide legacy arguments that have been deprecated.

    Returns:
        Union[Tuple[mindspore.Tensor], dict]

    Example:
        ```python
        >>> from transformers import AutoTokenizer, ErnieForPreTraining
        ...
        ...
        >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
        >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")
        ...
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="ms")
        >>> outputs = model(**inputs)
        ...
        >>> prediction_logits = outputs.prediction_logits
        >>> seq_relationship_logits = outputs.seq_relationship_logits
        ```
    """
    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output, pooled_output = outputs[:2]
    prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

    total_loss = None
    if labels is not None and next_sentence_label is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
        next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
        total_loss = masked_lm_loss + next_sentence_loss

    output = (prediction_scores, seq_relationship_score) + outputs[2:]
    return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForPreTraining.get_output_embeddings()

Returns the output embeddings from the MSErnieForPreTraining model.

PARAMETER DESCRIPTION
self

An instance of the MSErnieForPreTraining class.
RETURNS DESCRIPTION

The output embeddings from the model.
Example 
>>> model = MSErnieForPreTraining()
>>> embeddings = model.get_output_embeddings()
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
def get_output_embeddings(self):
    """
    Returns the output embeddings from the MSErnieForPreTraining model.

    Args:
        self: An instance of the MSErnieForPreTraining class.

    Returns:
        The output embeddings from the model.

    Raises:
        None.

    Example:
        ```python
        >>> model = MSErnieForPreTraining()
        >>> embeddings = model.get_output_embeddings()
        ```
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForPreTraining.set_output_embeddings(new_embeddings)

Method to set new output embeddings in the MSErnieForPreTraining model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieForPreTraining class. This parameter represents the current instance of the class.

TYPE: MSErnieForPreTraining

new_embeddings

The new output embeddings to be set in the model. This parameter should be of the desired type for output embeddings.

TYPE: object

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
def set_output_embeddings(self, new_embeddings):
    """
    Method to set new output embeddings in the MSErnieForPreTraining model.

    Args:
        self (MSErnieForPreTraining): The instance of the MSErnieForPreTraining class.
            This parameter represents the current instance of the class.
        new_embeddings (object): The new output embeddings to be set in the model.
            This parameter should be of the desired type for output embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForQuestionAnswering

Bases: MSErniePreTrainedModel

MSErnieForQuestionAnswering represents a model for question answering tasks using the MSErnie architecture. This class inherits from MSErniePreTrainedModel and includes methods for initializing the model and forwarding the forward pass for predicting start and end positions of answers within a text sequence.

ATTRIBUTE DESCRIPTION
num_labels

The number of labels for the classifier output.

TYPE: int

ernie

The MSErnie model used as the base for question answering.

TYPE: MSErnieModel

qa_outputs

The fully connected layer for predicting start and end positions within the sequence.

TYPE: Linear

METHOD DESCRIPTION
__init__

Initializes the model with the given configuration.
forward

Constructs the forward pass of the model for question answering, predicting start and end positions within the input sequence. Returns the total loss and output logits for start and end positions, along with any additional model outputs.
PARAMETER DESCRIPTION
config

The configuration object containing model hyperparameters.
input_ids

The input token IDs of the sequence.

TYPE: Tensor

attention_mask

The attention mask to prevent attention to padding tokens.

TYPE: Tensor

token_type_ids

The token type IDs to distinguish between question and context tokens.

TYPE: Tensor

task_type_ids

The task type IDs for multi-task learning.

TYPE: Tensor

position_ids

The position IDs for positional embeddings.

TYPE: Tensor

head_mask

The mask for attention heads.

TYPE: Tensor

inputs_embeds

The input embeddings instead of input IDs.

TYPE: Tensor

start_positions

The start positions of the answer span in the sequence.

TYPE: Tensor

end_positions

The end positions of the answer span in the sequence.

TYPE: Tensor

output_attentions

Flag to output attentions weights.

TYPE: bool

output_hidden_states

Flag to output hidden states of the model.

TYPE: bool

RETURNS DESCRIPTION

Tuple containing the total loss, start logits, end logits, and any additional model outputs.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
class MSErnieForQuestionAnswering(MSErniePreTrainedModel):

    """
    MSErnieForQuestionAnswering represents a model for question answering tasks using the MSErnie architecture.
    This class inherits from MSErniePreTrainedModel and includes methods for initializing the model and forwarding the forward pass
    for predicting start and end positions of answers within a text sequence.

    Attributes:
        num_labels (int): The number of labels for the classifier output.
        ernie (MSErnieModel): The MSErnie model used as the base for question answering.
        qa_outputs (nn.Linear): The fully connected layer for predicting start and end positions within the sequence.

    Methods:
        __init__: Initializes the model with the given configuration.
        forward:
            Constructs the forward pass of the model for question answering, predicting start and end positions within
            the input sequence.
            Returns the total loss and output logits for start and end positions, along with any additional model outputs.

    Args:
        config: The configuration object containing model hyperparameters.
        input_ids (mindspore.Tensor): The input token IDs of the sequence.
        attention_mask (mindspore.Tensor): The attention mask to prevent attention to padding tokens.
        token_type_ids (mindspore.Tensor): The token type IDs to distinguish between question and context tokens.
        task_type_ids (mindspore.Tensor): The task type IDs for multi-task learning.
        position_ids (mindspore.Tensor): The position IDs for positional embeddings.
        head_mask (mindspore.Tensor): The mask for attention heads.
        inputs_embeds (mindspore.Tensor): The input embeddings instead of input IDs.
        start_positions (mindspore.Tensor): The start positions of the answer span in the sequence.
        end_positions (mindspore.Tensor): The end positions of the answer span in the sequence.
        output_attentions (bool): Flag to output attentions weights.
        output_hidden_states (bool): Flag to output hidden states of the model.

    Returns:
        Tuple containing the total loss, start logits, end logits, and any additional model outputs.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes a new instance of the MSErnieForQuestionAnswering class.

        Args:
            self: The object instance.
            config:
                An instance of the MSErnieConfig class containing the configuration parameters for the model.

                - Type: MSErnieConfig
                - Purpose: Specifies the model configuration.
                - Restrictions: Must be a valid MSErnieConfig object.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.ernie = MSErnieModel(config, add_pooling_layer=False)
        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the start of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
                are not taken into account for computing the loss.
            end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the end of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
                are not taken into account for computing the loss.
        """
        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output = outputs[0]

        logits = self.qa_outputs(sequence_output)
        start_logits, end_logits = logits.split(1, axis=-1)
        start_logits = start_logits.squeeze(-1)
        end_logits = end_logits.squeeze(-1)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, split add a dimension
            if len(start_positions.shape) > 1:
                start_positions = start_positions.squeeze(-1)
            if len(end_positions.shape) > 1:
                end_positions = end_positions.squeeze(-1)
            # sometimes the start/end positions are outside our model inputs, we ignore these terms
            ignored_index = start_logits.shape[1]
            start_positions = start_positions.clamp(0, ignored_index)
            end_positions = end_positions.clamp(0, ignored_index)

            start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
            end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
            total_loss = (start_loss + end_loss) / 2

        output = (start_logits, end_logits) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForQuestionAnswering.__init__(config)

Initializes a new instance of the MSErnieForQuestionAnswering class.

PARAMETER DESCRIPTION
self

The object instance.
config

An instance of the MSErnieConfig class containing the configuration parameters for the model.

  • Type: MSErnieConfig
  • Purpose: Specifies the model configuration.
  • Restrictions: Must be a valid MSErnieConfig object.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
def __init__(self, config):
    """
    Initializes a new instance of the MSErnieForQuestionAnswering class.

    Args:
        self: The object instance.
        config:
            An instance of the MSErnieConfig class containing the configuration parameters for the model.

            - Type: MSErnieConfig
            - Purpose: Specifies the model configuration.
            - Restrictions: Must be a valid MSErnieConfig object.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.ernie = MSErnieModel(config, add_pooling_layer=False)
    self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForQuestionAnswering.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
start_positions

Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

end_positions

Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss.

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
    """
    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output = outputs[0]

    logits = self.qa_outputs(sequence_output)
    start_logits, end_logits = logits.split(1, axis=-1)
    start_logits = start_logits.squeeze(-1)
    end_logits = end_logits.squeeze(-1)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        # If we are on multi-GPU, split add a dimension
        if len(start_positions.shape) > 1:
            start_positions = start_positions.squeeze(-1)
        if len(end_positions.shape) > 1:
            end_positions = end_positions.squeeze(-1)
        # sometimes the start/end positions are outside our model inputs, we ignore these terms
        ignored_index = start_logits.shape[1]
        start_positions = start_positions.clamp(0, ignored_index)
        end_positions = end_positions.clamp(0, ignored_index)

        start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
        end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
        total_loss = (start_loss + end_loss) / 2

    output = (start_logits, end_logits) + outputs[2:]
    return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForSequenceClassification

Bases: MSErniePreTrainedModel

This class represents an implementation of MSErnie for sequence classification. It is a subclass of MSErniePreTrainedModel.

ATTRIBUTE DESCRIPTION
`num_labels`

The number of labels for sequence classification.

TYPE: int

`config`

The configuration object for MSErnie.

TYPE: object

`ernie`

The MSErnieModel instance for feature extraction.

TYPE: object

`dropout`

The dropout layer for regularization.

TYPE: object

`classifier`

The fully connected layer for classification.

TYPE: object

`problem_type`

The type of problem being solved for classification. Options are 'regression', 'single_label_classification', and 'multi_label_classification'.

TYPE: str

METHOD DESCRIPTION
`forward`

Constructs the MSErnie model for sequence classification.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
class MSErnieForSequenceClassification(MSErniePreTrainedModel):

    """
    This class represents an implementation of MSErnie for sequence classification. It is a subclass of `MSErniePreTrainedModel`.

    Attributes:
        `num_labels` (int): The number of labels for sequence classification.
        `config` (object): The configuration object for MSErnie.
        `ernie` (object): The MSErnieModel instance for feature extraction.
        `dropout` (object): The dropout layer for regularization.
        `classifier` (object): The fully connected layer for classification.
        `problem_type` (str): The type of problem being solved for classification.
            Options are 'regression', 'single_label_classification', and 'multi_label_classification'.

    Methods:
        `forward`: Constructs the MSErnie model for sequence classification.

    """
    # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """Initializes an instance of the `MSErnieForSequenceClassification` class.

        Args:
            self: The instance of the class.
            config (MSErnieConfig): The configuration object for the model.
                It contains various hyperparameters and settings.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config

        self.ernie = MSErnieModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
                `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        """
        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.num_labels == 1:
                    loss = F.mse_loss(logits.squeeze(), labels.squeeze())
                else:
                    loss = F.mse_loss(logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = F.binary_cross_entropy_with_logits(logits, labels)

        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForSequenceClassification.__init__(config)

Initializes an instance of the MSErnieForSequenceClassification class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object for the model. It contains various hyperparameters and settings.

TYPE: MSErnieConfig

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
def __init__(self, config):
    """Initializes an instance of the `MSErnieForSequenceClassification` class.

    Args:
        self: The instance of the class.
        config (MSErnieConfig): The configuration object for the model.
            It contains various hyperparameters and settings.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.config = config

    self.ernie = MSErnieModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForSequenceClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
labels

Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If config.num_labels > 1 a classification loss is computed (Cross-Entropy).

TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
    """
    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.num_labels == 1:
                loss = F.mse_loss(logits.squeeze(), labels.squeeze())
            else:
                loss = F.mse_loss(logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = F.binary_cross_entropy_with_logits(logits, labels)

    output = (logits,) + outputs[2:]
    return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForTokenClassification

Bases: MSErniePreTrainedModel

MSErnieForTokenClassification is a class that represents a token classification model based on MSErnie (MindSpore implementation of ERNIE) for sequence labeling tasks.

This class inherits from MSErniePreTrainedModel and provides functionality for token classification by utilizing an ERNIE-based model architecture. It includes methods for initializing the model with configuration parameters, forwarding the model for inference or training, and computing token classification loss.

ATTRIBUTE DESCRIPTION
num_labels

The number of labels for token classification tasks.

TYPE: int

ernie

The ERNIE model used for token classification.

TYPE: MSErnieModel

dropout

Dropout layer for regularization.

TYPE: Dropout

classifier

Fully connected layer for classification.

TYPE: Linear

METHOD DESCRIPTION
__init__

Initializes the MSErnieForTokenClassification model with the given configuration.
forward

Constructs the model for inference or training and computes token classification loss if labels are provided.
PARAMETER DESCRIPTION
config

The configuration object containing model hyperparameters.

TYPE: object

input_ids

Tensor of input token IDs for the model.

TYPE: Tensor

attention_mask

Tensor representing the attention mask for input tokens.

TYPE: Tensor

token_type_ids

Tensor for token type IDs.

TYPE: Tensor

task_type_ids

Tensor for task type IDs.

TYPE: Tensor

position_ids

Tensor for position IDs.

TYPE: Tensor

head_mask

Tensor for head mask.

TYPE: Tensor

inputs_embeds

Tensor for input embeddings.

TYPE: Tensor

labels

Tensor of labels for token classification.

TYPE: Tensor

output_attentions

Flag to output attentions.

TYPE: bool

output_hidden_states

Flag to output hidden states.

TYPE: bool

RETURNS DESCRIPTION

Union[Tuple[mindspore.Tensor], dict]: Tuple containing model outputs and optionally additional information such as attentions and hidden states.
RAISES DESCRIPTION
ValueError

If the number of labels provided is not compatible with the model architecture.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
class MSErnieForTokenClassification(MSErniePreTrainedModel):

    """
    MSErnieForTokenClassification is a class that represents a token classification model based on MSErnie
    (MindSpore implementation of ERNIE) for sequence labeling tasks.

    This class inherits from MSErniePreTrainedModel and provides functionality for token classification by utilizing
    an ERNIE-based model architecture.
    It includes methods for initializing the model with configuration parameters, forwarding the model
    for inference or training, and computing token classification loss.

    Attributes:
        num_labels (int): The number of labels for token classification tasks.
        ernie (MSErnieModel): The ERNIE model used for token classification.
        dropout (nn.Dropout): Dropout layer for regularization.
        classifier (nn.Linear): Fully connected layer for classification.

    Methods:
        __init__: Initializes the MSErnieForTokenClassification model with the given configuration.
        forward: Constructs the model for inference or training and computes token classification loss if labels
            are provided.

    Args:
        config (object): The configuration object containing model hyperparameters.
        input_ids (mindspore.Tensor, optional): Tensor of input token IDs for the model.
        attention_mask (mindspore.Tensor, optional): Tensor representing the attention mask for input tokens.
        token_type_ids (mindspore.Tensor, optional): Tensor for token type IDs.
        task_type_ids (mindspore.Tensor, optional): Tensor for task type IDs.
        position_ids (mindspore.Tensor, optional): Tensor for position IDs.
        head_mask (mindspore.Tensor, optional): Tensor for head mask.
        inputs_embeds (mindspore.Tensor, optional): Tensor for input embeddings.
        labels (mindspore.Tensor, optional): Tensor of labels for token classification.
        output_attentions (bool, optional): Flag to output attentions.
        output_hidden_states (bool, optional): Flag to output hidden states.

    Returns:
        Union[Tuple[mindspore.Tensor], dict]:
            Tuple containing model outputs and optionally additional information such as attentions and hidden states.

    Raises:
        ValueError: If the number of labels provided is not compatible with the model architecture.
    """
    # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification.__init__ with Bert->Ernie,bert->ernie
    def __init__(self, config):
        """
        Initializes a new instance of the `MSErnieForTokenClassification` class.

        Args:
            self: The object itself.
            config: An instance of the `MSErnieConfig` class containing the model configuration.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.ernie = MSErnieModel(config, add_pooling_layer=False)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        """
        outputs = self.ernie(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output = outputs[0]

        sequence_output = self.dropout(sequence_output)
        logits = self.classifier(sequence_output)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForTokenClassification.__init__(config)

Initializes a new instance of the MSErnieForTokenClassification class.

PARAMETER DESCRIPTION
self

The object itself.
config

An instance of the MSErnieConfig class containing the model configuration.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
def __init__(self, config):
    """
    Initializes a new instance of the `MSErnieForTokenClassification` class.

    Args:
        self: The object itself.
        config: An instance of the `MSErnieConfig` class containing the model configuration.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.ernie = MSErnieModel(config, add_pooling_layer=False)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieForTokenClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
labels

Labels for computing the token classification loss. Indices should be in [0, ..., config.num_labels - 1].

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
    """
    outputs = self.ernie(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output = outputs[0]

    sequence_output = self.dropout(sequence_output)
    logits = self.classifier(sequence_output)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

    output = (logits,) + outputs[2:]
    return ((loss,) + output) if loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieIntermediate

Bases: Module

This class represents the intermediate layer of the MSErnie model, which is used for feature extraction and transformation.

The MSErnieIntermediate class inherits from the nn.Module class, which is a base class for all neural network layers in the MindSpore framework.

ATTRIBUTE DESCRIPTION
dense

A fully connected layer that transforms the input tensor to the hidden size defined in the configuration.

TYPE: Linear

intermediate_act_fn

The activation function applied to the hidden states after the dense layer.

TYPE: function

METHOD DESCRIPTION
__init__

Initializes the MSErnieIntermediate instance with the given configuration.
forward

mindspore.Tensor) -> mindspore.Tensor: Performs the forward pass of the intermediate layer.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
class MSErnieIntermediate(nn.Module):

    """
    This class represents the intermediate layer of the MSErnie model, which is used for feature extraction and transformation.

    The MSErnieIntermediate class inherits from the nn.Module class, which is a base class for all neural network layers
    in the MindSpore framework.

    Attributes:
        dense (nn.Linear): A fully connected layer that transforms the input tensor to the hidden size defined
            in the configuration.
        intermediate_act_fn (function): The activation function applied to the hidden states after the dense layer.

    Methods:
        __init__(self, config): Initializes the MSErnieIntermediate instance with the given configuration.
        forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
            Performs the forward pass of the intermediate layer.

    """
    def __init__(self, config):
        """Initializes an instance of the MSErnieIntermediate class.

        Args:
            self: An instance of the MSErnieIntermediate class.
            config:
                A configuration object that contains the following attributes:

                - hidden_size (int): The size of the hidden layer.
                - intermediate_size (int): The size of the intermediate layer.
                - hidden_act (str or function): The activation function to use in the hidden layer.
                If a string is provided, the corresponding function will be retrieved from the ACT2FN dictionary.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        Method to forward intermediate hidden states in the MSErnieIntermediate class.

        Args:
            self (MSErnieIntermediate): The instance of the MSErnieIntermediate class.
            hidden_states (mindspore.Tensor): A tensor containing the hidden states to be processed.

        Returns:
            mindspore.Tensor: The processed hidden states after passing through the dense layer and activation function.

        Raises:
            None
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieIntermediate.__init__(config)

Initializes an instance of the MSErnieIntermediate class.

PARAMETER DESCRIPTION
self

An instance of the MSErnieIntermediate class.
config

A configuration object that contains the following attributes:

  • hidden_size (int): The size of the hidden layer.
  • intermediate_size (int): The size of the intermediate layer.
  • hidden_act (str or function): The activation function to use in the hidden layer. If a string is provided, the corresponding function will be retrieved from the ACT2FN dictionary.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
def __init__(self, config):
    """Initializes an instance of the MSErnieIntermediate class.

    Args:
        self: An instance of the MSErnieIntermediate class.
        config:
            A configuration object that contains the following attributes:

            - hidden_size (int): The size of the hidden layer.
            - intermediate_size (int): The size of the intermediate layer.
            - hidden_act (str or function): The activation function to use in the hidden layer.
            If a string is provided, the corresponding function will be retrieved from the ACT2FN dictionary.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
    if isinstance(config.hidden_act, str):
        self.intermediate_act_fn = ACT2FN[config.hidden_act]
    else:
        self.intermediate_act_fn = config.hidden_act

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieIntermediate.forward(hidden_states)

Method to forward intermediate hidden states in the MSErnieIntermediate class.

PARAMETER DESCRIPTION
self

The instance of the MSErnieIntermediate class.

TYPE: MSErnieIntermediate

hidden_states

A tensor containing the hidden states to be processed.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The processed hidden states after passing through the dense layer and activation function.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    Method to forward intermediate hidden states in the MSErnieIntermediate class.

    Args:
        self (MSErnieIntermediate): The instance of the MSErnieIntermediate class.
        hidden_states (mindspore.Tensor): A tensor containing the hidden states to be processed.

    Returns:
        mindspore.Tensor: The processed hidden states after passing through the dense layer and activation function.

    Raises:
        None
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.intermediate_act_fn(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLMPredictionHead

Bases: Module

This class represents a prediction head for the MSErnie language model, which is used for language modeling tasks. It is a subclass of nn.Module.

ATTRIBUTE DESCRIPTION
transform

An instance of the MSErniePredictionHeadTransform class that applies transformations to the input hidden states.

TYPE: MSErniePredictionHeadTransform

decoder

A fully connected layer that takes the transformed hidden states as input and produces predictions.

TYPE: Linear

bias

The bias term used in the fully connected layer.

TYPE: Parameter

METHOD DESCRIPTION
__init__

Initializes an instance of the MSErnieLMPredictionHead class.
forward

Applies transformations and produces predictions based on the input hidden states.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
class MSErnieLMPredictionHead(nn.Module):

    """
    This class represents a prediction head for the MSErnie language model, which is used for language modeling tasks.
    It is a subclass of `nn.Module`.

    Attributes:
        transform (MSErniePredictionHeadTransform):
            An instance of the MSErniePredictionHeadTransform class that applies transformations to the input hidden states.
        decoder (nn.Linear):
            A fully connected layer that takes the transformed hidden states as input and produces predictions.
        bias (Parameter): The bias term used in the fully connected layer.

    Methods:
        __init__: Initializes an instance of the MSErnieLMPredictionHead class.
        forward: Applies transformations and produces predictions based on the input hidden states.

    """
    def __init__(self, config):
        """
        Initialize the MSErnieLMPredictionHead class.

        Args:
            self: The current instance of the class.
            config: An object containing configuration settings for the prediction head.
                It is expected to have attributes including 'hidden_size' and 'vocab_size'.
                'hidden_size' specifies the size of the hidden layer, and 'vocab_size' specifies
                the size of the vocabulary.

        Returns:
            None:
                This method initializes various components of the prediction head such as the transform, decoder, and bias.

        Raises:
            None.
        """
        super().__init__()
        self.transform = MSErniePredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = Parameter(ops.zeros(config.vocab_size), 'bias')

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        """
        Constructs the MSErnieLMPredictionHead.

        Args:
            self (MSErnieLMPredictionHead): An instance of the MSErnieLMPredictionHead class.
            hidden_states (Tensor): The input hidden states. Expected shape is (batch_size, sequence_length, hidden_size).

        Returns:
            None

        Raises:
            None
        """
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLMPredictionHead.__init__(config)

Initialize the MSErnieLMPredictionHead class.

PARAMETER DESCRIPTION
self

The current instance of the class.
config

An object containing configuration settings for the prediction head. It is expected to have attributes including 'hidden_size' and 'vocab_size'. 'hidden_size' specifies the size of the hidden layer, and 'vocab_size' specifies the size of the vocabulary.
RETURNS DESCRIPTION
None

This method initializes various components of the prediction head such as the transform, decoder, and bias.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
def __init__(self, config):
    """
    Initialize the MSErnieLMPredictionHead class.

    Args:
        self: The current instance of the class.
        config: An object containing configuration settings for the prediction head.
            It is expected to have attributes including 'hidden_size' and 'vocab_size'.
            'hidden_size' specifies the size of the hidden layer, and 'vocab_size' specifies
            the size of the vocabulary.

    Returns:
        None:
            This method initializes various components of the prediction head such as the transform, decoder, and bias.

    Raises:
        None.
    """
    super().__init__()
    self.transform = MSErniePredictionHeadTransform(config)

    # The output weights are the same as the input embeddings, but there is
    # an output-only bias for each token.
    self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

    self.bias = Parameter(ops.zeros(config.vocab_size), 'bias')

    # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
    self.decoder.bias = self.bias

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLMPredictionHead.forward(hidden_states)

Constructs the MSErnieLMPredictionHead.

PARAMETER DESCRIPTION
self

An instance of the MSErnieLMPredictionHead class.

TYPE: MSErnieLMPredictionHead

hidden_states

The input hidden states. Expected shape is (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
def forward(self, hidden_states):
    """
    Constructs the MSErnieLMPredictionHead.

    Args:
        self (MSErnieLMPredictionHead): An instance of the MSErnieLMPredictionHead class.
        hidden_states (Tensor): The input hidden states. Expected shape is (batch_size, sequence_length, hidden_size).

    Returns:
        None

    Raises:
        None
    """
    hidden_states = self.transform(hidden_states)
    hidden_states = self.decoder(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLayer

Bases: Module

This class represents a layer of the MSErnie model, designed for natural language processing tasks. The MSErnieLayer class is responsible for handling self-attention and cross-attention mechanisms within the model. It inherits from nn.Module and contains methods for initialization, forwarding the layer, and performing feed-forward operations on the attention output.

ATTRIBUTE DESCRIPTION
chunk_size_feed_forward

The size of the chunk for feed-forward operations.
seq_len_dim

The dimension of the sequence length.
attention

The self-attention mechanism used in the layer.
is_decoder

A flag indicating whether the layer is used as a decoder in the model.
add_cross_attention

A flag indicating whether cross-attention is added to the layer.
crossattention

The cross-attention mechanism used in the layer.
intermediate

The intermediate layer in the feed-forward network.
output

The output layer in the feed-forward network.
METHOD DESCRIPTION
__init__

Initializes the MSErnieLayer with the provided configuration.
forward

Constructs the layer by processing the input hidden states and optional arguments.
feed_forward_chunk

Performs feed-forward operations on the attention output to generate the final layer output.
Note

If cross-attention is added, the layer should be used as a decoder model. Instantiation with cross-attention layers requires setting config.add_cross_attention=True. The forward method processes hidden states and optional arguments to generate the final outputs. The feed_forward_chunk method handles the feed-forward operations on the attention output to produce the layer output.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
class MSErnieLayer(nn.Module):

    """
    This class represents a layer of the MSErnie model, designed for natural language processing tasks.
    The MSErnieLayer class is responsible for handling self-attention and cross-attention mechanisms within the model.
    It inherits from nn.Module and contains methods for initialization, forwarding the layer,
    and performing feed-forward operations on the attention output.

    Attributes:
        chunk_size_feed_forward: The size of the chunk for feed-forward operations.
        seq_len_dim: The dimension of the sequence length.
        attention: The self-attention mechanism used in the layer.
        is_decoder: A flag indicating whether the layer is used as a decoder in the model.
        add_cross_attention: A flag indicating whether cross-attention is added to the layer.
        crossattention: The cross-attention mechanism used in the layer.
        intermediate: The intermediate layer in the feed-forward network.
        output: The output layer in the feed-forward network.

    Methods:
        __init__: Initializes the MSErnieLayer with the provided configuration.
        forward: Constructs the layer by processing the input hidden states and optional arguments.
        feed_forward_chunk: Performs feed-forward operations on the attention output to generate the final layer output.

    Note:
        If cross-attention is added, the layer should be used as a decoder model.
        Instantiation with cross-attention layers requires setting `config.add_cross_attention=True`.
        The forward method processes hidden states and optional arguments to generate the final outputs.
        The feed_forward_chunk method handles the feed-forward operations on the attention output to produce the layer output.
    """
    def __init__(self, config):
        """
        Initializes a MSErnieLayer object with the given configuration.

        Args:
            self: The MSErnieLayer instance.
            config:
                An object containing the configuration parameters for the MSErnieLayer.

                - chunk_size_feed_forward (int): The chunk size for feed forward operations.
                - is_decoder (bool): Indicates if the layer is used as a decoder model.
                - add_cross_attention (bool): Indicates if cross attention is added.
                - position_embedding_type (str): The type of position embedding for cross attention.
                Only applicable if add_cross_attention is True.

        Returns:
            None.

        Raises:
            ValueError: If add_cross_attention is True and the layer is not used as a decoder model.

        """
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = MSErnieAttention(config)
        self.is_decoder = config.is_decoder
        self.add_cross_attention = config.add_cross_attention
        if self.add_cross_attention:
            if not self.is_decoder:
                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
            self.crossattention = MSErnieAttention(config, position_embedding_type="absolute")
        self.intermediate = MSErnieIntermediate(config)
        self.output = MSErnieOutput(config)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        Constructs an MSErnieLayer.

        This method applies the MSErnie layer to the input hidden states and returns the output of the layer.
        The MSErnie layer consists of self-attention, cross-attention (if decoder), and feed-forward sublayers.

        Args:
            self: The object instance.
            hidden_states (mindspore.Tensor): The input hidden states. Shape: (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]):
                The attention mask indicating which positions should be attended to and which should be ignored.
                Shape: (batch_size, sequence_length).
            head_mask (Optional[mindspore.Tensor]):
                The mask for the individual attention heads.
                Shape: (num_heads,) or (num_layers, num_heads) or (batch_size, num_heads, sequence_length, sequence_length).
            encoder_hidden_states (Optional[mindspore.Tensor]):
                The hidden states of the encoder if cross-attention is enabled.
                Shape: (batch_size, encoder_sequence_length, hidden_size).
            encoder_attention_mask (Optional[mindspore.Tensor]):
                The attention mask for the encoder if cross-attention is enabled.
                Shape: (batch_size, encoder_sequence_length).
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                The cached key-value pairs of the self-attention and cross-attention layers from previous steps.
                Shape: (2, num_layers, num_heads, sequence_length, key_value_size).
            output_attentions (Optional[bool]): Whether to output the attention weights. Default: False.

        Returns:
            Tuple[mindspore.Tensor]:
                A tuple containing the output of the MSErnie layer.
                The first element is the output of the feed-forward sublayer.
                If the layer is a decoder, the tuple also includes the cached key-value pairs for self-attention
                and cross-attention.

        Raises:
            ValueError: If `encoder_hidden_states` are provided but the model is not instantiated with cross-attention
                layers by setting `config.add_cross_attention=True`.
        """
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        self_attention_outputs = self.attention(
            hidden_states,
            attention_mask,
            head_mask,
            output_attentions=output_attentions,
            past_key_value=self_attn_past_key_value,
        )
        attention_output = self_attention_outputs[0]

        # if decoder, the last output is tuple of self-attn cache
        if self.is_decoder:
            outputs = self_attention_outputs[1:-1]
            present_key_value = self_attention_outputs[-1]
        else:
            outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
            present_key_value = None

        cross_attn_present_key_value = None
        if self.is_decoder and encoder_hidden_states is not None:
            if not hasattr(self, "crossattention"):
                raise ValueError(
                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                    " by setting `config.add_cross_attention=True`"
                )

            # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask,
                head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                cross_attn_past_key_value,
                output_attentions,
            )
            attention_output = cross_attention_outputs[0]
            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

            # add cross-attn cache to positions 3,4 of present_key_value tuple
            cross_attn_present_key_value = cross_attention_outputs[-1]
            present_key_value = present_key_value + cross_attn_present_key_value

        # do not support `apply_chunking_to_forward` on graph mode
        # layer_output = apply_chunking_to_forward(
        #     self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
        # )
        layer_output = self.feed_forward_chunk(attention_output)
        outputs = (layer_output,) + outputs

        # if decoder, return the attn key/values as the last output
        if self.is_decoder:
            outputs = outputs + (present_key_value,)

        return outputs

    def feed_forward_chunk(self, attention_output):
        """
        Performs a feed forward chunk operation on the given attention output.

        Args:
            self (MSErnieLayer): An instance of the MSErnieLayer class.
            attention_output: The attention output tensor to be processed.
                It should be a tensor of shape (batch_size, sequence_length, hidden_size).

        Returns:
            None: This method does not directly return any value. Instead, it updates the layer output.

        Raises:
            None.
        """
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLayer.__init__(config)

Initializes a MSErnieLayer object with the given configuration.

PARAMETER DESCRIPTION
self

The MSErnieLayer instance.
config

An object containing the configuration parameters for the MSErnieLayer.

  • chunk_size_feed_forward (int): The chunk size for feed forward operations.
  • is_decoder (bool): Indicates if the layer is used as a decoder model.
  • add_cross_attention (bool): Indicates if cross attention is added.
  • position_embedding_type (str): The type of position embedding for cross attention. Only applicable if add_cross_attention is True.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
ValueError

If add_cross_attention is True and the layer is not used as a decoder model.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
def __init__(self, config):
    """
    Initializes a MSErnieLayer object with the given configuration.

    Args:
        self: The MSErnieLayer instance.
        config:
            An object containing the configuration parameters for the MSErnieLayer.

            - chunk_size_feed_forward (int): The chunk size for feed forward operations.
            - is_decoder (bool): Indicates if the layer is used as a decoder model.
            - add_cross_attention (bool): Indicates if cross attention is added.
            - position_embedding_type (str): The type of position embedding for cross attention.
            Only applicable if add_cross_attention is True.

    Returns:
        None.

    Raises:
        ValueError: If add_cross_attention is True and the layer is not used as a decoder model.

    """
    super().__init__()
    self.chunk_size_feed_forward = config.chunk_size_feed_forward
    self.seq_len_dim = 1
    self.attention = MSErnieAttention(config)
    self.is_decoder = config.is_decoder
    self.add_cross_attention = config.add_cross_attention
    if self.add_cross_attention:
        if not self.is_decoder:
            raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
        self.crossattention = MSErnieAttention(config, position_embedding_type="absolute")
    self.intermediate = MSErnieIntermediate(config)
    self.output = MSErnieOutput(config)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLayer.feed_forward_chunk(attention_output)

Performs a feed forward chunk operation on the given attention output.

PARAMETER DESCRIPTION
self

An instance of the MSErnieLayer class.

TYPE: MSErnieLayer

attention_output

The attention output tensor to be processed. It should be a tensor of shape (batch_size, sequence_length, hidden_size).
RETURNS DESCRIPTION
None

This method does not directly return any value. Instead, it updates the layer output.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
def feed_forward_chunk(self, attention_output):
    """
    Performs a feed forward chunk operation on the given attention output.

    Args:
        self (MSErnieLayer): An instance of the MSErnieLayer class.
        attention_output: The attention output tensor to be processed.
            It should be a tensor of shape (batch_size, sequence_length, hidden_size).

    Returns:
        None: This method does not directly return any value. Instead, it updates the layer output.

    Raises:
        None.
    """
    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)
    return layer_output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieLayer.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

Constructs an MSErnieLayer.

This method applies the MSErnie layer to the input hidden states and returns the output of the layer. The MSErnie layer consists of self-attention, cross-attention (if decoder), and feed-forward sublayers.

PARAMETER DESCRIPTION
self

The object instance.
hidden_states

The input hidden states. Shape: (batch_size, sequence_length, hidden_size).

TYPE: Tensor

attention_mask

The attention mask indicating which positions should be attended to and which should be ignored. Shape: (batch_size, sequence_length).

TYPE: Optional[Tensor] DEFAULT: None

head_mask

The mask for the individual attention heads. Shape: (num_heads,) or (num_layers, num_heads) or (batch_size, num_heads, sequence_length, sequence_length).

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

The hidden states of the encoder if cross-attention is enabled. Shape: (batch_size, encoder_sequence_length, hidden_size).

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

The attention mask for the encoder if cross-attention is enabled. Shape: (batch_size, encoder_sequence_length).

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

The cached key-value pairs of the self-attention and cross-attention layers from previous steps. Shape: (2, num_layers, num_heads, sequence_length, key_value_size).

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

Whether to output the attention weights. Default: False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: A tuple containing the output of the MSErnie layer. The first element is the output of the feed-forward sublayer. If the layer is a decoder, the tuple also includes the cached key-value pairs for self-attention and cross-attention.
RAISES DESCRIPTION
ValueError

If encoder_hidden_states are provided but the model is not instantiated with cross-attention layers by setting config.add_cross_attention=True.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    Constructs an MSErnieLayer.

    This method applies the MSErnie layer to the input hidden states and returns the output of the layer.
    The MSErnie layer consists of self-attention, cross-attention (if decoder), and feed-forward sublayers.

    Args:
        self: The object instance.
        hidden_states (mindspore.Tensor): The input hidden states. Shape: (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]):
            The attention mask indicating which positions should be attended to and which should be ignored.
            Shape: (batch_size, sequence_length).
        head_mask (Optional[mindspore.Tensor]):
            The mask for the individual attention heads.
            Shape: (num_heads,) or (num_layers, num_heads) or (batch_size, num_heads, sequence_length, sequence_length).
        encoder_hidden_states (Optional[mindspore.Tensor]):
            The hidden states of the encoder if cross-attention is enabled.
            Shape: (batch_size, encoder_sequence_length, hidden_size).
        encoder_attention_mask (Optional[mindspore.Tensor]):
            The attention mask for the encoder if cross-attention is enabled.
            Shape: (batch_size, encoder_sequence_length).
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            The cached key-value pairs of the self-attention and cross-attention layers from previous steps.
            Shape: (2, num_layers, num_heads, sequence_length, key_value_size).
        output_attentions (Optional[bool]): Whether to output the attention weights. Default: False.

    Returns:
        Tuple[mindspore.Tensor]:
            A tuple containing the output of the MSErnie layer.
            The first element is the output of the feed-forward sublayer.
            If the layer is a decoder, the tuple also includes the cached key-value pairs for self-attention
            and cross-attention.

    Raises:
        ValueError: If `encoder_hidden_states` are provided but the model is not instantiated with cross-attention
            layers by setting `config.add_cross_attention=True`.
    """
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    self_attention_outputs = self.attention(
        hidden_states,
        attention_mask,
        head_mask,
        output_attentions=output_attentions,
        past_key_value=self_attn_past_key_value,
    )
    attention_output = self_attention_outputs[0]

    # if decoder, the last output is tuple of self-attn cache
    if self.is_decoder:
        outputs = self_attention_outputs[1:-1]
        present_key_value = self_attention_outputs[-1]
    else:
        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
        present_key_value = None

    cross_attn_present_key_value = None
    if self.is_decoder and encoder_hidden_states is not None:
        if not hasattr(self, "crossattention"):
            raise ValueError(
                f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                " by setting `config.add_cross_attention=True`"
            )

        # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        cross_attention_outputs = self.crossattention(
            attention_output,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            cross_attn_past_key_value,
            output_attentions,
        )
        attention_output = cross_attention_outputs[0]
        outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

        # add cross-attn cache to positions 3,4 of present_key_value tuple
        cross_attn_present_key_value = cross_attention_outputs[-1]
        present_key_value = present_key_value + cross_attn_present_key_value

    # do not support `apply_chunking_to_forward` on graph mode
    # layer_output = apply_chunking_to_forward(
    #     self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
    # )
    layer_output = self.feed_forward_chunk(attention_output)
    outputs = (layer_output,) + outputs

    # if decoder, return the attn key/values as the last output
    if self.is_decoder:
        outputs = outputs + (present_key_value,)

    return outputs

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieModel

Bases: MSErniePreTrainedModel

The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in Attention is all you need by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

To behave as an decoder the model needs to be initialized with the is_decoder argument of the configuration set to True. To be used in a Seq2Seq model, the model needs to initialized with both is_decoder argument and add_cross_attention set to True; an encoder_hidden_states is then expected as an input to the forward pass.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
class MSErnieModel(MSErniePreTrainedModel):
    """
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    """
    # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Ernie
    def __init__(self, config, add_pooling_layer=True):
        """
        Initializes an instance of the MSErnieModel class.

        Args:
            self: The object instance.
            config (object): The configuration object that contains the model parameters.
            add_pooling_layer (bool): Indicates whether to add a pooling layer. Default is True.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.config = config

        self.embeddings = MSErnieEmbeddings(config)
        self.encoder = MSErnieEncoder(config)

        self.pooler = MSErniePooler(config) if add_pooling_layer else None

        # Initialize weights and apply final processing
        self.post_init()

    # Copied from transformers.models.bert.modeling_bert.BertModel.get_input_embeddings
    def get_input_embeddings(self):
        """
        Get the input embeddings for the MSErnieModel.

        Args:
            self (MSErnieModel): An instance of the MSErnieModel class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.embeddings.word_embeddings

    # Copied from transformers.models.bert.modeling_bert.BertModel.set_input_embeddings
    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the MSErnieModel.

        Args:
            self (MSErnieModel): The instance of the MSErnieModel class.
            value: The new input embeddings to be set. This should be of type torch.Tensor.

        Returns:
            None.

        Raises:
            None.
        """
        self.embeddings.word_embeddings = value

    # Copied from transformers.models.bert.modeling_bert.BertModel._prune_heads
    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        task_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], dict]:
        r"""
        Args:
            encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
                the model is configured as a decoder.
            encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
                the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4 tensors
                of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
                Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
                `past_key_values`).
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )

        if self.config.is_decoder:
            use_cache = use_cache if use_cache is not None else self.config.use_cache
        else:
            use_cache = False

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            input_shape = input_ids.shape
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        batch_size, seq_length = input_shape

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if attention_mask is None:
            attention_mask = ops.ones(((batch_size, seq_length + past_key_values_length)))

        if token_type_ids is None:
            if hasattr(self.embeddings, "token_type_ids"):
                buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

        # If a 2D or 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = ops.ones(encoder_hidden_shape)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            input_ids=input_ids,
            position_ids=position_ids,
            token_type_ids=token_type_ids,
            task_type_ids=task_type_ids,
            inputs_embeds=inputs_embeds,
            past_key_values_length=past_key_values_length,
        )
        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )
        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

        return (sequence_output, pooled_output) + encoder_outputs[1:]

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieModel.__init__(config, add_pooling_layer=True)

Initializes an instance of the MSErnieModel class.

PARAMETER DESCRIPTION
self

The object instance.
config

The configuration object that contains the model parameters.

TYPE: object

add_pooling_layer

Indicates whether to add a pooling layer. Default is True.

TYPE: bool DEFAULT: True

RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
def __init__(self, config, add_pooling_layer=True):
    """
    Initializes an instance of the MSErnieModel class.

    Args:
        self: The object instance.
        config (object): The configuration object that contains the model parameters.
        add_pooling_layer (bool): Indicates whether to add a pooling layer. Default is True.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.config = config

    self.embeddings = MSErnieEmbeddings(config)
    self.encoder = MSErnieEncoder(config)

    self.pooler = MSErniePooler(config) if add_pooling_layer else None

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieModel.forward(input_ids=None, attention_mask=None, token_type_ids=None, task_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
encoder_hidden_states

Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder.

TYPE: (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None

encoder_attention_mask

Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in [0, 1]:

  • 1 for tokens that are not masked,
  • 0 for tokens that are masked.

TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

use_cache

If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values).

TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    task_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], dict]:
    r"""
    Args:
        encoder_hidden_states  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(mindspore.Tensor))` of length `config.n_layers` with each tuple having 4 tensors
            of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )

    if self.config.is_decoder:
        use_cache = use_cache if use_cache is not None else self.config.use_cache
    else:
        use_cache = False

    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        input_shape = input_ids.shape
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    batch_size, seq_length = input_shape

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if attention_mask is None:
        attention_mask = ops.ones(((batch_size, seq_length + past_key_values_length)))

    if token_type_ids is None:
        if hasattr(self.embeddings, "token_type_ids"):
            buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(input_shape, dtype=mindspore.int64)

    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

    # If a 2D or 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if self.config.is_decoder and encoder_hidden_states is not None:
        encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
        encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
            encoder_attention_mask = ops.ones(encoder_hidden_shape)
        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
    else:
        encoder_extended_attention_mask = None

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x n_heads x N x N
    # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
    # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
    head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

    embedding_output = self.embeddings(
        input_ids=input_ids,
        position_ids=position_ids,
        token_type_ids=token_type_ids,
        task_type_ids=task_type_ids,
        inputs_embeds=inputs_embeds,
        past_key_values_length=past_key_values_length,
    )
    encoder_outputs = self.encoder(
        embedding_output,
        attention_mask=extended_attention_mask,
        head_mask=head_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_extended_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )
    sequence_output = encoder_outputs[0]
    pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

    return (sequence_output, pooled_output) + encoder_outputs[1:]

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieModel.get_input_embeddings()

Get the input embeddings for the MSErnieModel.

PARAMETER DESCRIPTION
self

An instance of the MSErnieModel class.

TYPE: MSErnieModel

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
def get_input_embeddings(self):
    """
    Get the input embeddings for the MSErnieModel.

    Args:
        self (MSErnieModel): An instance of the MSErnieModel class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.embeddings.word_embeddings

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieModel.set_input_embeddings(value)

Sets the input embeddings for the MSErnieModel.

PARAMETER DESCRIPTION
self

The instance of the MSErnieModel class.

TYPE: MSErnieModel

value

The new input embeddings to be set. This should be of type torch.Tensor.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the MSErnieModel.

    Args:
        self (MSErnieModel): The instance of the MSErnieModel class.
        value: The new input embeddings to be set. This should be of type torch.Tensor.

    Returns:
        None.

    Raises:
        None.
    """
    self.embeddings.word_embeddings = value

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyMLMHead

Bases: Module

This class represents a prediction head for Masked Language Modeling (MLM) tasks using the MSErnie model.

This class inherits from nn.Module and is responsible for forwarding prediction scores based on the sequence output from the MSErnie model.

ATTRIBUTE DESCRIPTION
predictions

Instance of MSErnieLMPredictionHead used for generating prediction scores.

TYPE: MSErnieLMPredictionHead

METHOD DESCRIPTION
forward

mindspore.Tensor) -> mindspore.Tensor: Constructs prediction scores based on the input sequence_output tensor.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
class MSErnieOnlyMLMHead(nn.Module):

    """
    This class represents a prediction head for Masked Language Modeling (MLM) tasks using the MSErnie model.

    This class inherits from nn.Module and is responsible for forwarding prediction scores based on the sequence output
    from the MSErnie model.

    Attributes:
        predictions (MSErnieLMPredictionHead): Instance of MSErnieLMPredictionHead used for generating prediction scores.

    Methods:
        forward(sequence_output: mindspore.Tensor) -> mindspore.Tensor:
            Constructs prediction scores based on the input sequence_output tensor.
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieOnlyMLMHead class.

        Args:
            self: The instance of the MSErnieOnlyMLMHead class.

            config:
                A configuration object containing settings for the MSErnieOnlyMLMHead instance.

                - Type: Any
                - Purpose: Specifies the configuration settings for the model.
                - Restrictions: The config object must be compatible with the MSErnieOnlyMLMHead class.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.predictions = MSErnieLMPredictionHead(config)

    def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
        """
        This method forwards the masked language model (MLM) head for the MSErnie model.

        Args:
            self (MSErnieOnlyMLMHead): The instance of the MSErnieOnlyMLMHead class.
            sequence_output (mindspore.Tensor): The output tensor from the preceding layer, typically the encoder.
                It represents the sequence output that will be used for predicting masked tokens.

        Returns:
            mindspore.Tensor: The prediction scores tensor generated by the MLM head.
                This tensor contains the predicted scores for each token in the input sequence.

        Raises:
            None
        """
        prediction_scores = self.predictions(sequence_output)
        return prediction_scores

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyMLMHead.__init__(config)

Initializes an instance of the MSErnieOnlyMLMHead class.

PARAMETER DESCRIPTION
self

The instance of the MSErnieOnlyMLMHead class.
config

A configuration object containing settings for the MSErnieOnlyMLMHead instance.

  • Type: Any
  • Purpose: Specifies the configuration settings for the model.
  • Restrictions: The config object must be compatible with the MSErnieOnlyMLMHead class.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
def __init__(self, config):
    """
    Initializes an instance of the MSErnieOnlyMLMHead class.

    Args:
        self: The instance of the MSErnieOnlyMLMHead class.

        config:
            A configuration object containing settings for the MSErnieOnlyMLMHead instance.

            - Type: Any
            - Purpose: Specifies the configuration settings for the model.
            - Restrictions: The config object must be compatible with the MSErnieOnlyMLMHead class.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.predictions = MSErnieLMPredictionHead(config)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyMLMHead.forward(sequence_output)

This method forwards the masked language model (MLM) head for the MSErnie model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieOnlyMLMHead class.

TYPE: MSErnieOnlyMLMHead

sequence_output

The output tensor from the preceding layer, typically the encoder. It represents the sequence output that will be used for predicting masked tokens.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The prediction scores tensor generated by the MLM head. This tensor contains the predicted scores for each token in the input sequence.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
    """
    This method forwards the masked language model (MLM) head for the MSErnie model.

    Args:
        self (MSErnieOnlyMLMHead): The instance of the MSErnieOnlyMLMHead class.
        sequence_output (mindspore.Tensor): The output tensor from the preceding layer, typically the encoder.
            It represents the sequence output that will be used for predicting masked tokens.

    Returns:
        mindspore.Tensor: The prediction scores tensor generated by the MLM head.
            This tensor contains the predicted scores for each token in the input sequence.

    Raises:
        None
    """
    prediction_scores = self.predictions(sequence_output)
    return prediction_scores

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyNSPHead

Bases: Module

The MSErnieOnlyNSPHead class is a subclass of nn.Module that represents a neural network head for the MSErnie model, specifically designed for the Next Sentence Prediction (NSP) task.

This class initializes an instance of MSErnieOnlyNSPHead with a configuration object, which is used to define the hidden size of the model. The config parameter should be an instance of MSErnieConfig or a class derived from it.

The forward method takes a pooled_output tensor as input and computes the next sentence prediction score using a dense layer. The pooled_output tensor should be of shape (batch_size, hidden_size), where hidden_size is the size of the hidden layers in the model.

The seq_relationship attribute is an instance of nn.Linear that performs the computation of the next sentence prediction score. It takes the pooled_output tensor as input and returns a tensor of shape (batch_size, 2), where the second dimension represents the probability scores for two possible sentence relationships.

The forward method returns the computed seq_relationship_score tensor.

Example 
>>> config = MSErnieConfig(hidden_size=768)
>>> head = MSErnieOnlyNSPHead(config)
>>> output = head.forward(pooled_output)
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
class MSErnieOnlyNSPHead(nn.Module):

    """
    The `MSErnieOnlyNSPHead` class is a subclass of `nn.Module` that represents a neural network head for the MSErnie model,
    specifically designed for the Next Sentence Prediction (NSP) task.

    This class initializes an instance of `MSErnieOnlyNSPHead` with a configuration object, which is used to define
    the hidden size of the model.
    The `config` parameter should be an instance of `MSErnieConfig` or a class derived from it.

    The `forward` method takes a `pooled_output` tensor as input and computes the next sentence prediction score
    using a dense layer.
    The `pooled_output` tensor should be of shape (batch_size, hidden_size), where `hidden_size` is the size of
    the hidden layers in the model.

    The `seq_relationship` attribute is an instance of `nn.Linear` that performs the computation of the next sentence
    prediction score.
    It takes the `pooled_output` tensor as input and returns a tensor of shape (batch_size, 2),
    where the second dimension represents the probability scores for two possible sentence relationships.

    The `forward` method returns the computed `seq_relationship_score` tensor.

    Example:
        ```python
        >>> config = MSErnieConfig(hidden_size=768)
        >>> head = MSErnieOnlyNSPHead(config)
        >>> output = head.forward(pooled_output)
        ```
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieOnlyNSPHead class.

        Args:
            self (MSErnieOnlyNSPHead): An instance of the MSErnieOnlyNSPHead class.
            config:
                A configuration object containing the model's settings.

                - Type: Any valid object
                - Purpose: Specifies the configuration parameters for the model.
                - Restrictions: None

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, pooled_output):
        """
        This method forwards the sequence relationship score based on the pooled output for the MSErnieOnlyNSPHead class.

        Args:
            self (object): The instance of the MSErnieOnlyNSPHead class.
            pooled_output (object): The pooled output obtained from the model.

        Returns:
            None: This method does not return any value, but calculates the sequence relationship score
                based on the pooled output.

        Raises:
            NNone.
        """
        seq_relationship_score = self.seq_relationship(pooled_output)
        return seq_relationship_score

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyNSPHead.__init__(config)

Initializes an instance of the MSErnieOnlyNSPHead class.

PARAMETER DESCRIPTION
self

An instance of the MSErnieOnlyNSPHead class.

TYPE: MSErnieOnlyNSPHead

config

A configuration object containing the model's settings.

  • Type: Any valid object
  • Purpose: Specifies the configuration parameters for the model.
  • Restrictions: None
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
def __init__(self, config):
    """
    Initializes an instance of the MSErnieOnlyNSPHead class.

    Args:
        self (MSErnieOnlyNSPHead): An instance of the MSErnieOnlyNSPHead class.
        config:
            A configuration object containing the model's settings.

            - Type: Any valid object
            - Purpose: Specifies the configuration parameters for the model.
            - Restrictions: None

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOnlyNSPHead.forward(pooled_output)

This method forwards the sequence relationship score based on the pooled output for the MSErnieOnlyNSPHead class.

PARAMETER DESCRIPTION
self

The instance of the MSErnieOnlyNSPHead class.

TYPE: object

pooled_output

The pooled output obtained from the model.

TYPE: object

RETURNS DESCRIPTION
None

This method does not return any value, but calculates the sequence relationship score based on the pooled output.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
def forward(self, pooled_output):
    """
    This method forwards the sequence relationship score based on the pooled output for the MSErnieOnlyNSPHead class.

    Args:
        self (object): The instance of the MSErnieOnlyNSPHead class.
        pooled_output (object): The pooled output obtained from the model.

    Returns:
        None: This method does not return any value, but calculates the sequence relationship score
            based on the pooled output.

    Raises:
        NNone.
    """
    seq_relationship_score = self.seq_relationship(pooled_output)
    return seq_relationship_score

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOutput

Bases: Module

MSErnieOutput is a class that represents the output layer for the MSErnie model in MindSpore. This class inherits from nn.Module and contains methods to process hidden states and input tensors.

ATTRIBUTE DESCRIPTION
dense

A fully connected layer to transform the hidden states.

TYPE: Linear

LayerNorm

A layer normalization module to normalize the hidden states.

TYPE: LayerNorm

dropout

A dropout layer to apply dropout to the hidden states.

TYPE: Dropout

METHOD DESCRIPTION
__init__

Initializes the MSErnieOutput class with the provided configuration.
forward

Processes the hidden states and input tensor to generate the output tensor.
Note

This class is specifically designed for the MSErnie model in MindSpore and should be used as the final output layer.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
class MSErnieOutput(nn.Module):

    """
    MSErnieOutput is a class that represents the output layer for the MSErnie model in MindSpore.
    This class inherits from nn.Module and contains methods to process hidden states and input tensors.

    Attributes:
        dense (nn.Linear): A fully connected layer to transform the hidden states.
        LayerNorm (nn.LayerNorm): A layer normalization module to normalize the hidden states.
        dropout (nn.Dropout): A dropout layer to apply dropout to the hidden states.

    Methods:
        __init__: Initializes the MSErnieOutput class with the provided configuration.
        forward: Processes the hidden states and input tensor to generate the output tensor.

    Note:
        This class is specifically designed for the MSErnie model in MindSpore and should be used as the final output layer.
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieOutput class.

        Args:
            self: The instance of the class.
            config: An object of type 'Config' containing configuration settings for the MSErnieOutput.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the output tensor of the MSErnie model.

        Args:
            self (MSErnieOutput): The instance of the MSErnieOutput class.
            hidden_states (mindspore.Tensor): The hidden states tensor generated by the model.
                Shape: (batch_size, sequence_length, hidden_size).
            input_tensor (mindspore.Tensor): The input tensor to the layer.
                Shape: (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: The output tensor of the MSErnie model after processing the hidden states and input tensor.
                Shape: (batch_size, sequence_length, hidden_size).

        Raises:
            TypeError: If the input parameters are not of the expected types.
            ValueError: If the shapes of the input tensors are incompatible for addition.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOutput.__init__(config)

Initializes an instance of the MSErnieOutput class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An object of type 'Config' containing configuration settings for the MSErnieOutput.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
def __init__(self, config):
    """
    Initializes an instance of the MSErnieOutput class.

    Args:
        self: The instance of the class.
        config: An object of type 'Config' containing configuration settings for the MSErnieOutput.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieOutput.forward(hidden_states, input_tensor)

Constructs the output tensor of the MSErnie model.

PARAMETER DESCRIPTION
self

The instance of the MSErnieOutput class.

TYPE: MSErnieOutput

hidden_states

The hidden states tensor generated by the model. Shape: (batch_size, sequence_length, hidden_size).

TYPE: Tensor

input_tensor

The input tensor to the layer. Shape: (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The output tensor of the MSErnie model after processing the hidden states and input tensor. Shape: (batch_size, sequence_length, hidden_size).
RAISES DESCRIPTION
TypeError

If the input parameters are not of the expected types.
ValueError

If the shapes of the input tensors are incompatible for addition.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the output tensor of the MSErnie model.

    Args:
        self (MSErnieOutput): The instance of the MSErnieOutput class.
        hidden_states (mindspore.Tensor): The hidden states tensor generated by the model.
            Shape: (batch_size, sequence_length, hidden_size).
        input_tensor (mindspore.Tensor): The input tensor to the layer.
            Shape: (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: The output tensor of the MSErnie model after processing the hidden states and input tensor.
            Shape: (batch_size, sequence_length, hidden_size).

    Raises:
        TypeError: If the input parameters are not of the expected types.
        ValueError: If the shapes of the input tensors are incompatible for addition.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePooler

Bases: Module

This class represents a pooler for the MSErnie model. It inherits from nn.Module.

ATTRIBUTE DESCRIPTION
dense

A fully connected layer used for pooling operations.

TYPE: Linear

activation

An activation function applied to the pooled output.

TYPE: Tanh

METHOD DESCRIPTION
__init__

Initializes the MSErniePooler class.
forward

Constructs the pooled output tensor.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
class MSErniePooler(nn.Module):

    """
    This class represents a pooler for the MSErnie model. It inherits from nn.Module.

    Attributes:
        dense (nn.Linear): A fully connected layer used for pooling operations.
        activation (nn.Tanh): An activation function applied to the pooled output.

    Methods:
        __init__: Initializes the MSErniePooler class.
        forward: Constructs the pooled output tensor.

    """
    def __init__(self, config):
        """
        __init__

        Initializes an instance of the MSErniePooler class.

        Args:
            self (MSErniePooler): The instance of the MSErniePooler class.
            config: The configuration object containing the parameters for the MSErniePooler instance.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected type.
            ValueError: If the config parameter does not contain the required attributes.
            RuntimeError: If there is an issue with initializing the dense layer or activation function.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        This method is part of the class MSErniePooler and is used to forward a pooled output from the given hidden states tensor.

        Args:
            self: The instance of the MSErniePooler class.
            hidden_states (mindspore.Tensor): A tensor containing the hidden states.
                It is expected to be of shape (batch_size, sequence_length, hidden_size)
                where batch_size represents the number of input sequences in the batch, sequence_length represents
                the length of the sequences, and hidden_size represents the size of the hidden states.
                The hidden states are the output of the Ernie model and are used to forward the pooled output.

        Returns:
            mindspore.Tensor: The forwarded pooled output tensor.
                It represents the aggregated representation of the input sequences and is of shape
                (batch_size, hidden_size) where batch_size represents the number of input sequences in the batch and
                hidden_size represents the size of the hidden states.

        Raises:
            None
        """
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token.
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePooler.__init__(config)

init

Initializes an instance of the MSErniePooler class.

PARAMETER DESCRIPTION
self

The instance of the MSErniePooler class.

TYPE: MSErniePooler

config

The configuration object containing the parameters for the MSErniePooler instance.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the config parameter is not of the expected type.
ValueError

If the config parameter does not contain the required attributes.
RuntimeError

If there is an issue with initializing the dense layer or activation function.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
def __init__(self, config):
    """
    __init__

    Initializes an instance of the MSErniePooler class.

    Args:
        self (MSErniePooler): The instance of the MSErniePooler class.
        config: The configuration object containing the parameters for the MSErniePooler instance.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected type.
        ValueError: If the config parameter does not contain the required attributes.
        RuntimeError: If there is an issue with initializing the dense layer or activation function.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.activation = nn.Tanh()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePooler.forward(hidden_states)

This method is part of the class MSErniePooler and is used to forward a pooled output from the given hidden states tensor.

PARAMETER DESCRIPTION
self

The instance of the MSErniePooler class.
hidden_states

A tensor containing the hidden states. It is expected to be of shape (batch_size, sequence_length, hidden_size) where batch_size represents the number of input sequences in the batch, sequence_length represents the length of the sequences, and hidden_size represents the size of the hidden states. The hidden states are the output of the Ernie model and are used to forward the pooled output.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The forwarded pooled output tensor. It represents the aggregated representation of the input sequences and is of shape (batch_size, hidden_size) where batch_size represents the number of input sequences in the batch and hidden_size represents the size of the hidden states.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    This method is part of the class MSErniePooler and is used to forward a pooled output from the given hidden states tensor.

    Args:
        self: The instance of the MSErniePooler class.
        hidden_states (mindspore.Tensor): A tensor containing the hidden states.
            It is expected to be of shape (batch_size, sequence_length, hidden_size)
            where batch_size represents the number of input sequences in the batch, sequence_length represents
            the length of the sequences, and hidden_size represents the size of the hidden states.
            The hidden states are the output of the Ernie model and are used to forward the pooled output.

    Returns:
        mindspore.Tensor: The forwarded pooled output tensor.
            It represents the aggregated representation of the input sequences and is of shape
            (batch_size, hidden_size) where batch_size represents the number of input sequences in the batch and
            hidden_size represents the size of the hidden states.

    Raises:
        None
    """
    # We "pool" the model by simply taking the hidden state corresponding
    # to the first token.
    first_token_tensor = hidden_states[:, 0]
    pooled_output = self.dense(first_token_tensor)
    pooled_output = self.activation(pooled_output)
    return pooled_output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
class MSErniePreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """
    config_class = ErnieConfig
    base_model_prefix = "ernie"
    supports_gradient_checkpointing = True

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(initializer(Normal(self.config.initializer_range),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.bias is not None:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, self.config.initializer_range, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))
        elif isinstance(cell, nn.LayerNorm):
            cell.weight.set_data(initializer('ones', cell.weight.shape, cell.weight.dtype))
            cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePreTrainingHeads

Bases: Module

This class represents the pre-training heads of the MSErnie model, which includes prediction scores and sequence relationship scores.

The class inherits from the nn.Module class.

ATTRIBUTE DESCRIPTION
predictions

An instance of the MSErnieLMPredictionHead class, responsible for generating prediction scores based on sequence outputs.

TYPE: MSErnieLMPredictionHead

seq_relationship

A fully connected layer that produces sequence relationship scores based on pooled outputs.

TYPE: Linear

METHOD DESCRIPTION
forward

Constructs the pre-training heads by generating prediction scores and sequence relationship scores based on the given sequence and pooled outputs.

Args:

  • sequence_output (Tensor): The sequence output from the MSErnie model.
  • pooled_output (Tensor): The pooled output from the MSErnie model.

Returns:

  • prediction_scores (Tensor): The prediction scores generated by the predictions module.
  • seq_relationship_score (Tensor): The sequence relationship scores generated by the seq_relationship module.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
class MSErniePreTrainingHeads(nn.Module):

    """
    This class represents the pre-training heads of the MSErnie model, which includes prediction scores and
    sequence relationship scores.

    The class inherits from the nn.Module class.

    Attributes:
        predictions (MSErnieLMPredictionHead):
            An instance of the MSErnieLMPredictionHead class, responsible for generating prediction scores
            based on sequence outputs.
        seq_relationship (nn.Linear):
            A fully connected layer that produces sequence relationship scores based on pooled outputs.

    Methods:
        forward(sequence_output, pooled_output):
            Constructs the pre-training heads by generating prediction scores and sequence relationship scores
            based on the given sequence and pooled outputs.

            Args:

            - sequence_output (Tensor): The sequence output from the MSErnie model.
            - pooled_output (Tensor): The pooled output from the MSErnie model.

            Returns:

            - prediction_scores (Tensor): The prediction scores generated by the predictions module.
            - seq_relationship_score (Tensor): The sequence relationship scores generated by the seq_relationship module.
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErniePreTrainingHeads class.

        Args:
            self (MSErniePreTrainingHeads): The instance of the class itself.
            config: An object containing the configuration parameters for the model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.predictions = MSErnieLMPredictionHead(config)
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, sequence_output, pooled_output):
        """
        This method forwards prediction scores and sequence relationship scores for pre-training tasks in the MSErnie model.

        Args:
            self (object): The instance of the MSErniePreTrainingHeads class.
            sequence_output (object):
                The output sequence generated by the model.

                - Type: Any
                - Purpose: Represents the input sequence for pre-training tasks.
                - Restrictions: Should be a valid output sequence object.
            pooled_output (object):
                The pooled output generated by the model.

                - Type: Any
                - Purpose: Represents the pooled output for pre-training tasks.
                - Restrictions: Should be a valid pooled output object.

        Returns:
            tuple:
                A tuple containing the prediction scores and sequence relationship score.

                - Type: tuple
                - Purpose: Contains the prediction scores for the input sequence and the sequence relationship score.
                - Restrictions: None

        Raises:
            None
        """
        prediction_scores = self.predictions(sequence_output)
        seq_relationship_score = self.seq_relationship(pooled_output)
        return prediction_scores, seq_relationship_score

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePreTrainingHeads.__init__(config)

Initializes an instance of the MSErniePreTrainingHeads class.

PARAMETER DESCRIPTION
self

The instance of the class itself.

TYPE: MSErniePreTrainingHeads

config

An object containing the configuration parameters for the model.
RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
def __init__(self, config):
    """
    Initializes an instance of the MSErniePreTrainingHeads class.

    Args:
        self (MSErniePreTrainingHeads): The instance of the class itself.
        config: An object containing the configuration parameters for the model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.predictions = MSErnieLMPredictionHead(config)
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePreTrainingHeads.forward(sequence_output, pooled_output)

This method forwards prediction scores and sequence relationship scores for pre-training tasks in the MSErnie model.

PARAMETER DESCRIPTION
self

The instance of the MSErniePreTrainingHeads class.

TYPE: object

sequence_output

The output sequence generated by the model.

  • Type: Any
  • Purpose: Represents the input sequence for pre-training tasks.
  • Restrictions: Should be a valid output sequence object.

TYPE: object

pooled_output

The pooled output generated by the model.

  • Type: Any
  • Purpose: Represents the pooled output for pre-training tasks.
  • Restrictions: Should be a valid pooled output object.

TYPE: object

RETURNS DESCRIPTION
tuple

A tuple containing the prediction scores and sequence relationship score.

  • Type: tuple
  • Purpose: Contains the prediction scores for the input sequence and the sequence relationship score.
  • Restrictions: None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
def forward(self, sequence_output, pooled_output):
    """
    This method forwards prediction scores and sequence relationship scores for pre-training tasks in the MSErnie model.

    Args:
        self (object): The instance of the MSErniePreTrainingHeads class.
        sequence_output (object):
            The output sequence generated by the model.

            - Type: Any
            - Purpose: Represents the input sequence for pre-training tasks.
            - Restrictions: Should be a valid output sequence object.
        pooled_output (object):
            The pooled output generated by the model.

            - Type: Any
            - Purpose: Represents the pooled output for pre-training tasks.
            - Restrictions: Should be a valid pooled output object.

    Returns:
        tuple:
            A tuple containing the prediction scores and sequence relationship score.

            - Type: tuple
            - Purpose: Contains the prediction scores for the input sequence and the sequence relationship score.
            - Restrictions: None

    Raises:
        None
    """
    prediction_scores = self.predictions(sequence_output)
    seq_relationship_score = self.seq_relationship(pooled_output)
    return prediction_scores, seq_relationship_score

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePredictionHeadTransform

Bases: Module

The MSErniePredictionHeadTransform class represents a transformation module for an ERNIE prediction head. This class inherits from nn.Module and is used to process hidden states for ERNIE predictions.

ATTRIBUTE DESCRIPTION
dense

A fully connected neural network layer with input and output size of config.hidden_size.
transform_act_fn

Activation function used for transforming hidden states.
LayerNorm

Layer normalization module with hidden size specified by config.hidden_size and epsilon specified by config.layer_norm_eps.
METHOD DESCRIPTION
__init__

Initializes the MSErniePredictionHeadTransform instance with the provided configuration.
forward

Applies transformations to the input hidden states and returns the processed hidden states.
Usage

Instantiate an MSErniePredictionHeadTransform object with the desired configuration and utilize the forward method to process hidden states for ERNIE predictions.

Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
class MSErniePredictionHeadTransform(nn.Module):

    '''
    The MSErniePredictionHeadTransform class represents a transformation module for an ERNIE prediction head.
    This class inherits from nn.Module and is used to process hidden states for ERNIE predictions.

    Attributes:
        dense: A fully connected neural network layer with input and output size of config.hidden_size.
        transform_act_fn: Activation function used for transforming hidden states.
        LayerNorm: Layer normalization module with hidden size specified by config.hidden_size
            and epsilon specified by config.layer_norm_eps.

    Methods:
        __init__: Initializes the MSErniePredictionHeadTransform instance with the provided configuration.
        forward: Applies transformations to the input hidden states and returns the processed hidden states.

    Usage:
        Instantiate an MSErniePredictionHeadTransform object with the desired configuration and utilize the
        forward method to process hidden states for ERNIE predictions.
    '''
    def __init__(self, config):
        """
        Initializes the MSErniePredictionHeadTransform class.

        Args:
            self (MSErniePredictionHeadTransform): The instance of the MSErniePredictionHeadTransform class.
            config:
                A configuration object containing settings for the transformation.

                - Type: object
                - Purpose: Specifies the configuration parameters for the transformation.
                - Restrictions: Must be a valid configuration object.

        Returns:
            None.

        Raises:
            TypeError: If the configuration object is not of the expected type.
            KeyError: If the specified hidden activation function is not found in the ACT2FN dictionary.
            ValueError: If there are issues with the provided configuration parameters.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        if isinstance(config.hidden_act, str):
            self.transform_act_fn = ACT2FN[config.hidden_act]
        else:
            self.transform_act_fn = config.hidden_act
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        This method 'forward' in the class 'MSErniePredictionHeadTransform' processes the hidden states using
        a series of transformations and returns the processed hidden states as a 'mindspore.Tensor'  object.

        Args:
            self (MSErniePredictionHeadTransform): The instance of the class MSErniePredictionHeadTransform.
            hidden_states (mindspore.Tensor): The input hidden states to be processed.
                It should be a tensor object containing the hidden states information.

        Returns:
            mindspore.Tensor:
                Returns the processed hidden states after applying dense layer, activation function,
                and layer normalization.

        Raises:
            No specific exceptions are documented to be raised by this method.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.transform_act_fn(hidden_states)
        hidden_states = self.LayerNorm(hidden_states)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePredictionHeadTransform.__init__(config)

Initializes the MSErniePredictionHeadTransform class.

PARAMETER DESCRIPTION
self

The instance of the MSErniePredictionHeadTransform class.

TYPE: MSErniePredictionHeadTransform

config

A configuration object containing settings for the transformation.

  • Type: object
  • Purpose: Specifies the configuration parameters for the transformation.
  • Restrictions: Must be a valid configuration object.
RETURNS DESCRIPTION

None.
RAISES DESCRIPTION
TypeError

If the configuration object is not of the expected type.
KeyError

If the specified hidden activation function is not found in the ACT2FN dictionary.
ValueError

If there are issues with the provided configuration parameters.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
def __init__(self, config):
    """
    Initializes the MSErniePredictionHeadTransform class.

    Args:
        self (MSErniePredictionHeadTransform): The instance of the MSErniePredictionHeadTransform class.
        config:
            A configuration object containing settings for the transformation.

            - Type: object
            - Purpose: Specifies the configuration parameters for the transformation.
            - Restrictions: Must be a valid configuration object.

    Returns:
        None.

    Raises:
        TypeError: If the configuration object is not of the expected type.
        KeyError: If the specified hidden activation function is not found in the ACT2FN dictionary.
        ValueError: If there are issues with the provided configuration parameters.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    if isinstance(config.hidden_act, str):
        self.transform_act_fn = ACT2FN[config.hidden_act]
    else:
        self.transform_act_fn = config.hidden_act
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErniePredictionHeadTransform.forward(hidden_states)

This method 'forward' in the class 'MSErniePredictionHeadTransform' processes the hidden states using a series of transformations and returns the processed hidden states as a 'mindspore.Tensor' object.

PARAMETER DESCRIPTION
self

The instance of the class MSErniePredictionHeadTransform.

TYPE: MSErniePredictionHeadTransform

hidden_states

The input hidden states to be processed. It should be a tensor object containing the hidden states information.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: Returns the processed hidden states after applying dense layer, activation function, and layer normalization.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    This method 'forward' in the class 'MSErniePredictionHeadTransform' processes the hidden states using
    a series of transformations and returns the processed hidden states as a 'mindspore.Tensor'  object.

    Args:
        self (MSErniePredictionHeadTransform): The instance of the class MSErniePredictionHeadTransform.
        hidden_states (mindspore.Tensor): The input hidden states to be processed.
            It should be a tensor object containing the hidden states information.

    Returns:
        mindspore.Tensor:
            Returns the processed hidden states after applying dense layer, activation function,
            and layer normalization.

    Raises:
        No specific exceptions are documented to be raised by this method.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.transform_act_fn(hidden_states)
    hidden_states = self.LayerNorm(hidden_states)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfAttention

Bases: Module

This class represents the self-attention mechanism for the MSErnie model. It calculates attention scores between input sequences and produces context layers based on the attention weights. The class inherits from nn.Module and is designed to be used within the MSErnie model for natural language processing tasks.

ATTRIBUTE DESCRIPTION
num_attention_heads

The number of attention heads in the self-attention mechanism.

TYPE: int

attention_head_size

The size of each attention head.

TYPE: int

all_head_size

The total size of all attention heads combined.

TYPE: int

query

A dense layer for query transformations.

TYPE: Linear

key

A dense layer for key transformations.

TYPE: Linear

value

A dense layer for value transformations.

TYPE: Linear

dropout

Dropout layer for attention probabilities.

TYPE: Dropout

position_embedding_type

The type of position embedding used in the self-attention mechanism.

TYPE: str

max_position_embeddings

The maximum number of position embeddings.

TYPE: int

distance_embedding

Embedding layer for distance-based positional encodings.

TYPE: Embedding

is_decoder

Indicates if the self-attention mechanism is used in a decoder context.

TYPE: bool

METHOD DESCRIPTION
transpose_for_scores

Transposes the input tensor to prepare it for attention score calculations.
forward

Constructs the self-attention mechanism using the provided input tensors and masks. It calculates attention scores, applies position embeddings, performs softmax, and produces context layers.

Args:

  • hidden_states (mindspore.Tensor): The input tensor to the self-attention mechanism.
  • attention_mask (Optional[mindspore.Tensor]): Optional tensor for masking attention scores.
  • head_mask (Optional[mindspore.Tensor]): Optional tensor for masking attention heads.
  • encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states from an encoder, if cross-attention is used.
  • encoder_attention_mask (Optional[mindspore.Tensor]): Attention mask for encoder hidden states.
  • past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]): Cached key and value tensors from previous iterations.
  • output_attentions (Optional[bool]): Flag to indicate whether to output attention weights.

Returns:

  • Tuple[mindspore.Tensor]: Tuple containing the context layer and optionally the attention probabilities and cached key-value pairs.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
class MSErnieSelfAttention(nn.Module):

    """
    This class represents the self-attention mechanism for the MSErnie model.
    It calculates attention scores between input sequences and produces context layers based on the attention weights.
    The class inherits from nn.Module and is designed to be used within the MSErnie model for natural language processing
    tasks.

    Attributes:
        num_attention_heads (int): The number of attention heads in the self-attention mechanism.
        attention_head_size (int): The size of each attention head.
        all_head_size (int): The total size of all attention heads combined.
        query (nn.Linear): A dense layer for query transformations.
        key (nn.Linear): A dense layer for key transformations.
        value (nn.Linear): A dense layer for value transformations.
        dropout (nn.Dropout): Dropout layer for attention probabilities.
        position_embedding_type (str): The type of position embedding used in the self-attention mechanism.
        max_position_embeddings (int): The maximum number of position embeddings.
        distance_embedding (nn.Embedding): Embedding layer for distance-based positional encodings.
        is_decoder (bool): Indicates if the self-attention mechanism is used in a decoder context.

    Methods:
        transpose_for_scores:
            Transposes the input tensor to prepare it for attention score calculations.

        forward:
            Constructs the self-attention mechanism using the provided input tensors and masks.
            It calculates attention scores, applies position embeddings, performs softmax, and produces context layers.

            Args:

            - hidden_states (mindspore.Tensor): The input tensor to the self-attention mechanism.
            - attention_mask (Optional[mindspore.Tensor]): Optional tensor for masking attention scores.
            - head_mask (Optional[mindspore.Tensor]): Optional tensor for masking attention heads.
            - encoder_hidden_states (Optional[mindspore.Tensor]):
                Hidden states from an encoder, if cross-attention is used.
            - encoder_attention_mask (Optional[mindspore.Tensor]): Attention mask for encoder hidden states.
            - past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                Cached key and value tensors from previous iterations.
            - output_attentions (Optional[bool]): Flag to indicate whether to output attention weights.

            Returns:

            - Tuple[mindspore.Tensor]:
                Tuple containing the context layer and optionally the attention probabilities and cached key-value pairs.
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes an instance of the MSErnieSelfAttention class.

        Args:
            self: The object instance.
            config:
                The configuration object containing various parameters.

                - Type: object
                - Purpose: Specifies the configuration settings for the self-attention module.
                - Restrictions: None

            position_embedding_type:
                The type of position embedding to use.

                - Type: str or None
                - Purpose: Specifies the type of position embedding to use in the self-attention module.
                - Restrictions: If None, the position_embedding_type will default to 'absolute'.
        Returns:
            None.

        Raises:
            ValueError: If the hidden size is not a multiple of the number of attention heads and
                the config object does not have an 'embedding_size' attribute.
        """
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
                f"heads ({config.num_attention_heads})"
            )

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        self.key = nn.Linear(config.hidden_size, self.all_head_size)
        self.value = nn.Linear(config.hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
        self.position_embedding_type = position_embedding_type or getattr(
            config, "position_embedding_type", "absolute"
        )
        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            self.max_position_embeddings = config.max_position_embeddings
            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

        self.is_decoder = config.is_decoder

    def transpose_for_scores(self, x: mindspore.Tensor) -> mindspore.Tensor:
        """
        This method transposes the input tensor for attention scores calculation.

        Args:
            self (MSErnieSelfAttention): The instance of the MSErnieSelfAttention class.
            x (mindspore.Tensor): The input tensor of shape (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor:
                A transposed tensor of shape (batch_size, num_attention_heads, sequence_length, attention_head_size).

        Raises:
            None.
        """
        new_x_shape = x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ) -> Tuple[mindspore.Tensor]:
        """
        This method forwards the self-attention mechanism for MSErnie model.

        Args:
            self: The instance of the class.
            hidden_states (mindspore.Tensor): The input hidden states. Shape (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]):
                An optional attention mask tensor. Shape (batch_size, num_heads, sequence_length, sequence_length).
            head_mask (Optional[mindspore.Tensor]):
                An optional head mask tensor for controlling the attention heads. Shape (num_heads,).
            encoder_hidden_states (Optional[mindspore.Tensor]):
                Optional encoder hidden states for cross-attention. Shape (batch_size, encoder_seq_length, hidden_size).
            encoder_attention_mask (Optional[mindspore.Tensor]):
                Optional attention mask for encoder_hidden_states.
                Shape (batch_size, num_heads, sequence_length, encoder_seq_length).
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                Optional tuple of past key and value tensors. Shape ((past_key_tensor, past_value_tensor)).
            output_attentions (Optional[bool]): Flag to output attentions. Default is False.

        Returns:
            Tuple[mindspore.Tensor]: Tuple containing the context layer tensor and optionally
                the attention probabilities tensor.
            The context layer tensor represents the output of the self-attention mechanism.
                Shape (batch_size, sequence_length, hidden_size).
            The attention probabilities tensor represents the attention distribution.
                Shape (batch_size, num_heads, sequence_length, encoder_seq_length).

        Raises:
            ValueError: If the dimensions of input tensors are not compatible for matrix multiplication.
            IndexError: If accessing past key and value tensors leads to index out of range.
            RuntimeError: If there is an issue with the computation or masking operations.
        """
        mixed_query_layer = self.query(hidden_states)

        # If this is instantiated as a cross-attention module, the keys
        # and values come from an encoder; the attention mask needs to be
        # such that the encoder's padding tokens are not attended to.
        is_cross_attention = encoder_hidden_states is not None

        if is_cross_attention and past_key_value is not None:
            # reuse k,v, cross_attentions
            key_layer = past_key_value[0]
            value_layer = past_key_value[1]
            attention_mask = encoder_attention_mask
        elif is_cross_attention:
            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
            attention_mask = encoder_attention_mask
        elif past_key_value is not None:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))
            key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
            value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
        else:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))

        query_layer = self.transpose_for_scores(mixed_query_layer)

        use_cache = past_key_value is not None
        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_layer, value_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            query_length, key_length = query_layer.shape[2], key_layer.shape[2]
            if use_cache:
                position_ids_l = mindspore.Tensor(key_length - 1, dtype=mindspore.int64).view(
                    -1, 1
                )
            else:
                position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
            position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
            distance = position_ids_l - position_ids_r

            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

            if self.position_embedding_type == "relative_key":
                relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores
            elif self.position_embedding_type == "relative_key_query":
                relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

        attention_scores = attention_scores / ops.sqrt(ops.scalar_to_tensor(self.attention_head_size, attention_scores.dtype))
        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in ErnieModel forward() function)
            attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = ops.softmax(attention_scores, dim=-1)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = ops.matmul(attention_probs, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3)
        new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(new_context_layer_shape)

        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

        if self.is_decoder:
            outputs = outputs + (past_key_value,)
        return outputs

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfAttention.__init__(config, position_embedding_type=None)

Initializes an instance of the MSErnieSelfAttention class.

PARAMETER DESCRIPTION
self

The object instance.
config

The configuration object containing various parameters.

  • Type: object
  • Purpose: Specifies the configuration settings for the self-attention module.
  • Restrictions: None
position_embedding_type

The type of position embedding to use.

  • Type: str or None
  • Purpose: Specifies the type of position embedding to use in the self-attention module.
  • Restrictions: If None, the position_embedding_type will default to 'absolute'.

DEFAULT: None

RAISES DESCRIPTION
ValueError

If the hidden size is not a multiple of the number of attention heads and the config object does not have an 'embedding_size' attribute.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
def __init__(self, config, position_embedding_type=None):
    """
    Initializes an instance of the MSErnieSelfAttention class.

    Args:
        self: The object instance.
        config:
            The configuration object containing various parameters.

            - Type: object
            - Purpose: Specifies the configuration settings for the self-attention module.
            - Restrictions: None

        position_embedding_type:
            The type of position embedding to use.

            - Type: str or None
            - Purpose: Specifies the type of position embedding to use in the self-attention module.
            - Restrictions: If None, the position_embedding_type will default to 'absolute'.
    Returns:
        None.

    Raises:
        ValueError: If the hidden size is not a multiple of the number of attention heads and
            the config object does not have an 'embedding_size' attribute.
    """
    super().__init__()
    if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
        raise ValueError(
            f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
            f"heads ({config.num_attention_heads})"
        )

    self.num_attention_heads = config.num_attention_heads
    self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
    self.all_head_size = self.num_attention_heads * self.attention_head_size

    self.query = nn.Linear(config.hidden_size, self.all_head_size)
    self.key = nn.Linear(config.hidden_size, self.all_head_size)
    self.value = nn.Linear(config.hidden_size, self.all_head_size)

    self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
    self.position_embedding_type = position_embedding_type or getattr(
        config, "position_embedding_type", "absolute"
    )
    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        self.max_position_embeddings = config.max_position_embeddings
        self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

    self.is_decoder = config.is_decoder

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards the self-attention mechanism for MSErnie model.

PARAMETER DESCRIPTION
self

The instance of the class.
hidden_states

The input hidden states. Shape (batch_size, sequence_length, hidden_size).

TYPE: Tensor

attention_mask

An optional attention mask tensor. Shape (batch_size, num_heads, sequence_length, sequence_length).

TYPE: Optional[Tensor] DEFAULT: None

head_mask

An optional head mask tensor for controlling the attention heads. Shape (num_heads,).

TYPE: Optional[Tensor] DEFAULT: None

encoder_hidden_states

Optional encoder hidden states for cross-attention. Shape (batch_size, encoder_seq_length, hidden_size).

TYPE: Optional[Tensor] DEFAULT: None

encoder_attention_mask

Optional attention mask for encoder_hidden_states. Shape (batch_size, num_heads, sequence_length, encoder_seq_length).

TYPE: Optional[Tensor] DEFAULT: None

past_key_value

Optional tuple of past key and value tensors. Shape ((past_key_tensor, past_value_tensor)).

TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None

output_attentions

Flag to output attentions. Default is False.

TYPE: Optional[bool] DEFAULT: False

RETURNS DESCRIPTION
Tuple[Tensor]

Tuple[mindspore.Tensor]: Tuple containing the context layer tensor and optionally the attention probabilities tensor.
Tuple[Tensor]

The context layer tensor represents the output of the self-attention mechanism. Shape (batch_size, sequence_length, hidden_size).
Tuple[Tensor]

The attention probabilities tensor represents the attention distribution. Shape (batch_size, num_heads, sequence_length, encoder_seq_length).
RAISES DESCRIPTION
ValueError

If the dimensions of input tensors are not compatible for matrix multiplication.
IndexError

If accessing past key and value tensors leads to index out of range.
RuntimeError

If there is an issue with the computation or masking operations.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
) -> Tuple[mindspore.Tensor]:
    """
    This method forwards the self-attention mechanism for MSErnie model.

    Args:
        self: The instance of the class.
        hidden_states (mindspore.Tensor): The input hidden states. Shape (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]):
            An optional attention mask tensor. Shape (batch_size, num_heads, sequence_length, sequence_length).
        head_mask (Optional[mindspore.Tensor]):
            An optional head mask tensor for controlling the attention heads. Shape (num_heads,).
        encoder_hidden_states (Optional[mindspore.Tensor]):
            Optional encoder hidden states for cross-attention. Shape (batch_size, encoder_seq_length, hidden_size).
        encoder_attention_mask (Optional[mindspore.Tensor]):
            Optional attention mask for encoder_hidden_states.
            Shape (batch_size, num_heads, sequence_length, encoder_seq_length).
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            Optional tuple of past key and value tensors. Shape ((past_key_tensor, past_value_tensor)).
        output_attentions (Optional[bool]): Flag to output attentions. Default is False.

    Returns:
        Tuple[mindspore.Tensor]: Tuple containing the context layer tensor and optionally
            the attention probabilities tensor.
        The context layer tensor represents the output of the self-attention mechanism.
            Shape (batch_size, sequence_length, hidden_size).
        The attention probabilities tensor represents the attention distribution.
            Shape (batch_size, num_heads, sequence_length, encoder_seq_length).

    Raises:
        ValueError: If the dimensions of input tensors are not compatible for matrix multiplication.
        IndexError: If accessing past key and value tensors leads to index out of range.
        RuntimeError: If there is an issue with the computation or masking operations.
    """
    mixed_query_layer = self.query(hidden_states)

    # If this is instantiated as a cross-attention module, the keys
    # and values come from an encoder; the attention mask needs to be
    # such that the encoder's padding tokens are not attended to.
    is_cross_attention = encoder_hidden_states is not None

    if is_cross_attention and past_key_value is not None:
        # reuse k,v, cross_attentions
        key_layer = past_key_value[0]
        value_layer = past_key_value[1]
        attention_mask = encoder_attention_mask
    elif is_cross_attention:
        key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
        value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
        attention_mask = encoder_attention_mask
    elif past_key_value is not None:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))
        key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
        value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
    else:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))

    query_layer = self.transpose_for_scores(mixed_query_layer)

    use_cache = past_key_value is not None
    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_layer, value_layer)

    # Take the dot product between "query" and "key" to get the raw attention scores.
    attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        query_length, key_length = query_layer.shape[2], key_layer.shape[2]
        if use_cache:
            position_ids_l = mindspore.Tensor(key_length - 1, dtype=mindspore.int64).view(
                -1, 1
            )
        else:
            position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
        position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
        distance = position_ids_l - position_ids_r

        positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
        positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

        if self.position_embedding_type == "relative_key":
            relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
            attention_scores = attention_scores + relative_position_scores
        elif self.position_embedding_type == "relative_key_query":
            relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
            relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
            attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

    attention_scores = attention_scores / ops.sqrt(ops.scalar_to_tensor(self.attention_head_size, attention_scores.dtype))
    if attention_mask is not None:
        # Apply the attention mask is (precomputed for all layers in ErnieModel forward() function)
        attention_scores = attention_scores + attention_mask

    # Normalize the attention scores to probabilities.
    attention_probs = ops.softmax(attention_scores, dim=-1)

    # This is actually dropping out entire tokens to attend to, which might
    # seem a bit unusual, but is taken from the original Transformer paper.
    attention_probs = self.dropout(attention_probs)

    # Mask heads if we want to
    if head_mask is not None:
        attention_probs = attention_probs * head_mask

    context_layer = ops.matmul(attention_probs, value_layer)

    context_layer = context_layer.permute(0, 2, 1, 3)
    new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
    context_layer = context_layer.view(new_context_layer_shape)

    outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

    if self.is_decoder:
        outputs = outputs + (past_key_value,)
    return outputs

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfAttention.transpose_for_scores(x)

This method transposes the input tensor for attention scores calculation.

PARAMETER DESCRIPTION
self

The instance of the MSErnieSelfAttention class.

TYPE: MSErnieSelfAttention

x

The input tensor of shape (batch_size, sequence_length, hidden_size).

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: A transposed tensor of shape (batch_size, num_attention_heads, sequence_length, attention_head_size).
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def transpose_for_scores(self, x: mindspore.Tensor) -> mindspore.Tensor:
    """
    This method transposes the input tensor for attention scores calculation.

    Args:
        self (MSErnieSelfAttention): The instance of the MSErnieSelfAttention class.
        x (mindspore.Tensor): The input tensor of shape (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor:
            A transposed tensor of shape (batch_size, num_attention_heads, sequence_length, attention_head_size).

    Raises:
        None.
    """
    new_x_shape = x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
    x = x.view(new_x_shape)
    return x.permute(0, 2, 1, 3)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfOutput

Bases: Module

MSErnieSelfOutput represents the self-output layer of the Ernie model in MindSpore.

This class inherits from nn.Module and contains methods for initializing and forwarding the self-output layer, which includes dense, LayerNorm, and dropout operations.

ATTRIBUTE DESCRIPTION
dense

The dense layer for linear transformation of hidden states.

TYPE: Linear

LayerNorm

The layer normalization for normalizing hidden states.

TYPE: LayerNorm

dropout

The dropout layer for adding regularization to hidden states.

TYPE: Dropout

METHOD DESCRIPTION
__init__

Initializes the MSErnieSelfOutput instance with the provided configuration.
forward

Constructs the self-output layer by performing dense, dropout, and LayerNorm operations on the hidden states.
RETURNS DESCRIPTION

mindspore.Tensor: The output tensor after passing through the self-output layer transformations.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
class MSErnieSelfOutput(nn.Module):

    """
    MSErnieSelfOutput represents the self-output layer of the Ernie model in MindSpore.

    This class inherits from nn.Module and contains methods for initializing and forwarding the self-output layer,
    which includes dense, LayerNorm, and dropout operations.

    Attributes:
        dense (nn.Linear): The dense layer for linear transformation of hidden states.
        LayerNorm (nn.LayerNorm): The layer normalization for normalizing hidden states.
        dropout (nn.Dropout): The dropout layer for adding regularization to hidden states.

    Methods:
        __init__: Initializes the MSErnieSelfOutput instance with the provided configuration.
        forward: Constructs the self-output layer by performing dense, dropout, and LayerNorm operations on
            the hidden states.

    Returns:
        mindspore.Tensor: The output tensor after passing through the self-output layer transformations.
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSErnieSelfOutput class.

        Args:
            self: The instance of the class.
            config:
                An object containing configuration parameters for the MSErnieSelfOutput class.

                - Type: object
                - Purpose: Specifies the configuration settings for the MSErnieSelfOutput instance.
                - Restrictions: Must be a valid configuration object.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
        """
        This method forwards the output of the MSErnieSelfOutput class by performing a series of operations on the
        input hidden_states and input_tensor.

        Args:
            self (MSErnieSelfOutput): The instance of the MSErnieSelfOutput class.
            hidden_states (mindspore.Tensor): The input tensor representing the hidden states.
            input_tensor (mindspore.Tensor): The input tensor used for the addition operation.

        Returns:
            mindspore.Tensor: The output tensor after the series of operations.

        Raises:
            None
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfOutput.__init__(config)

Initializes an instance of the MSErnieSelfOutput class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

An object containing configuration parameters for the MSErnieSelfOutput class.

  • Type: object
  • Purpose: Specifies the configuration settings for the MSErnieSelfOutput instance.
  • Restrictions: Must be a valid configuration object.
RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
def __init__(self, config):
    """
    Initializes an instance of the MSErnieSelfOutput class.

    Args:
        self: The instance of the class.
        config:
            An object containing configuration parameters for the MSErnieSelfOutput class.

            - Type: object
            - Purpose: Specifies the configuration settings for the MSErnieSelfOutput instance.
            - Restrictions: Must be a valid configuration object.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.LayerNorm = nn.LayerNorm([config.hidden_size], eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSErnieSelfOutput.forward(hidden_states, input_tensor)

This method forwards the output of the MSErnieSelfOutput class by performing a series of operations on the input hidden_states and input_tensor.

PARAMETER DESCRIPTION
self

The instance of the MSErnieSelfOutput class.

TYPE: MSErnieSelfOutput

hidden_states

The input tensor representing the hidden states.

TYPE: Tensor

input_tensor

The input tensor used for the addition operation.

TYPE: Tensor

RETURNS DESCRIPTION
Tensor

mindspore.Tensor: The output tensor after the series of operations.
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
def forward(self, hidden_states: mindspore.Tensor, input_tensor: mindspore.Tensor) -> mindspore.Tensor:
    """
    This method forwards the output of the MSErnieSelfOutput class by performing a series of operations on the
    input hidden_states and input_tensor.

    Args:
        self (MSErnieSelfOutput): The instance of the MSErnieSelfOutput class.
        hidden_states (mindspore.Tensor): The input tensor representing the hidden states.
        input_tensor (mindspore.Tensor): The input tensor used for the addition operation.

    Returns:
        mindspore.Tensor: The output tensor after the series of operations.

    Raises:
        None
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSUIE

Bases: MSErniePreTrainedModel

Ernie Model with two linear layer on top of the hidden-states output to compute start_prob and end_prob, designed for Universal Information Extraction.

PARAMETER DESCRIPTION
config

An instance of ErnieConfig used to forward UIE
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
class MSUIE(MSErniePreTrainedModel):
    """
    Ernie Model with two linear layer on top of the hidden-states output to compute `start_prob` and `end_prob`,
    designed for Universal Information Extraction.

    Args:
        config (:class:`ErnieConfig`):
            An instance of ErnieConfig used to forward UIE
    """
    def __init__(self, config: ErnieConfig):
        """
        Initializes an instance of the MSUIE class.

        Args:
            self: The instance of the class.
            config (ErnieConfig): The configuration object for the Ernie model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.ernie = MSErnieModel(config)
        self.linear_start = nn.Linear(config.hidden_size, 1)
        self.linear_end = nn.Linear(config.hidden_size, 1)
        self.sigmoid = nn.Sigmoid()

        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ):
        r"""
        Args:
            input_ids (Tensor):
                See :class:`ErnieModel`.
            token_type_ids (Tensor, optional):
                See :class:`ErnieModel`.
            position_ids (Tensor, optional):
                See :class:`ErnieModel`.
            attention_mask (Tensor, optional):
                See :class:`ErnieModel`.

        Example:
            ```python
            >>> import paddle
            >>> from paddlenlp.transformers import UIE, ErnieTokenizer
            >>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
            >>> model = UIE.from_pretrained('uie-base')
            >>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
            >>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
            >>> start_prob, end_prob = model(**inputs)
            ```
        """
        outputs = self.ernie(
            input_ids=input_ids,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            attention_mask=attention_mask,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        sequence_output = outputs[0]

        start_logits = self.linear_start(sequence_output)
        start_logits = ops.squeeze(start_logits, -1)
        start_prob = self.sigmoid(start_logits)
        end_logits = self.linear_end(sequence_output)
        end_logits = ops.squeeze(end_logits, -1)
        end_prob = self.sigmoid(end_logits)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            start_loss = F.binary_cross_entropy_with_logits(start_prob, start_positions)
            end_loss = F.binary_cross_entropy_with_logits(end_prob, end_positions)
            total_loss = (start_loss + end_loss) / 2.0

        output = (start_prob, end_prob) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSUIE.__init__(config)

Initializes an instance of the MSUIE class.

PARAMETER DESCRIPTION
self

The instance of the class.
config

The configuration object for the Ernie model.

TYPE: ErnieConfig

RETURNS DESCRIPTION

None
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
def __init__(self, config: ErnieConfig):
    """
    Initializes an instance of the MSUIE class.

    Args:
        self: The instance of the class.
        config (ErnieConfig): The configuration object for the Ernie model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.ernie = MSErnieModel(config)
    self.linear_start = nn.Linear(config.hidden_size, 1)
    self.linear_end = nn.Linear(config.hidden_size, 1)
    self.sigmoid = nn.Sigmoid()

    self.post_init()

mindnlp.transformers.models.ernie.modeling_graph_ernie.MSUIE.forward(input_ids=None, token_type_ids=None, position_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, output_attentions=None, output_hidden_states=None)

PARAMETER DESCRIPTION
input_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

token_type_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

position_ids

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

attention_mask

See :class:ErnieModel.

TYPE: Tensor DEFAULT: None

Example 
>>> import paddle
>>> from paddlenlp.transformers import UIE, ErnieTokenizer
>>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
>>> model = UIE.from_pretrained('uie-base')
>>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
>>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
>>> start_prob, end_prob = model(**inputs)
Source code in mindnlp\transformers\models\ernie\modeling_graph_ernie.py 
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
):
    r"""
    Args:
        input_ids (Tensor):
            See :class:`ErnieModel`.
        token_type_ids (Tensor, optional):
            See :class:`ErnieModel`.
        position_ids (Tensor, optional):
            See :class:`ErnieModel`.
        attention_mask (Tensor, optional):
            See :class:`ErnieModel`.

    Example:
        ```python
        >>> import paddle
        >>> from paddlenlp.transformers import UIE, ErnieTokenizer
        >>> tokenizer = ErnieTokenizer.from_pretrained('uie-base')
        >>> model = UIE.from_pretrained('uie-base')
        >>> inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
        >>> inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
        >>> start_prob, end_prob = model(**inputs)
        ```
    """
    outputs = self.ernie(
        input_ids=input_ids,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        attention_mask=attention_mask,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    sequence_output = outputs[0]

    start_logits = self.linear_start(sequence_output)
    start_logits = ops.squeeze(start_logits, -1)
    start_prob = self.sigmoid(start_logits)
    end_logits = self.linear_end(sequence_output)
    end_logits = ops.squeeze(end_logits, -1)
    end_prob = self.sigmoid(end_logits)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        start_loss = F.binary_cross_entropy_with_logits(start_prob, start_positions)
        end_loss = F.binary_cross_entropy_with_logits(end_prob, end_positions)
        total_loss = (start_loss + end_loss) / 2.0

    output = (start_prob, end_prob) + outputs[2:]
    return ((total_loss,) + output) if total_loss is not None else output

mindnlp.transformers.models.ernie.configuration_ernie

ERNIE model configuration

mindnlp.transformers.models.ernie.configuration_ernie.ErnieConfig

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [ErnieModel] or a [TFErnieModel]. It is used to instantiate a ERNIE model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the ERNIE nghuyong/ernie-3.0-base-zh architecture.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER DESCRIPTION
vocab_size

Vocabulary size of the ERNIE model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [ErnieModel] or [TFErnieModel].

TYPE: `int`, *optional*, defaults to 30522 DEFAULT: 30522

hidden_size

Dimensionality of the encoder layers and the pooler layer.

TYPE: `int`, *optional*, defaults to 768 DEFAULT: 768

num_hidden_layers

Number of hidden layers in the Transformer encoder.

TYPE: `int`, *optional*, defaults to 12 DEFAULT: 12

num_attention_heads

Number of attention heads for each attention layer in the Transformer encoder.

TYPE: `int`, *optional*, defaults to 12 DEFAULT: 12

intermediate_size

Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.

TYPE: `int`, *optional*, defaults to 3072 DEFAULT: 3072

hidden_act

The non-linear activation function (function or string) in the encoder and pooler. If string, "gelu", "relu", "silu" and "gelu_new" are supported.

TYPE: `str` or `Callable`, *optional*, defaults to `"gelu"` DEFAULT: 'gelu'

hidden_dropout_prob

The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.

TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1

attention_probs_dropout_prob

The dropout ratio for the attention probabilities.

TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1

max_position_embeddings

The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048).

TYPE: `int`, *optional*, defaults to 512 DEFAULT: 512

type_vocab_size

The vocabulary size of the token_type_ids passed when calling [ErnieModel] or [TFErnieModel].

TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2

task_type_vocab_size

The vocabulary size of the task_type_ids for ERNIE2.0/ERNIE3.0 model

TYPE: `int`, *optional*, defaults to 3 DEFAULT: 3

use_task_id

Whether or not the model support task_type_ids

TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

initializer_range

The standard deviation of the truncated_normal_initializer for initializing all weight matrices.

TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02

layer_norm_eps

The epsilon used by the layer normalization layers.

TYPE: `float`, *optional*, defaults to 1e-12 DEFAULT: 1e-12

position_embedding_type

Type of position embedding. Choose one of "absolute", "relative_key", "relative_key_query". For positional embeddings use "absolute". For more information on "relative_key", please refer to Self-Attention with Relative Position Representations (Shaw et al.). For more information on "relative_key_query", please refer to Method 4 in Improve Transformer Models with Better Relative Position Embeddings (Huang et al.).

TYPE: `str`, *optional*, defaults to `"absolute"` DEFAULT: 'absolute'

is_decoder

Whether the model is used as a decoder or not. If False, the model is used as an encoder.

TYPE: `bool`, *optional*, defaults to `False`

use_cache

Whether or not the model should return the last key/values attentions (not used by all models). Only relevant if config.is_decoder=True.

TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True

classifier_dropout

The dropout ratio for the classification head.

TYPE: `float`, *optional* DEFAULT: None

Example 
>>> from transformers import ErnieConfig, ErnieModel
...
>>> # Initializing a ERNIE nghuyong/ernie-3.0-base-zh style configuration
>>> configuration = ErnieConfig()
...
>>> # Initializing a model (with random weights) from the nghuyong/ernie-3.0-base-zh style configuration
>>> model = ErnieModel(configuration)
... 
>>> # Accessing the model configuration
>>> configuration = model.config
Source code in mindnlp\transformers\models\ernie\configuration_ernie.py 
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
class ErnieConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`ErnieModel`] or a [`TFErnieModel`]. It is used to
    instantiate a ERNIE model according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the ERNIE
    [nghuyong/ernie-3.0-base-zh](https://hf-mirror.com/nghuyong/ernie-3.0-base-zh) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 30522):
            Vocabulary size of the ERNIE model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`ErnieModel`] or [`TFErnieModel`].
        hidden_size (`int`, *optional*, defaults to 768):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (`int`, *optional*, defaults to 12):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 12):
            Number of attention heads for each attention layer in the Transformer encoder.
        intermediate_size (`int`, *optional*, defaults to 3072):
            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
        hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
            The dropout ratio for the attention probabilities.
        max_position_embeddings (`int`, *optional*, defaults to 512):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        type_vocab_size (`int`, *optional*, defaults to 2):
            The vocabulary size of the `token_type_ids` passed when calling [`ErnieModel`] or [`TFErnieModel`].
        task_type_vocab_size (`int`, *optional*, defaults to 3):
            The vocabulary size of the `task_type_ids` for ERNIE2.0/ERNIE3.0 model
        use_task_id (`bool`, *optional*, defaults to `False`):
            Whether or not the model support `task_type_ids`
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-12):
            The epsilon used by the layer normalization layers.
        position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
            Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
            positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
            [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
            For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
            with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
        is_decoder (`bool`, *optional*, defaults to `False`):
            Whether the model is used as a decoder or not. If `False`, the model is used as an encoder.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        classifier_dropout (`float`, *optional*):
            The dropout ratio for the classification head.

    Example:
        ```python
        >>> from transformers import ErnieConfig, ErnieModel
        ...
        >>> # Initializing a ERNIE nghuyong/ernie-3.0-base-zh style configuration
        >>> configuration = ErnieConfig()
        ...
        >>> # Initializing a model (with random weights) from the nghuyong/ernie-3.0-base-zh style configuration
        >>> model = ErnieModel(configuration)
        ... 
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "ernie"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        task_type_vocab_size=3,
        use_task_id=False,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        position_embedding_type="absolute",
        use_cache=True,
        classifier_dropout=None,
        **kwargs,
    ):
        """
        Initialize the ErnieConfig class.

        Args:
            self (object): The instance of the class.
            vocab_size (int, optional): The size of the vocabulary. Defaults to 30522.
            hidden_size (int, optional): The size of the hidden layers. Defaults to 768.
            num_hidden_layers (int, optional): The number of hidden layers. Defaults to 12.
            num_attention_heads (int, optional): The number of attention heads. Defaults to 12.
            intermediate_size (int, optional): The size of the intermediate layer in the transformer encoder. Defaults to 3072.
            hidden_act (str, optional): The activation function for the hidden layers. Defaults to 'gelu'.
            hidden_dropout_prob (float, optional): The dropout probability for the hidden layers. Defaults to 0.1.
            attention_probs_dropout_prob (float, optional): The dropout probability for the attention layers. Defaults to 0.1.
            max_position_embeddings (int, optional): The maximum position embeddings. Defaults to 512.
            type_vocab_size (int, optional): The size of the type vocabulary. Defaults to 2.
            task_type_vocab_size (int, optional): The size of the task type vocabulary. Defaults to 3.
            use_task_id (bool, optional): Whether to use task IDs. Defaults to False.
            initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
            layer_norm_eps (float, optional): The epsilon value for layer normalization. Defaults to 1e-12.
            pad_token_id (int, optional): The ID for padding tokens. Defaults to 0.
            position_embedding_type (str, optional): The type of position embedding. Defaults to 'absolute'.
            use_cache (bool, optional): Whether to use caching. Defaults to True.
            classifier_dropout (float, optional): The dropout probability for the classifier. Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(pad_token_id=pad_token_id, **kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.task_type_vocab_size = task_type_vocab_size
        self.use_task_id = use_task_id
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.position_embedding_type = position_embedding_type
        self.use_cache = use_cache
        self.classifier_dropout = classifier_dropout

mindnlp.transformers.models.ernie.configuration_ernie.ErnieConfig.__init__(vocab_size=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, hidden_act='gelu', hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=512, type_vocab_size=2, task_type_vocab_size=3, use_task_id=False, initializer_range=0.02, layer_norm_eps=1e-12, pad_token_id=0, position_embedding_type='absolute', use_cache=True, classifier_dropout=None, **kwargs)

Initialize the ErnieConfig class.

PARAMETER DESCRIPTION
self

The instance of the class.

TYPE: object

vocab_size

The size of the vocabulary. Defaults to 30522.

TYPE: int DEFAULT: 30522

hidden_size

The size of the hidden layers. Defaults to 768.

TYPE: int DEFAULT: 768

num_hidden_layers

The number of hidden layers. Defaults to 12.

TYPE: int DEFAULT: 12

num_attention_heads

The number of attention heads. Defaults to 12.

TYPE: int DEFAULT: 12

intermediate_size

The size of the intermediate layer in the transformer encoder. Defaults to 3072.

TYPE: int DEFAULT: 3072

hidden_act

The activation function for the hidden layers. Defaults to 'gelu'.

TYPE: str DEFAULT: 'gelu'

hidden_dropout_prob

The dropout probability for the hidden layers. Defaults to 0.1.

TYPE: float DEFAULT: 0.1

attention_probs_dropout_prob

The dropout probability for the attention layers. Defaults to 0.1.

TYPE: float DEFAULT: 0.1

max_position_embeddings

The maximum position embeddings. Defaults to 512.

TYPE: int DEFAULT: 512

type_vocab_size

The size of the type vocabulary. Defaults to 2.

TYPE: int DEFAULT: 2

task_type_vocab_size

The size of the task type vocabulary. Defaults to 3.

TYPE: int DEFAULT: 3

use_task_id

Whether to use task IDs. Defaults to False.

TYPE: bool DEFAULT: False

initializer_range

The range for weight initialization. Defaults to 0.02.

TYPE: float DEFAULT: 0.02

layer_norm_eps

The epsilon value for layer normalization. Defaults to 1e-12.

TYPE: float DEFAULT: 1e-12

pad_token_id

The ID for padding tokens. Defaults to 0.

TYPE: int DEFAULT: 0

position_embedding_type

The type of position embedding. Defaults to 'absolute'.

TYPE: str DEFAULT: 'absolute'

use_cache

Whether to use caching. Defaults to True.

TYPE: bool DEFAULT: True

classifier_dropout

The dropout probability for the classifier. Defaults to None.

TYPE: float DEFAULT: None

RETURNS DESCRIPTION

None.
Source code in mindnlp\transformers\models\ernie\configuration_ernie.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
def __init__(
    self,
    vocab_size=30522,
    hidden_size=768,
    num_hidden_layers=12,
    num_attention_heads=12,
    intermediate_size=3072,
    hidden_act="gelu",
    hidden_dropout_prob=0.1,
    attention_probs_dropout_prob=0.1,
    max_position_embeddings=512,
    type_vocab_size=2,
    task_type_vocab_size=3,
    use_task_id=False,
    initializer_range=0.02,
    layer_norm_eps=1e-12,
    pad_token_id=0,
    position_embedding_type="absolute",
    use_cache=True,
    classifier_dropout=None,
    **kwargs,
):
    """
    Initialize the ErnieConfig class.

    Args:
        self (object): The instance of the class.
        vocab_size (int, optional): The size of the vocabulary. Defaults to 30522.
        hidden_size (int, optional): The size of the hidden layers. Defaults to 768.
        num_hidden_layers (int, optional): The number of hidden layers. Defaults to 12.
        num_attention_heads (int, optional): The number of attention heads. Defaults to 12.
        intermediate_size (int, optional): The size of the intermediate layer in the transformer encoder. Defaults to 3072.
        hidden_act (str, optional): The activation function for the hidden layers. Defaults to 'gelu'.
        hidden_dropout_prob (float, optional): The dropout probability for the hidden layers. Defaults to 0.1.
        attention_probs_dropout_prob (float, optional): The dropout probability for the attention layers. Defaults to 0.1.
        max_position_embeddings (int, optional): The maximum position embeddings. Defaults to 512.
        type_vocab_size (int, optional): The size of the type vocabulary. Defaults to 2.
        task_type_vocab_size (int, optional): The size of the task type vocabulary. Defaults to 3.
        use_task_id (bool, optional): Whether to use task IDs. Defaults to False.
        initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
        layer_norm_eps (float, optional): The epsilon value for layer normalization. Defaults to 1e-12.
        pad_token_id (int, optional): The ID for padding tokens. Defaults to 0.
        position_embedding_type (str, optional): The type of position embedding. Defaults to 'absolute'.
        use_cache (bool, optional): Whether to use caching. Defaults to True.
        classifier_dropout (float, optional): The dropout probability for the classifier. Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(pad_token_id=pad_token_id, **kwargs)

    self.vocab_size = vocab_size
    self.hidden_size = hidden_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads
    self.hidden_act = hidden_act
    self.intermediate_size = intermediate_size
    self.hidden_dropout_prob = hidden_dropout_prob
    self.attention_probs_dropout_prob = attention_probs_dropout_prob
    self.max_position_embeddings = max_position_embeddings
    self.type_vocab_size = type_vocab_size
    self.task_type_vocab_size = task_type_vocab_size
    self.use_task_id = use_task_id
    self.initializer_range = initializer_range
    self.layer_norm_eps = layer_norm_eps
    self.position_embedding_type = position_embedding_type
    self.use_cache = use_cache
    self.classifier_dropout = classifier_dropout