Class: Transformers::XlmRoberta::XLMRobertaForMultipleChoice

Inherits:
XLMRobertaPreTrainedModel show all
Defined in:
lib/transformers/models/xlm_roberta/modeling_xlm_roberta.rb

Instance Attribute Summary

Attributes inherited from PreTrainedModel

#config

Instance Method Summary collapse

Methods inherited from XLMRobertaPreTrainedModel

#_init_weights

Methods inherited from PreTrainedModel

#_backward_compatibility_gradient_checkpointing, #_init_weights, #_initialize_weights, #base_model, #can_generate, #dequantize, #dummy_inputs, #framework, from_pretrained, #get_input_embeddings, #get_output_embeddings, #init_weights, #post_init, #prune_heads, #set_input_embeddings, #tie_weights, #warn_if_padding_and_no_attention_mask

Methods included from ClassAttribute

#class_attribute

Methods included from ModuleUtilsMixin

#device, #get_extended_attention_mask, #get_head_mask

Constructor Details

#initialize(config) ⇒ XLMRobertaForMultipleChoice

Returns a new instance of XLMRobertaForMultipleChoice.



1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
 
# File 'lib/transformers/models/xlm_roberta/modeling_xlm_roberta.rb', line 1023

def initialize(config)
  super(config)

  @roberta = XLMRobertaModel.new(config)
  @dropout = Torch::NN::Dropout.new(p: config.hidden_dropout_prob)
  @classifier = Torch::NN::Linear.new(config.hidden_size, 1)

  # Initialize weights and apply final processing
  post_init
end

Instance Method Details

#forward(input_ids: nil, token_type_ids: nil, attention_mask: nil, labels: nil, position_ids: nil, head_mask: nil, inputs_embeds: nil, output_attentions: nil, output_hidden_states: nil, return_dict: nil) ⇒ Object 



1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
 
# File 'lib/transformers/models/xlm_roberta/modeling_xlm_roberta.rb', line 1034

def forward(
  input_ids: nil,
  token_type_ids: nil,
  attention_mask: nil,
  labels: nil,
  position_ids: nil,
  head_mask: nil,
  inputs_embeds: nil,
  output_attentions: nil,
  output_hidden_states: nil,
  return_dict: nil
)
  return_dict = !return_dict.nil? ? return_dict : @config.use_return_dict
  num_choices = !input_ids.nil? ? input_ids.shape[1] : inputs_embeds.shape[1]

  flat_input_ids = !input_ids.nil? ? input_ids.view(-1, input_ids.size(-1)) : nil
  flat_position_ids = !position_ids.nil? ? position_ids.view(-1, position_ids.size(-1)) : nil
  flat_token_type_ids = !token_type_ids.nil? ? token_type_ids.view(-1, token_type_ids.size(-1)) : nil
  flat_attention_mask = !attention_mask.nil? ? attention_mask.view(-1, attention_mask.size(-1)) : nil
  flat_inputs_embeds = !inputs_embeds.nil? ? inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1)) : nil

  outputs = @roberta.(flat_input_ids, position_ids: flat_position_ids, token_type_ids: flat_token_type_ids, attention_mask: flat_attention_mask, head_mask: head_mask, inputs_embeds: flat_inputs_embeds, output_attentions: output_attentions, output_hidden_states: output_hidden_states, return_dict: return_dict)
  pooled_output = outputs[1]

  pooled_output = @dropout.(pooled_output)
  logits = @classifier.(pooled_output)
  reshaped_logits = logits.view(-1, num_choices)

  loss = nil
  if !labels.nil?
    # move labels to correct device to enable model parallelism
    labels = labels.to(reshaped_logits.device)
    loss_fct = Torch::NN::CrossEntropyLoss.new
    loss = loss_fct.(reshaped_logits, labels)
  end

  if !return_dict
    output = [reshaped_logits] + outputs[2..]
    return !loss.nil? ? [loss] + output : output
  end

  MultipleChoiceModelOutput.new(loss: loss, logits: reshaped_logits, hidden_states: outputs.hidden_states, attentions: outputs.attentions)
end