Class: Torch::NN::TransformerDecoderLayer

Inherits:
Module
  • Object
show all
Defined in:
lib/torch/nn/transformer_decoder_layer.rb

Instance Attribute Summary

Attributes inherited from Module

#training

Instance Method Summary collapse

Methods inherited from Module

#_apply, #add_module, #apply, #buffers, #call, #children, #cpu, #cuda, #deep_dup, #double, #eval, #float, #half, #inspect, #load_state_dict, #method_missing, #modules, #named_buffers, #named_children, #named_modules, #named_parameters, #parameters, #register_buffer, #register_parameter, #requires_grad!, #respond_to?, #share_memory, #state_dict, #to, #train, #type, #zero_grad

Methods included from Utils

#_activation_fn, #_clones, #_ntuple, #_pair, #_quadrupal, #_single, #_triple

Constructor Details

#initialize(d_model, n_head, dim_feedforward: 2048, dropout: 0.1, activation: :relu, layer_norm_eps: 1e-5, batch_first: false) ⇒ TransformerDecoderLayer

Returns a new instance of TransformerDecoderLayer.



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/torch/nn/transformer_decoder_layer.rb', line 4

def initialize(
  d_model, n_head,
  dim_feedforward: 2048, dropout: 0.1, activation: :relu,
  layer_norm_eps: 1e-5, batch_first: false
)

  super()

  @self_attn = MultiheadAttention.new(d_model, n_head, dropout: dropout, batch_first: batch_first)
  @multihead_attn = MultiheadAttention.new(d_model, n_head, dropout: dropout, batch_first: batch_first)

  @linear1 = Linear.new(d_model, dim_feedforward)
  @dropout = Dropout.new(p: dropout)
  @linear2 = Linear.new(dim_feedforward, d_model)

  @norm1 = LayerNorm.new(d_model, eps: layer_norm_eps)
  @norm2 = LayerNorm.new(d_model, eps: layer_norm_eps)
  @norm3 = LayerNorm.new(d_model, eps: layer_norm_eps)

  @dropout1 = Dropout.new(p: dropout)
  @dropout2 = Dropout.new(p: dropout)
  @dropout3 = Dropout.new(p: dropout)

  @activation = _activation_fn(activation)
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method in the class Torch::NN::Module

Instance Method Details

#forward(tgt, memory, tgt_mask: nil, memory_mask: nil, tgt_key_padding_mask: nil, memory_key_padding_mask: nil) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/torch/nn/transformer_decoder_layer.rb', line 30

def forward(tgt, memory, tgt_mask: nil, memory_mask: nil, tgt_key_padding_mask: nil, memory_key_padding_mask: nil)
  tgt2 = @self_attn.(tgt, tgt, tgt, attn_mask: tgt_mask, key_padding_mask: tgt_key_padding_mask).first
  tgt += @dropout1.(tgt2)
  tgt = @norm1.(tgt)
  tgt2 = @multihead_attn.(tgt, memory, memory, attn_mask: memory_mask, key_padding_mask: memory_key_padding_mask).first
  tgt += @dropout2.(tgt2)
  tgt = @norm2.(tgt)
  tgt2 = @linear2.(@dropout.(@activation.(@linear1.(tgt))))
  tgt += @dropout3.(tgt2)
  @norm3.(tgt)
end