Class: Transformers::GELUActivation

Inherits:
Torch::NN::Module
  • Object
show all
Defined in:
lib/transformers/activations.rb

Instance Method Summary collapse

Constructor Details

#initialize(use_gelu_python: false) ⇒ GELUActivation

Returns a new instance of GELUActivation.



17
18
19
20
21
22
23
24
 
# File 'lib/transformers/activations.rb', line 17

def initialize(use_gelu_python: false)
  super()
  if use_gelu_python
    @act = _gelu_python
  else
    @act = Torch::NN::Functional.method(:gelu)
  end
end

Instance Method Details

#_gelu_python(input) ⇒ Object 



26
27
28
 
# File 'lib/transformers/activations.rb', line 26

def _gelu_python(input)
  input * 0.5 * (1.0 + Torch.erf(input / Math.sqrt(2.0)))
end

#forward(input) ⇒ Object 



30
31
32
 
# File 'lib/transformers/activations.rb', line 30

def forward(input)
  @act.(input)
end