Class: NluAdapter::Metrics

Inherits:
Object
  • Object
show all
Defined in:
lib/nlu_adapter/metrics.rb

Overview

A utility class to calculate classification matrics

See Also:

Instance Method Summary collapse

Constructor Details

#initialize(actual, predicted, class_labels = []) ⇒ Metrics

Constructor

Parameters:

  • actual (Array) —

    an array of actual results (true values)

  • predicted (Array) —

    an array of predicted results (predicted values)

  • class_labels (Array) (defaults to: []) —

    an array of class_labels for the results, this is required if actual & predicted values are numeric



16
17
18
19
20
21
22
23
24
 
# File 'lib/nlu_adapter/metrics.rb', line 16

def initialize(actual, predicted, class_labels = [])
	@actual = actual
	@predicted = predicted
	@class_labels = class_labels

	@actual.map!{ |a| a.is_a?(String)? a.intern : a }
	@predicted.map!{ |p| p.is_a?(String)? p.intern : p }
	@class_labels.map!{ |l| l.is_a?(String)? l.intern : l }
end

Instance Method Details

#accuracy ⇒ Float

Caclulate the accuracy

Returns:

  • (Float) —

    accuracy

See Also:



30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/nlu_adapter/metrics.rb', line 30

def accuracy
	if @actual.size != @predicted.size
		#todo: throw error
		puts "actual & predicted array are not of same size"
		return
	end

	total = @actual.size
	correct = 0
	@actual.each_with_index do |v, i|
		correct+=1 if @predicted[i] == v
	end
	(correct.fdiv(total) * 100).round(4)
end

#class_totals ⇒ Hash

Get totals of actual values per class

Returns:

  • (Hash) —

    Hash of class totals



98
99
100
 
# File 'lib/nlu_adapter/metrics.rb', line 98

def class_totals
	return @class_totals
end

#classification_report ⇒ Hash

Generate classification report

Returns:

  • (Hash) —

    precision, recall and totals for each class name as key



161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/nlu_adapter/metrics.rb', line 161

def classification_report
	confusion_matrix if !@m
	report = {}
	@class_labels.each do |label|
		report[label] = {
			precision: precision(label),
			recall: recall(label),
			class_total: class_totals[label]
		}
	end

	return report
end

#confusion_matrix ⇒ Matrix

Get the confusion matrix

Returns:

  • (Matrix) —

    confusion matrix

See Also:



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/nlu_adapter/metrics.rb', line 49

def confusion_matrix
	class_labels = @class_labels
	actual = @actual
	predicted = @predicted

	#if no class_labels, convert to numeric values, and extract the class_labels
	if @class_labels.empty?
		class_labels = (@actual + @predicted).sort.uniq.sort
		class_labels.map!{|c| c.intern}
		@actual.each_with_index do |v, i|
			actual[i] = class_labels.index(v)
		end

		@predicted.each_with_index do |v, i|
			predicted[i] = class_labels.index(v)
		end
	else
		#check if any string passed in actual/predicted
		i = actual.select { |a| a.is_a?(Symbol) }.size
		j = predicted.select { |p| p.is_a?(Symbol) }.size

		if i > 0 || j > 0
			#todo: fix it OR throw error
			puts "actual, predicted & class_labels array having string values not implemented yet"
			return

		end
	end

	m = Matrix.zero(class_labels.size)

	@class_totals = Hash[class_labels.collect { |c| [c, 0] }]
	actual.each_with_index do |vi, i|
		vj = predicted[i]
		m[vi, vj] = m[vi, vj] + 1
		@class_totals[class_labels[vi]] += 1
	end

	@class_labels = class_labels
	@actual = actual
	@predicted = predicted

	@m = m
	return m
end

#fn(class_name) ⇒ Integer

Get false negative

Parameters:

  • class_name (String) —

    class name

Returns:

  • (Integer) —

    false negative value for class_name



128
129
130
131
132
133
 
# File 'lib/nlu_adapter/metrics.rb', line 128

def fn(class_name)
	i = @class_labels.index(class_name.intern)
	return nil if i == nil || @m == nil || @m.empty?
	fn = @m.row(i).sum - tp(class_name)
	return fn
end

#fp(class_name) ⇒ Integer

Get false positive

Parameters:

  • class_name (String) —

    class name

Returns:

  • (Integer) —

    false positive value for class_name



117
118
119
120
121
122
 
# File 'lib/nlu_adapter/metrics.rb', line 117

def fp(class_name)
	i = @class_labels.index(class_name.intern)
	return nil if i == nil || @m == nil || @m.empty?
	fp = @m.column(i).sum - tp(class_name)
	return fp
end

#precision(class_name) ⇒ Float

Get the precision for given class

Parameters:

  • class_name (String) —

    class name

Returns:

  • (Float) —

    precision rounded off to 4 decimal places



139
140
141
142
143
144
 
# File 'lib/nlu_adapter/metrics.rb', line 139

def precision(class_name)
	confusion_matrix if !@m
	return 0.00 if tp(class_name) == 0
	precision = tp(class_name).fdiv((tp(class_name) + fp(class_name))).round(4)
	return precision
end

#recall(class_name) ⇒ Float

Get the recall for given class

Parameters:

  • class_name (String) —

    class name

Returns:

  • (Float) —

    recall rounded off to 4 decimal places



150
151
152
153
154
155
156
 
# File 'lib/nlu_adapter/metrics.rb', line 150

def recall(class_name)
	confusion_matrix if !@m
	return 0.00 if tp(class_name) == 0
	recall = tp(class_name).fdiv((tp(class_name) + fn(class_name))).round(4)

	return recall
end

#tp(class_name) ⇒ Integer

Get total positives

Parameters:

  • class_name (String) —

    class name

Returns:

  • (Integer) —

    total positive value for class_name



106
107
108
109
110
111
 
# File 'lib/nlu_adapter/metrics.rb', line 106

def tp(class_name)
	i = @class_labels.index(class_name.intern)
	return nil if i == nil || @m == nil || @m.empty?
	tp = @m[i, i]
	return tp
end