Class: OnnxRuntime::InferenceSession
- Inherits:
-
Object
- Object
- OnnxRuntime::InferenceSession
- Defined in:
- lib/onnxruntime/inference_session.rb
Instance Attribute Summary collapse
-
#inputs ⇒ Object
readonly
Returns the value of attribute inputs.
-
#outputs ⇒ Object
readonly
Returns the value of attribute outputs.
Instance Method Summary collapse
-
#end_profiling ⇒ Object
return value has double underscore like Python.
-
#initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, free_dimension_overrides_by_denotation: nil, free_dimension_overrides_by_name: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil, profile_file_prefix: nil, session_config_entries: nil, providers: []) ⇒ InferenceSession
constructor
A new instance of InferenceSession.
- #modelmeta ⇒ Object
-
#providers ⇒ Object
no way to set providers with C API yet so we can return all available providers.
-
#run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby) ⇒ Object
TODO support logid.
Constructor Details
#initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, free_dimension_overrides_by_denotation: nil, free_dimension_overrides_by_name: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil, profile_file_prefix: nil, session_config_entries: nil, providers: []) ⇒ InferenceSession
Returns a new instance of InferenceSession.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
# File 'lib/onnxruntime/inference_session.rb', line 5 def initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, free_dimension_overrides_by_denotation: nil, free_dimension_overrides_by_name: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil, profile_file_prefix: nil, session_config_entries: nil, providers: []) # session options = ::FFI::MemoryPointer.new(:pointer) check_status api[:CreateSessionOptions].call() if enable_cpu_mem_arena check_status api[:EnableCpuMemArena].call(.read_pointer) else check_status api[:DisableCpuMemArena].call(.read_pointer) end if enable_mem_pattern check_status api[:EnableMemPattern].call(.read_pointer) else check_status api[:DisableMemPattern].call(.read_pointer) end if enable_profiling check_status api[:EnableProfiling].call(.read_pointer, ort_string(profile_file_prefix || "onnxruntime_profile_")) else check_status api[:DisableProfiling].call(.read_pointer) end if execution_mode execution_modes = {sequential: 0, parallel: 1} mode = execution_modes[execution_mode] raise ArgumentError, "Invalid execution mode" unless mode check_status api[:SetSessionExecutionMode].call(.read_pointer, mode) end if free_dimension_overrides_by_denotation free_dimension_overrides_by_denotation.each do |k, v| check_status api[:AddFreeDimensionOverride].call(.read_pointer, k.to_s, v) end end if free_dimension_overrides_by_name free_dimension_overrides_by_name.each do |k, v| check_status api[:AddFreeDimensionOverrideByName].call(.read_pointer, k.to_s, v) end end if graph_optimization_level optimization_levels = {none: 0, basic: 1, extended: 2, all: 99} level = optimization_levels[graph_optimization_level] raise ArgumentError, "Invalid graph optimization level" unless level check_status api[:SetSessionGraphOptimizationLevel].call(.read_pointer, level) end check_status api[:SetInterOpNumThreads].call(.read_pointer, inter_op_num_threads) if inter_op_num_threads check_status api[:SetIntraOpNumThreads].call(.read_pointer, intra_op_num_threads) if intra_op_num_threads check_status api[:SetSessionLogSeverityLevel].call(.read_pointer, log_severity_level) if log_severity_level check_status api[:SetSessionLogVerbosityLevel].call(.read_pointer, log_verbosity_level) if log_verbosity_level check_status api[:SetSessionLogId].call(.read_pointer, logid) if logid check_status api[:SetOptimizedModelFilePath].call(.read_pointer, ort_string(optimized_model_filepath)) if optimized_model_filepath if session_config_entries session_config_entries.each do |k, v| check_status api[:AddSessionConfigEntry].call(.read_pointer, k.to_s, v.to_s) end end providers.each do |provider| unless self.providers.include?(provider) warn "Provider not available: #{provider}" next end case provider when "CUDAExecutionProvider" = ::FFI::MemoryPointer.new(:pointer) check_status api[:CreateCUDAProviderOptions].call() check_status api[:SessionOptionsAppendExecutionProvider_CUDA_V2].call(.read_pointer, .read_pointer) release :CUDAProviderOptions, when "CoreMLExecutionProvider" unless FFI.respond_to?(:OrtSessionOptionsAppendExecutionProvider_CoreML) raise ArgumentError, "Provider not available: #{provider}" end coreml_flags = 0 check_status FFI.OrtSessionOptionsAppendExecutionProvider_CoreML(.read_pointer, coreml_flags) when "CPUExecutionProvider" break else raise ArgumentError, "Provider not supported: #{provider}" end end @session = load_session(path_or_bytes, ) ObjectSpace.define_finalizer(@session, self.class.finalize(read_pointer.to_i)) @allocator = load_allocator @inputs = load_inputs @outputs = load_outputs ensure release :SessionOptions, end |
Instance Attribute Details
#inputs ⇒ Object (readonly)
Returns the value of attribute inputs.
3 4 5 |
# File 'lib/onnxruntime/inference_session.rb', line 3 def inputs @inputs end |
#outputs ⇒ Object (readonly)
Returns the value of attribute outputs.
3 4 5 |
# File 'lib/onnxruntime/inference_session.rb', line 3 def outputs @outputs end |
Instance Method Details
#end_profiling ⇒ Object
return value has double underscore like Python
182 183 184 185 186 |
# File 'lib/onnxruntime/inference_session.rb', line 182 def end_profiling out = ::FFI::MemoryPointer.new(:string) check_status api[:SessionEndProfiling].call(read_pointer, @allocator.read_pointer, out) out.read_pointer.read_string end |
#modelmeta ⇒ Object
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
# File 'lib/onnxruntime/inference_session.rb', line 129 def keys = ::FFI::MemoryPointer.new(:pointer) num_keys = ::FFI::MemoryPointer.new(:int64_t) description = ::FFI::MemoryPointer.new(:string) domain = ::FFI::MemoryPointer.new(:string) graph_name = ::FFI::MemoryPointer.new(:string) graph_description = ::FFI::MemoryPointer.new(:string) producer_name = ::FFI::MemoryPointer.new(:string) version = ::FFI::MemoryPointer.new(:int64_t) = ::FFI::MemoryPointer.new(:pointer) check_status api[:SessionGetModelMetadata].call(read_pointer, ) = {} check_status api[:ModelMetadataGetCustomMetadataMapKeys].call(.read_pointer, @allocator.read_pointer, keys, num_keys) num_keys.read(:int64_t).times do |i| key_ptr = keys.read_pointer[i * ::FFI::Pointer.size] key = key_ptr.read_pointer.read_string value = ::FFI::MemoryPointer.new(:string) check_status api[:ModelMetadataLookupCustomMetadataMap].call(.read_pointer, @allocator.read_pointer, key, value) [key] = value.read_pointer.read_string allocator_free key_ptr allocator_free value end allocator_free keys check_status api[:ModelMetadataGetDescription].call(.read_pointer, @allocator.read_pointer, description) check_status api[:ModelMetadataGetDomain].call(.read_pointer, @allocator.read_pointer, domain) check_status api[:ModelMetadataGetGraphName].call(.read_pointer, @allocator.read_pointer, graph_name) check_status api[:ModelMetadataGetGraphDescription].call(.read_pointer, @allocator.read_pointer, graph_description) check_status api[:ModelMetadataGetProducerName].call(.read_pointer, @allocator.read_pointer, producer_name) check_status api[:ModelMetadataGetVersion].call(.read_pointer, version) { custom_metadata_map: , description: description.read_pointer.read_string, domain: domain.read_pointer.read_string, graph_name: graph_name.read_pointer.read_string, graph_description: graph_description.read_pointer.read_string, producer_name: producer_name.read_pointer.read_string, version: version.read(:int64_t) } ensure release :ModelMetadata, allocator_free description allocator_free domain allocator_free graph_name allocator_free graph_description allocator_free producer_name end |
#providers ⇒ Object
no way to set providers with C API yet so we can return all available providers
190 191 192 193 194 195 196 197 198 199 200 201 |
# File 'lib/onnxruntime/inference_session.rb', line 190 def providers out_ptr = ::FFI::MemoryPointer.new(:pointer) length_ptr = ::FFI::MemoryPointer.new(:int) check_status api[:GetAvailableProviders].call(out_ptr, length_ptr) length = length_ptr.read_int providers = [] length.times do |i| providers << out_ptr.read_pointer[i * ::FFI::Pointer.size].read_pointer.read_string end api[:ReleaseAvailableProviders].call(out_ptr.read_pointer, length) providers end |
#run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby) ⇒ Object
TODO support logid
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
# File 'lib/onnxruntime/inference_session.rb', line 94 def run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby) # pointer references refs = [] input_tensor = create_input_tensor(input_feed, refs) output_names ||= @outputs.map { |v| v[:name] } output_tensor = ::FFI::MemoryPointer.new(:pointer, outputs.size) input_node_names = create_node_names(input_feed.keys.map(&:to_s), refs) output_node_names = create_node_names(output_names.map(&:to_s), refs) # run options = ::FFI::MemoryPointer.new(:pointer) check_status api[:CreateRunOptions].call() check_status api[:RunOptionsSetRunLogSeverityLevel].call(.read_pointer, log_severity_level) if log_severity_level check_status api[:RunOptionsSetRunLogVerbosityLevel].call(.read_pointer, log_verbosity_level) if log_verbosity_level check_status api[:RunOptionsSetRunTag].call(.read_pointer, logid) if logid check_status api[:RunOptionsSetTerminate].call(.read_pointer) if terminate check_status api[:Run].call(read_pointer, .read_pointer, input_node_names, input_tensor, input_feed.size, output_node_names, output_names.size, output_tensor) output_names.size.times.map do |i| create_from_onnx_value(output_tensor[i].read_pointer, output_type) end ensure release :RunOptions, if input_tensor input_feed.size.times do |i| release :Value, input_tensor[i] end end # output values released in create_from_onnx_value end |