Class: Mortar::Local::Python

Inherits:
Object
  • Object
show all
Includes:
Helpers, InstallUtil, Params
Defined in:
lib/mortar/local/python.rb

Constant Summary collapse

PYTHON_OSX_TGZ_NAME = 
"mortar-python-osx.tgz"
PYTHON_OSX_TGZ_DEFAULT_URL_PATH = 
"resource/python_osx"
PYPI_URL_PATH = 
"resource/mortar_pypi"
MORTAR_PYTHON_PACKAGES = 
["luigi", "mortar-luigi", "stillson"]

Instance Method Summary collapse

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Methods included from Params

#automatic_parameters, #merge_parameters

Methods included from InstallUtil

#download_file, #ensure_mortar_local_directory, #extract_tgz, #get_resource, #gitignore_template_path, #head_resource, #http_date_to_epoch, #install_date, #install_file_for, #is_newer_version, #jython_cache_directory, #jython_directory, #local_install_directory, #local_install_directory_name, #local_log_dir, #local_project_gitignore, #local_udf_log_dir, #make_call, #make_call_sleep_seconds, #note_install, #osx?, #project_root, #render_script_template, #reset_local_logs, #run_templated_script, #unset_hadoop_env_vars, #url_date

Instance Method Details

#candidates ⇒ Object 



102
103
104
 
# File 'lib/mortar/local/python.rb', line 102

def candidates
  @candidate_pythons.dup
end

#check_or_install ⇒ Object

Execute either an installation of python or an inspection of the local system to see if a usable python is available



42
43
44
45
46
47
48
49
50
 
# File 'lib/mortar/local/python.rb', line 42

def check_or_install
  if osx?
    # We currently only install python for osx
    install_or_update_osx
  else
    # Otherwise we check that the system supplied python will be sufficient
    check_system_python
  end
end

#check_pythons_for_virtenv ⇒ Object

Inspects the list of found python installations and checks if they have virtualenv installed. The first one found will be used.



115
116
117
118
119
120
121
122
123
 
# File 'lib/mortar/local/python.rb', line 115

def check_pythons_for_virtenv
  @candidate_pythons.each{ |py|
    if has_virtualenv_installed(py)
      @command = py
      return true
    end
  }
  return false
end

#check_system_python ⇒ Object

Checks if there is a usable versionpython already installed



107
108
109
110
 
# File 'lib/mortar/local/python.rb', line 107

def check_system_python
  @candidate_pythons = lookup_local_pythons
  return 0 != @candidate_pythons.length
end

#check_virtualenv ⇒ Object 



52
53
54
55
56
57
58
59
60
61
 
# File 'lib/mortar/local/python.rb', line 52

def check_virtualenv
  # Assumes you've already called check_or_install(), in which case
  # we can skip osx as its installation includeds virtualenv
  if osx?
    return true
  else
    return check_pythons_for_virtenv
  end

end

#desired_python_minor_version ⇒ Object 



149
150
151
 
# File 'lib/mortar/local/python.rb', line 149

def desired_python_minor_version
  return "2.7"
end

#has_python_requirements ⇒ Object 



157
158
159
 
# File 'lib/mortar/local/python.rb', line 157

def has_python_requirements
  return File.exists?(pip_requirements_path)
end

#has_valid_virtualenv? ⇒ Boolean

Returns:

  • (Boolean) 


174
175
176
177
178
179
180
181
182
183
 
# File 'lib/mortar/local/python.rb', line 174

def has_valid_virtualenv?
  output = `#{@command} -m virtualenv #{python_env_dir} 2>&1`
  if 0 != $?.to_i
    File.open(virtualenv_error_log_path, 'w') { |f|
      f.write(output)
    }
    return false
  end
  return true
end

#has_virtualenv_installed(python) ⇒ Object

Checks if the specified python command has virtualenv installed



127
128
129
130
131
132
133
134
 
# File 'lib/mortar/local/python.rb', line 127

def has_virtualenv_installed(python)
  `#{python} -m virtualenv --help 2>&1`
  if (0 != $?.to_i)
    false
  else
    true
  end
end

#install_mortar_python_package(package_name) ⇒ Object 



324
325
326
327
328
329
330
 
# File 'lib/mortar/local/python.rb', line 324

def install_mortar_python_package(package_name)
  unless pip_install mortar_package_url(package_name)
      return false
  end
  ensure_mortar_local_directory mortar_package_dir(package_name)
  note_install mortar_package_dir(package_name)
end

#install_or_update_osx ⇒ Object

Performs an installation of python specific to this project, this install includes pip and virtualenv



69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/mortar/local/python.rb', line 69

def install_or_update_osx
  @command = "#{local_install_directory}/python/bin/python"
  if should_do_python_install?
    action "Installing python to #{local_install_directory_name}" do
      install_osx
    end
  elsif should_do_update?
    action "Updating to latest python in #{local_install_directory_name}" do
      install_osx
    end
  end
  true
end

#install_osx ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/mortar/local/python.rb', line 83

def install_osx
  FileUtils.mkdir_p(local_install_directory)
  python_tgz_path = File.join(local_install_directory, PYTHON_OSX_TGZ_NAME)
  download_file(python_archive_url, python_tgz_path)
  extract_tgz(python_tgz_path, local_install_directory)

  # This has been seening coming out of the tgz w/o +x so we do
  # here to be sure it has the necessary permissions
  FileUtils.chmod(0755, @command)
  File.delete(python_tgz_path)
  note_install("python")
end

#install_python_dependencies ⇒ Object 



271
272
273
274
275
276
277
278
279
280
281
 
# File 'lib/mortar/local/python.rb', line 271

def install_python_dependencies
  action "Installing python dependencies to #{local_install_directory_name}" do
    ensure_mortar_local_directory mortar_packages_dir
    MORTAR_PYTHON_PACKAGES.each{ |package_name|
      unless install_mortar_python_package(package_name)
        return false
      end
    }
  end
  return true
end

#install_user_python_dependencies ⇒ Object 



316
317
318
 
# File 'lib/mortar/local/python.rb', line 316

def install_user_python_dependencies
  return run_pip_command "install --requirement #{pip_requirements_path}"
end

#local_activate_path ⇒ Object 



283
284
285
 
# File 'lib/mortar/local/python.rb', line 283

def local_activate_path
  return "#{python_env_dir}/bin/activate"
end

#local_pip_bin ⇒ Object 



291
292
293
 
# File 'lib/mortar/local/python.rb', line 291

def local_pip_bin
  return "#{python_env_dir}/bin/pip"
end

#local_python_bin ⇒ Object 



287
288
289
 
# File 'lib/mortar/local/python.rb', line 287

def local_python_bin
  return "#{python_env_dir}/bin/python"
end

#lookup_local_pythons ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/mortar/local/python.rb', line 136

def lookup_local_pythons
  # Check several python commands in decending level of desirability
  found_bins = []
  [ "python#{desired_python_minor_version}", "python" ].each{ |cmd|
    path_to_python = `which #{cmd}`.to_s.strip
    if path_to_python != ''
      found_bins << path_to_python
    end
  }
  return found_bins
end

#luigi_command_template_parameters(luigi_script, user_script_args) ⇒ Object 



363
364
365
366
367
368
369
370
371
372
373
374
375
 
# File 'lib/mortar/local/python.rb', line 363

def luigi_command_template_parameters(luigi_script, user_script_args)
  script_args = [
    "--local-scheduler",
    "--logging-conf-file #{luigi_logging_config_file_path}",
    user_script_args.join(" "),
  ]
  return {
    :python_arugments => "",
    :python_script => luigi_script.executable_path(),
    :script_arguments => script_args.join(" "),
    :python_path => File.join(project_root, 'luigiscripts')
  }
end

#luigi_logging_config_file_path ⇒ Object 



359
360
361
 
# File 'lib/mortar/local/python.rb', line 359

def luigi_logging_config_file_path
  File.expand_path("../../conf/luigi/logging.ini", __FILE__)
end

#mortar_package_dir(package) ⇒ Object 



258
259
260
 
# File 'lib/mortar/local/python.rb', line 258

def mortar_package_dir(package)
  package_dir = "#{mortar_packages_dir}/#{package}"
end

#mortar_package_url(package) ⇒ Object 



245
246
247
248
 
# File 'lib/mortar/local/python.rb', line 245

def mortar_package_url(package)
  default_url = full_host + "/" + PYPI_URL_PATH
  "#{ENV.fetch('MORTAR_PACKAGE_URL', default_url)}/#{package}"
end

#mortar_packages_dir ⇒ Object 



254
255
256
 
# File 'lib/mortar/local/python.rb', line 254

def mortar_packages_dir
  return "pythonenv/mortar-packages"
end

#pip_error_log_path ⇒ Object 



213
214
215
 
# File 'lib/mortar/local/python.rb', line 213

def pip_error_log_path
  return ENV.fetch('PIP_ERROR_LOG', "#{local_install_directory}/pip_dependency_install.log")
end

#pip_install(package_url) ⇒ Object 



320
321
322
 
# File 'lib/mortar/local/python.rb', line 320

def pip_install package_url
  return run_pip_command "install  #{package_url};"
end

#pip_requirements_path ⇒ Object 



153
154
155
 
# File 'lib/mortar/local/python.rb', line 153

def pip_requirements_path
  return ENV.fetch('PIP_REQ_FILE', File.join(Dir.getwd, "requirements.txt"))
end

#python_archive_url ⇒ Object 



169
170
171
172
 
# File 'lib/mortar/local/python.rb', line 169

def python_archive_url
  default_url = full_host + "/" + PYTHON_OSX_TGZ_DEFAULT_URL_PATH
  return ENV.fetch('PYTHON_DISTRO_URL', default_url)
end

#python_command_script_template_path ⇒ Object

Path to the template which generates the bash script for running python



350
351
352
 
# File 'lib/mortar/local/python.rb', line 350

def python_command_script_template_path
  File.expand_path("../../templates/script/runpython.sh", __FILE__)
end

#python_directory ⇒ Object 



165
166
167
 
# File 'lib/mortar/local/python.rb', line 165

def python_directory
  return "#{local_install_directory}/python"
end

#python_env_dir ⇒ Object 



161
162
163
 
# File 'lib/mortar/local/python.rb', line 161

def python_env_dir
  return "#{local_install_directory}/pythonenv"
end

#requirements_edit_date ⇒ Object

Date of last change to the requirements file



237
238
239
240
241
242
243
 
# File 'lib/mortar/local/python.rb', line 237

def requirements_edit_date
  if has_python_requirements
    return File.mtime(pip_requirements_path).to_i
  else
    return nil
  end
end

#run_luigi_script(luigi_script, user_script_args) ⇒ Object 



332
333
334
335
 
# File 'lib/mortar/local/python.rb', line 332

def run_luigi_script(luigi_script, user_script_args)
  template_params = luigi_command_template_parameters(luigi_script, user_script_args)
  run_templated_script(python_command_script_template_path, template_params)
end

#run_pip_command(subcmd) ⇒ Object 



295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
 
# File 'lib/mortar/local/python.rb', line 295

def run_pip_command subcmd
  # Note that we're executing pip by passing it as a script for python to execute, this is
  # explicitly done to deal with this command breaking due to the maximum size of the path
  # to the interpreter in a shebang.  Since the containing virtualenv is already buried
  # several layers deep in the .mortar-local directory we're very likely to (read: have) hit
  # this limit.  This unfortunately leads to very vague errors about pip not existing when
  # in fact it is the truncated path to the interpreter that does not exist.  I would now
  # like the last day of my life back.
  pip_output = `. #{local_activate_path} && #{local_python_bin} #{local_pip_bin} --log #{pip_error_log_path} #{subcmd}`
  if 0 != $?.to_i
    return false
  else
    return true
  end

end

#run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters) ⇒ Object 



337
338
339
340
341
342
343
344
345
346
347
 
# File 'lib/mortar/local/python.rb', line 337

def run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
  # combine automatic mortar parameters with
  # parameters provided in the project config
  auto_params = automatic_parameters()
  parameters = merge_parameters(auto_params, project_config_parameters)
  stillson_template_params = {
    :parameters => parameters,
    :luigiscripts_path => File.join(project_root, 'luigiscripts')
  }
  run_templated_script(stillson_command_script_template_path, stillson_template_params)
end

#setup_project_python_environment ⇒ Object

Creates a virtualenv in a well known location and installs any packages necessary for the users python udf



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/mortar/local/python.rb', line 187

def setup_project_python_environment
  if not has_valid_virtualenv?
    return false
  end
  if should_do_requirements_install
    action "Installing user defined python dependencies" do
      unless upgrade_pip()
        return false
      end
      unless install_user_python_dependencies()
        return false
      end
      note_install("pythonenv")
    end
  end
  if should_install_python_dependencies?
    unless upgrade_pip()
      return false
    end
    unless install_python_dependencies()
      return false
    end
  end
  return true
end

#should_do_python_install? ⇒ Boolean

Determines if a python install needs to occur, true if no python install present or a newer version is available

Returns:

  • (Boolean) 


98
99
100
 
# File 'lib/mortar/local/python.rb', line 98

def should_do_python_install?
  return (osx? and (not (File.exists?(python_directory))))
end

#should_do_requirements_install ⇒ Object

Whether or not we need to do a ‘pip install -r requirements.txt` because we’ve never done one before or the dependencies have changed



223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/mortar/local/python.rb', line 223

def should_do_requirements_install
  if has_python_requirements
    if not install_date('pythonenv')
      # We've never done an install from requirements.txt before
      return true
    else
      return (requirements_edit_date > install_date('pythonenv'))
    end
  else
    return false
  end
end

#should_do_update? ⇒ Boolean

Returns:

  • (Boolean) 


63
64
65
 
# File 'lib/mortar/local/python.rb', line 63

def should_do_update?
  return is_newer_version('python', python_archive_url)
end

#should_install_python_dependencies? ⇒ Boolean

Returns:

  • (Boolean) 


262
263
264
265
266
267
268
269
 
# File 'lib/mortar/local/python.rb', line 262

def should_install_python_dependencies?
  MORTAR_PYTHON_PACKAGES.each{ |package|
    if update_mortar_package? package
      return true
    end
  }
  return false
end

#stillson_command_script_template_path ⇒ Object

Path to the template which generates the bash script for running stillson



355
356
357
 
# File 'lib/mortar/local/python.rb', line 355

def stillson_command_script_template_path
  File.expand_path("../../templates/script/runstillson.sh", __FILE__)
end

#update_mortar_package?(package) ⇒ Boolean

Returns:

  • (Boolean) 


250
251
252
 
# File 'lib/mortar/local/python.rb', line 250

def update_mortar_package?(package)
  return is_newer_version(mortar_package_dir(package), mortar_package_url(package))
end

#upgrade_pip ⇒ Object 



312
313
314
 
# File 'lib/mortar/local/python.rb', line 312

def upgrade_pip
  return (run_pip_command "install --upgrade setuptools" and run_pip_command "install --upgrade pip")
end

#virtualenv_error_log_path ⇒ Object 



217
218
219
 
# File 'lib/mortar/local/python.rb', line 217

def virtualenv_error_log_path
  return ENV.fetch('VE_ERROR_LOG', "virtualenv.log")
end