Class: Metanorma::Compile

Inherits:
Object
  • Object
show all
Defined in:
lib/metanorma/compile/compile.rb,
lib/metanorma/compile/extract.rb,
lib/metanorma/compile/compile_options.rb,
lib/metanorma/compile/compile_validate.rb

Constant Summary collapse

REQUIREMENT_XPATH = 
"//requirement | //xmlns:requirement | //recommendation | "\
"//xmlns:recommendation | //permission | //xmlns:permission".freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Compile

Returns a new instance of Compile.



20
21
22
23
24
25
26
 
# File 'lib/metanorma/compile/compile.rb', line 20

def initialize
  @registry = Metanorma::Registry.instance
  @errors = []
  @isodoc = IsoDoc::Convert.new({})
  @fontist_installed = false
  @log = Metanorma::Utils::Log.new
end

Instance Attribute Details

#errors ⇒ Array<String> (readonly)

Returns:



18
19
20
 
# File 'lib/metanorma/compile/compile.rb', line 18

def errors
  @errors
end

#processor ⇒ Array<String> (readonly)

Returns:



18
19
20
 
# File 'lib/metanorma/compile/compile.rb', line 18

def processor
  @processor
end

Instance Method Details

#clean_exit(options) ⇒ Object 



48
49
50
51
 
# File 'lib/metanorma/compile/compile.rb', line 48

def clean_exit(options)
  options[:novalid] and return
  @log.write
end

#clean_sourcecode(xml) ⇒ Object 



12
13
14
15
16
17
 
# File 'lib/metanorma/compile/extract.rb', line 12

def clean_sourcecode(xml)
  xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
            ".//xmlns:annotation").each(&:remove)
  xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
  HTMLEntities.new.decode(xml.children.to_xml)
end

#compile(filename, options = {}) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/metanorma/compile/compile.rb', line 28

def compile(filename, options = {})
  options_process(filename, options)
  @processor = @registry.find_processor(options[:type].to_sym)
  (file, isodoc = process_input(filename, options)) or return nil
  extensions = get_extensions(options) or return nil
  relaton_export(isodoc, options)
  extract(isodoc, options[:extract], options[:extract_type])
  process_exts(filename, extensions, file, isodoc, options)
ensure
  clean_exit(options)
end

#export_output(fname, content, **options) ⇒ Object 



97
98
99
100
 
# File 'lib/metanorma/compile/compile.rb', line 97

def export_output(fname, content, **options)
  mode = options[:binary] ? "wb" : "w:UTF-8"
  File.open(fname, mode) { |f| f.write content }
end

#extract(isodoc, dirname, extract_types) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/metanorma/compile/extract.rb', line 19

def extract(isodoc, dirname, extract_types)
  dirname or return
  extract_types.nil? || extract_types.empty? and
    extract_types = %i[sourcecode image requirement]
  FileUtils.rm_rf dirname
  FileUtils.mkdir_p dirname
  xml = Nokogiri::XML(isodoc, &:huge)
  sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
  image_export(xml, dirname) if extract_types.include? :image
  extract_types.include?(:requirement) and
    requirement_export(xml, dirname)
end

#extract_extensions(options) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/metanorma/compile/compile_options.rb', line 47

def extract_extensions(options)
  options[:extension_keys] ||=
    @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
  options[:extension_keys].reduce([]) do |memo, e|
    if @processor.output_formats[e] then memo << e
    else
      unsupported_format_error(e)
      memo
    end
  end
end

#font_install(opt) ⇒ Object 



59
60
61
62
63
 
# File 'lib/metanorma/compile/compile_options.rb', line 59

def font_install(opt)
  @fontist_installed or
    Util::FontistHelper.install_fonts(@processor, opt)
  @fontist_installed = true
end

#gather_and_install_fonts(file, options, extensions) ⇒ Object 



138
139
140
141
142
143
 
# File 'lib/metanorma/compile/compile.rb', line 138

def gather_and_install_fonts(file, options, extensions)
  Util.sort_extensions_execution(extensions).each do |ext|
    isodoc_options = get_isodoc_options(file, options, ext)
    font_install(isodoc_options.merge(options))
  end
end

#get_extensions(options) ⇒ Object 



37
38
39
40
41
42
43
44
45
 
# File 'lib/metanorma/compile/compile_options.rb', line 37

def get_extensions(options)
  ext = extract_extensions(options)
  !ext.include?(:presentation) && ext.any? do |e|
    @processor.use_presentation_xml(e)
  end and ext << :presentation
  !ext.include?(:rxl) && options[:site_generate] and
    ext << :rxl
  ext
end

#image_export(xml, dirname) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/metanorma/compile/extract.rb', line 42

def image_export(xml, dirname)
  xml.at("//image | //xmlns:image") or return
  FileUtils.mkdir_p "#{dirname}/image"
  xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
    next unless /^data:image/.match? s["src"]

    %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
    fn = s["filename"] || sprintf("image-%<num>04d.%<name>s",
                                  num: i, name: imgtype)
    export_output("#{dirname}/image/#{fn}", Base64.strict_decode64(imgdata),
                  binary: true)
  end
end

#load_flavor(stdtype) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/metanorma/compile/compile_validate.rb', line 24

def load_flavor(stdtype)
  stdtype = stdtype.to_sym
  flavor = stdtype2flavor(stdtype)
  @registry.supported_backends.include? stdtype or
    Util.log("[metanorma] Info: Loading `#{flavor}` gem "\
             "for standard type `#{stdtype}`.", :info)
  require_flavor(flavor)
  @registry.supported_backends.include? stdtype or
    Util.log("[metanorma] Error: The `#{flavor}` gem does not "\
             "support the standard type #{stdtype}. Exiting.", :fatal)
end

#options_extract(filename, options) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/metanorma/compile/compile_options.rb', line 19

def options_extract(filename, options)
  content = read_file(filename)
  o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
    .merge(xml_options_extract(content))
  options[:type] ||= o[:type]&.to_sym
  t = @registry.alias(options[:type]) and options[:type] = t
  dir = filename.sub(%r(/[^/]+$), "/")
  options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
  options[:sourcecode] ||= File.join(dir, o[:sourcecode]) if o[:sourcecode]
  options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
  options[:extension_keys] = nil if options[:extension_keys] == [:all]
  options[:format] ||= :asciidoc
  options[:filename] = filename
  options[:fontlicenseagreement] ||= "no-install-fonts"
  options[:novalid] = o[:novalid] if o[:novalid]
  options
end

#options_process(filename, options) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/metanorma/compile/compile.rb', line 40

def options_process(filename, options)
  require_libraries(options)
  options = options_extract(filename, options)
  validate_options(options)
  @log.save_to(filename, options[:output_dir])
  options[:log] = @log
end

#process_ext(ext, file, isodoc, fnames, options) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/metanorma/compile/compile.rb', line 145

def process_ext(ext, file, isodoc, fnames, options)
  fnames[:ext] = @processor.output_formats[ext]
  fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
  isodoc_options = get_isodoc_options(file, options, ext)
  thread = true
  unless process_ext_simple(ext, isodoc, fnames, options,
                            isodoc_options)
    thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
  end
  thread
end

#process_ext_simple(ext, isodoc, fnames, options, isodoc_options) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/metanorma/compile/compile.rb', line 157

def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
  if ext == :rxl
    relaton_export(isodoc, options.merge(relaton: fnames[:out]))
  elsif options[:passthrough_presentation_xml] && ext == :presentation
    #f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
    f = File.exist?(fnames[:orig_filename]) ? fnames[:orig_filename] : fnames[:f]
    FileUtils.cp f, fnames[:presentationxml]
  elsif ext == :html && options[:sectionsplit]
    sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
                         isodoc_options)
  else return false
  end
  true
end

#process_exts(filename, extensions, file, isodoc, options) ⇒ Object

isodoc is Raw Metanorma XML



112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/metanorma/compile/compile.rb', line 112

def process_exts(filename, extensions, file, isodoc, options)
  f = File.expand_path(change_output_dir(options))
  fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
             orig_filename: File.expand_path(filename),
             presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
  if extensions == %i(presentation)
    process_ext(:presentation, file, isodoc, fnames, options)
  else
  process_exts_queue(fnames, extensions, file, isodoc, options)
  end
end

#process_exts1(ext, fnames, isodoc, options, isodoc_options) ⇒ Object 



172
173
174
175
176
177
178
179
180
181
 
# File 'lib/metanorma/compile/compile.rb', line 172

def process_exts1(ext, fnames, isodoc, options, isodoc_options)
  if @processor.use_presentation_xml(ext)
    @queue.schedule(ext, fnames.dup, options.dup,
                    isodoc_options.dup) do |a, b, c, d|
      process_output_threaded(a, b, c, d)
    end
  else
    process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  end
end

#process_exts_queue(fnames, extensions, file, isodoc, options) ⇒ Object 



124
125
126
127
128
129
130
 
# File 'lib/metanorma/compile/compile.rb', line 124

def process_exts_queue(fnames, extensions, file, isodoc, options)
  @queue = ::Metanorma::Util::WorkersPool
    .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
  gather_and_install_fonts(file, options.dup, extensions)
  process_exts_run(fnames, file, isodoc, extensions, options)
  @queue.shutdown
end

#process_exts_run(fnames, file, isodoc, extensions, options) ⇒ Object 



132
133
134
135
136
 
# File 'lib/metanorma/compile/compile.rb', line 132

def process_exts_run(fnames, file, isodoc, extensions, options)
  Util.sort_extensions_execution(extensions).each do |ext|
    process_ext(ext, file, isodoc, fnames, options) or break
  end
end

#process_input(filename, options) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
 
# File 'lib/metanorma/compile/compile.rb', line 53

def process_input(filename, options)
  case extname = File.extname(filename)
  when ".adoc" then process_input_adoc(filename, options)
  when ".xml" then process_input_xml(filename, options)
  else
    Util.log("[metanorma] Error: file extension #{extname} " \
             "is not supported.", :error)
    nil
  end
end

#process_input_adoc(filename, options) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/metanorma/compile/compile.rb', line 64

def process_input_adoc(filename, options)
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
  file = read_file(filename)
  options[:asciimath] and
    file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
  dir = File.dirname(filename)
  dir != "." and
    file = file.gsub(/^include::/, "include::#{dir}/")
      .gsub(/^embed::/, "embed::#{dir}/")
  [file, @processor.input_to_isodoc(file, filename, options)]
end

#process_input_xml(filename, _options) ⇒ Object 



76
77
78
79
80
81
 
# File 'lib/metanorma/compile/compile.rb', line 76

def process_input_xml(filename, _options)
  Util.log("[metanorma] Processing: Metanorma XML input.", :info)
  # TODO NN: this is a hack -- we should provide/bridge the
  # document attributes in Metanorma XML
  ["", read_file(filename)]
end

#process_output_threaded(ext, fnames1, options1, isodoc_options1) ⇒ Object 



183
184
185
186
187
188
189
190
 
# File 'lib/metanorma/compile/compile.rb', line 183

def process_output_threaded(ext, fnames1, options1, isodoc_options1)
  @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
                    isodoc_options1)
  wrap_html(options1, fnames1[:ext], fnames1[:out])
rescue StandardError => e
  strict = ext == :presentation || isodoc_options1[:strict] == true
  isodoc_error_process(e, strict, false)
end

#process_output_unthreaded(ext, fnames, isodoc, isodoc_options) ⇒ Object 



192
193
194
195
196
197
198
199
200
 
# File 'lib/metanorma/compile/compile.rb', line 192

def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
                    isodoc_options)
  true # return as Thread
rescue StandardError => e
  strict = ext == :presentation || isodoc_options[:strict] == "true"
  isodoc_error_process(e, strict, true)
  ext != :presentation
end

#read_file(filename) ⇒ Object 



83
84
85
 
# File 'lib/metanorma/compile/compile.rb', line 83

def read_file(filename)
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
end

#relaton_export(isodoc, options) ⇒ Object 



87
88
89
90
91
92
93
94
95
 
# File 'lib/metanorma/compile/compile.rb', line 87

def relaton_export(isodoc, options)
  return unless options[:relaton]

  xml = Nokogiri::XML(isodoc, &:huge)
  bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
  # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
  # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
  File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
end

#require_libraries(options) ⇒ Object 



5
6
7
 
# File 'lib/metanorma/compile/compile_options.rb', line 5

def require_libraries(options)
  options&.dig(:require)&.each { |r| require r }
end

#requirement_export(xml, dirname) ⇒ Object 



60
61
62
63
64
65
66
67
68
 
# File 'lib/metanorma/compile/extract.rb', line 60

def requirement_export(xml, dirname)
  xml.at(REQUIREMENT_XPATH) or return
  FileUtils.mkdir_p "#{dirname}/requirement"
  xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
    fn = s["filename"] ||
      sprintf("%<name>s-%<num>04d.xml", name: s.name, num: i)
    export_output("#{dirname}/requirement/#{fn}", s)
  end
end

#sectionsplit_convert(input_filename, file, output_filename = nil, opts = {}) ⇒ Object

assume we pass in Presentation XML, but we want to recover Semantic XML



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/metanorma/compile/compile.rb', line 203

def sectionsplit_convert(input_filename, file, output_filename = nil,
                         opts = {})
  @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
  input_filename += ".xml" unless input_filename.match?(/\.xml$/)
  File.exist?(input_filename) or
    File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
  presxml = File.read(input_filename, encoding: "utf-8")
  _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)

  ::Metanorma::Collection::Sectionsplit.new(
    input: input_filename,
    isodoc: @isodoc,
    xml: presxml,
    base: File.basename(output_filename || filename),
    output: output_filename || filename,
    dir: dir,
    compile_opts: opts,
  ).build_collection
end

#sourcecode_export(xml, dirname) ⇒ Object 



32
33
34
35
36
37
38
39
40
 
# File 'lib/metanorma/compile/extract.rb', line 32

def sourcecode_export(xml, dirname)
  xml.at("//sourcecode | //xmlns:sourcecode") or return
  FileUtils.mkdir_p "#{dirname}/sourcecode"
  xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
    filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
    export_output("#{dirname}/sourcecode/#{filename}",
                  clean_sourcecode(s.dup))
  end
end

#stdtype2flavor(stdtype) ⇒ Object 



36
37
38
39
 
# File 'lib/metanorma/compile/compile_validate.rb', line 36

def stdtype2flavor(stdtype)
  flavor = STDTYPE2FLAVOR[stdtype] || stdtype
  "metanorma-#{flavor}"
end

#validate_format(options) ⇒ Object 



17
18
19
20
21
22
 
# File 'lib/metanorma/compile/compile_validate.rb', line 17

def validate_format(options)
  unless options[:format] == :asciidoc
    Util.log("[metanorma] Error: Only source file format currently "\
             "supported is 'asciidoc'.", :fatal)
  end
end

#validate_options(options) ⇒ Object 



3
4
5
6
 
# File 'lib/metanorma/compile/compile_validate.rb', line 3

def validate_options(options)
  validate_type(options)
  validate_format(options)
end

#validate_type(options) ⇒ Object 



8
9
10
11
12
13
14
15
 
# File 'lib/metanorma/compile/compile_validate.rb', line 8

def validate_type(options)
  unless options[:type]
    Util.log("[metanorma] Error: Please specify a standard type: "\
             "#{@registry.supported_backends}.", :fatal)
  end
  stdtype = options[:type].to_sym
  load_flavor(stdtype)
end

#wrap_html(options, file_extension, outfilename) ⇒ Object 



102
103
104
105
106
107
108
109
 
# File 'lib/metanorma/compile/compile.rb', line 102

def wrap_html(options, file_extension, outfilename)
  if options[:wrapper] && /html$/.match(file_extension)
    outfilename = outfilename.sub(/\.html$/, "")
    FileUtils.mkdir_p outfilename
    FileUtils.mv "#{outfilename}.html", outfilename
    FileUtils.mv "#{outfilename}_images", outfilename, force: true
  end
end

#xml_options_extract(file) ⇒ Object 



9
10
11
12
13
14
15
16
17
 
# File 'lib/metanorma/compile/compile_options.rb', line 9

def xml_options_extract(file)
  xml = Nokogiri::XML(file, &:huge)
  if xml.root
    @registry.root_tags.each do |k, v|
      return { type: k } if v == xml.root.name
    end
  end
  {}
end