Module: ActiveSupport::Multibyte::Unicode

Extended by:
Unicode
Included in:
Unicode
Defined in:
activesupport/lib/active_support/multibyte/unicode.rb

Defined Under Namespace

Classes: Codepoint, UnicodeDatabase

Constant Summary collapse

NORMALIZATION_FORMS =

A list of all available normalization forms. See www.unicode.org/reports/tr15/tr15-29.html for more information about normalization.

[:c, :kc, :d, :kd]
UNICODE_VERSION =

The Unicode version that is supported by the implementation

'6.2.0'
HANGUL_SBASE =

Hangul character boundaries and properties

0xAC00
HANGUL_LBASE = 
0x1100
HANGUL_VBASE = 
0x1161
HANGUL_TBASE = 
0x11A7
HANGUL_LCOUNT = 
19
HANGUL_VCOUNT = 
21
HANGUL_TCOUNT = 
28
HANGUL_NCOUNT = 
HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 
11172
HANGUL_SLAST = 
HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 
0x1100
HANGUL_JAMO_LAST = 
0x11FF
WHITESPACE =

All the unicode whitespace

[
  (0x0009..0x000D).to_a, # White_Space # Cc   [5] <control-0009>..<control-000D>
  0x0020,                # White_Space # Zs       SPACE
  0x0085,                # White_Space # Cc       <control-0085>
  0x00A0,                # White_Space # Zs       NO-BREAK SPACE
  0x1680,                # White_Space # Zs       OGHAM SPACE MARK
  0x180E,                # White_Space # Zs       MONGOLIAN VOWEL SEPARATOR
  (0x2000..0x200A).to_a, # White_Space # Zs  [11] EN QUAD..HAIR SPACE
  0x2028,                # White_Space # Zl       LINE SEPARATOR
  0x2029,                # White_Space # Zp       PARAGRAPH SEPARATOR
  0x202F,                # White_Space # Zs       NARROW NO-BREAK SPACE
  0x205F,                # White_Space # Zs       MEDIUM MATHEMATICAL SPACE
  0x3000,                # White_Space # Zs       IDEOGRAPHIC SPACE
].flatten.freeze
LEADERS_AND_TRAILERS =

BOM (byte order mark) can also be seen as whitespace, it’s a non-rendering character used to distinguish between little and big endian. This is not an issue in utf-8, so it must be ignored.

WHITESPACE + [65279]
TRAILERS_PAT = 
/(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u
LEADERS_PAT = 
/\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#default_normalization_form ⇒ Object

The default normalization used for operations that require normalization. It can be set to any of the normalizations in NORMALIZATION_FORMS.

ActiveSupport::Multibyte::Unicode.default_normalization_form = :c



21
22
23
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 21

def default_normalization_form
  @default_normalization_form
end

Class Method Details

.codepoints_to_pattern(array_of_codepoints) ⇒ Object

Returns a regular expression pattern that matches the passed Unicode codepoints.



61
62
63
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 61

def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
  array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
end

Instance Method Details

#compose(codepoints) ⇒ Object

Compose decomposed characters to the composed form.



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 157

def compose(codepoints)
  pos = 0
  eoa = codepoints.length - 1
  starter_pos = 0
  starter_char = codepoints[0]
  previous_combining_class = -1
  while pos < eoa
    pos += 1
    lindex = starter_char - HANGUL_LBASE
    # -- Hangul
    if 0 <= lindex and lindex < HANGUL_LCOUNT
      vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
      if 0 <= vindex and vindex < HANGUL_VCOUNT
        tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
        if 0 <= tindex and tindex < HANGUL_TCOUNT
          j = starter_pos + 2
          eoa -= 2
        else
          tindex = 0
          j = starter_pos + 1
          eoa -= 1
        end
        codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
      end
      starter_pos += 1
      starter_char = codepoints[starter_pos]
    # -- Other characters
    else
      current_char = codepoints[pos]
      current = database.codepoints[current_char]
      if current.combining_class > previous_combining_class
        if ref = database.composition_map[starter_char]
          composition = ref[current_char]
        else
          composition = nil
        end
        unless composition.nil?
          codepoints[starter_pos] = composition
          starter_char = composition
          codepoints.delete_at pos
          eoa -= 1
          pos -= 1
          previous_combining_class = -1
        else
          previous_combining_class = current.combining_class
        end
      else
        previous_combining_class = current.combining_class
      end
      if current.combining_class == 0
        starter_pos = pos
        starter_char = codepoints[pos]
      end
    end
  end
  codepoints
end

#decompose(type, codepoints) ⇒ Object

Decompose composed characters to the decomposed form.



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 136

def decompose(type, codepoints)
  codepoints.inject([]) do |decomposed, cp|
    # if it's a hangul syllable starter character
    if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
      sindex = cp - HANGUL_SBASE
      ncp = [] # new codepoints
      ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
      ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
      tindex = sindex % HANGUL_TCOUNT
      ncp << (HANGUL_TBASE + tindex) unless tindex == 0
      decomposed.concat ncp
    # if the codepoint is decomposable in with the current decomposition type
    elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatibility)
      decomposed.concat decompose(type, ncp.dup)
    else
      decomposed << cp
    end
  end
end

#downcase(string) ⇒ Object 



274
275
276
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 274

def downcase(string)
  apply_mapping string, :lowercase_mapping
end

#in_char_class?(codepoint, classes) ⇒ Boolean

Detect whether the codepoint is in a certain character class. Returns true when it’s in the specified character class and false otherwise. Valid character classes are: :cr, :lf, :l, :v, :lv, :lvt and :t.

Primarily used by the grapheme cluster support.

Returns:

  • (Boolean) 


73
74
75
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 73

def in_char_class?(codepoint, classes)
  classes.detect { |c| database.boundary[c] === codepoint } ? true : false
end

#normalize(string, form = nil) ⇒ Object

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

  • string - The string to perform normalization on.

  • form - The form you want to normalize in. Should be one of the following: :c, :kc, :d, or :kd. Default is ActiveSupport::Multibyte.default_normalization_form.



256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 256

def normalize(string, form=nil)
  form ||= @default_normalization_form
  # See http://www.unicode.org/reports/tr15, Table 1
  codepoints = string.codepoints.to_a
  case form
    when :d
      reorder_characters(decompose(:canonical, codepoints))
    when :c
      compose(reorder_characters(decompose(:canonical, codepoints)))
    when :kd
      reorder_characters(decompose(:compatibility, codepoints))
    when :kc
      compose(reorder_characters(decompose(:compatibility, codepoints)))
    else
      raise ArgumentError, "#{form} is not a valid normalization variant", caller
  end.pack('U*')
end

#pack_graphemes(unpacked) ⇒ Object

Reverse operation of unpack_graphemes.

Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'



115
116
117
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 115

def pack_graphemes(unpacked)
  unpacked.flatten.pack('U*')
end

#reorder_characters(codepoints) ⇒ Object

Re-order codepoints so the string becomes canonical.



120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 120

def reorder_characters(codepoints)
  length = codepoints.length- 1
  pos = 0
  while pos < length do
    cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos+1]]
    if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
      codepoints[pos..pos+1] = cp2.code, cp1.code
      pos += (pos > 0 ? -1 : 1)
    else
      pos += 1
    end
  end
  codepoints
end

#swapcase(string) ⇒ Object 



282
283
284
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 282

def swapcase(string)
  apply_mapping string, :swapcase_mapping
end

#tidy_bytes(string, force = false) ⇒ Object

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing true will forcibly tidy all bytes, assuming that the string’s encoding is entirely CP1252 or ISO-8859-1.



220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 220

def tidy_bytes(string, force = false)
  return string if string.empty?

  if force
    return string.encode(Encoding::UTF_8, Encoding::Windows_1252, invalid: :replace, undef: :replace)
  end

  # We can't transcode to the same format, so we choose a nearly-identical encoding.
  # We're going to 'transcode' bytes from UTF-8 when possible, then fall back to
  # CP1252 when we get errors. The final string will be 'converted' back to UTF-8
  # before returning.
  reader = Encoding::Converter.new(Encoding::UTF_8, Encoding::UTF_8_MAC)

  source = string.dup
  out = ''.force_encoding(Encoding::UTF_8_MAC)

  loop do
    reader.primitive_convert(source, out)
    _, _, _, error_bytes, _ = reader.primitive_errinfo
    break if error_bytes.nil?
    out << error_bytes.encode(Encoding::UTF_8_MAC, Encoding::Windows_1252, invalid: :replace, undef: :replace)
  end

  reader.finish

  out.encode!(Encoding::UTF_8)
end

#unpack_graphemes(string) ⇒ Object

Unpack the string at grapheme boundaries. Returns a list of character lists.

Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 82

def unpack_graphemes(string)
  codepoints = string.codepoints.to_a
  unpacked = []
  pos = 0
  marker = 0
  eoc = codepoints.length
  while(pos < eoc)
    pos += 1
    previous = codepoints[pos-1]
    current = codepoints[pos]
    if (
        # CR X LF
        ( previous == database.boundary[:cr] and current == database.boundary[:lf] ) or
        # L X (L|V|LV|LVT)
        ( database.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
        # (LV|V) X (V|T)
        ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
        # (LVT|T) X (T)
        ( in_char_class?(previous, [:lvt,:t]) and database.boundary[:t] === current ) or
        # X Extend
        (database.boundary[:extend] === current)
      )
    else
      unpacked << codepoints[marker..pos-1]
      marker = pos
    end
  end
  unpacked
end

#upcase(string) ⇒ Object 



278
279
280
 
# File 'activesupport/lib/active_support/multibyte/unicode.rb', line 278

def upcase(string)
  apply_mapping string, :uppercase_mapping
end