Module: ActiveSupport::Multibyte::Unicode

Extended by:
Unicode
Included in:
Unicode
Defined in:
lib/active_support/multibyte/unicode.rb

Defined Under Namespace

Classes: Codepoint, UnicodeDatabase

Constant Summary collapse

NORMALIZATION_FORMS =

A list of all available normalization forms. See www.unicode.org/reports/tr15/tr15-29.html for more information about normalization.

[:c, :kc, :d, :kd]
UNICODE_VERSION =

The Unicode version that is supported by the implementation

"9.0.0"
HANGUL_SBASE =

Hangul character boundaries and properties

0xAC00
HANGUL_LBASE = 
0x1100
HANGUL_VBASE = 
0x1161
HANGUL_TBASE = 
0x11A7
HANGUL_LCOUNT = 
19
HANGUL_VCOUNT = 
21
HANGUL_TCOUNT = 
28
HANGUL_NCOUNT = 
HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 
11172
HANGUL_SLAST = 
HANGUL_SBASE + HANGUL_SCOUNT

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#default_normalization_form ⇒ Object

The default normalization used for operations that require normalization. It can be set to any of the normalizations in NORMALIZATION_FORMS.

ActiveSupport::Multibyte::Unicode.default_normalization_form = :c



21
22
23
 
# File 'lib/active_support/multibyte/unicode.rb', line 21

def default_normalization_form
  @default_normalization_form
end

Instance Method Details

#compose(codepoints) ⇒ Object

Compose decomposed characters to the composed form.



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/active_support/multibyte/unicode.rb', line 161

def compose(codepoints)
  pos = 0
  eoa = codepoints.length - 1
  starter_pos = 0
  starter_char = codepoints[0]
  previous_combining_class = -1
  while pos < eoa
    pos += 1
    lindex = starter_char - HANGUL_LBASE
    # -- Hangul
    if 0 <= lindex && lindex < HANGUL_LCOUNT
      vindex = codepoints[starter_pos + 1] - HANGUL_VBASE rescue vindex = -1
      if 0 <= vindex && vindex < HANGUL_VCOUNT
        tindex = codepoints[starter_pos + 2] - HANGUL_TBASE rescue tindex = -1
        if 0 <= tindex && tindex < HANGUL_TCOUNT
          j = starter_pos + 2
          eoa -= 2
        else
          tindex = 0
          j = starter_pos + 1
          eoa -= 1
        end
        codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
      end
      starter_pos += 1
      starter_char = codepoints[starter_pos]
    # -- Other characters
    else
      current_char = codepoints[pos]
      current = database.codepoints[current_char]
      if current.combining_class > previous_combining_class
        if ref = database.composition_map[starter_char]
          composition = ref[current_char]
        else
          composition = nil
        end
        unless composition.nil?
          codepoints[starter_pos] = composition
          starter_char = composition
          codepoints.delete_at pos
          eoa -= 1
          pos -= 1
          previous_combining_class = -1
        else
          previous_combining_class = current.combining_class
        end
      else
        previous_combining_class = current.combining_class
      end
      if current.combining_class == 0
        starter_pos = pos
        starter_char = codepoints[pos]
      end
    end
  end
  codepoints
end

#decompose(type, codepoints) ⇒ Object

Decompose composed characters to the decomposed form.



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/active_support/multibyte/unicode.rb', line 140

def decompose(type, codepoints)
  codepoints.inject([]) do |decomposed, cp|
    # if it's a hangul syllable starter character
    if HANGUL_SBASE <= cp && cp < HANGUL_SLAST
      sindex = cp - HANGUL_SBASE
      ncp = [] # new codepoints
      ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
      ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
      tindex = sindex % HANGUL_TCOUNT
      ncp << (HANGUL_TBASE + tindex) unless tindex == 0
      decomposed.concat ncp
    # if the codepoint is decomposable in with the current decomposition type
    elsif (ncp = database.codepoints[cp].decomp_mapping) && (!database.codepoints[cp].decomp_type || type == :compatibility)
      decomposed.concat decompose(type, ncp.dup)
    else
      decomposed << cp
    end
  end
end

#downcase(string) ⇒ Object 



284
285
286
 
# File 'lib/active_support/multibyte/unicode.rb', line 284

def downcase(string)
  apply_mapping string, :lowercase_mapping
end

#in_char_class?(codepoint, classes) ⇒ Boolean

Detect whether the codepoint is in a certain character class. Returns true when it’s in the specified character class and false otherwise. Valid character classes are: :cr, :lf, :l, :v, :lv, :lvt and :t.

Primarily used by the grapheme cluster support.

Returns:

  • (Boolean) 


42
43
44
 
# File 'lib/active_support/multibyte/unicode.rb', line 42

def in_char_class?(codepoint, classes)
  classes.detect { |c| database.boundary[c] === codepoint } ? true : false
end

#normalize(string, form = nil) ⇒ Object

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

  • string - The string to perform normalization on.

  • form - The form you want to normalize in. Should be one of the following: :c, :kc, :d, or :kd. Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.



266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
 
# File 'lib/active_support/multibyte/unicode.rb', line 266

def normalize(string, form = nil)
  form ||= @default_normalization_form
  # See http://www.unicode.org/reports/tr15, Table 1
  codepoints = string.codepoints.to_a
  case form
  when :d
    reorder_characters(decompose(:canonical, codepoints))
  when :c
    compose(reorder_characters(decompose(:canonical, codepoints)))
  when :kd
    reorder_characters(decompose(:compatibility, codepoints))
  when :kc
    compose(reorder_characters(decompose(:compatibility, codepoints)))
  else
    raise ArgumentError, "#{form} is not a valid normalization variant", caller
  end.pack("U*".freeze)
end

#pack_graphemes(unpacked) ⇒ Object

Reverse operation of unpack_graphemes.

Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'



119
120
121
 
# File 'lib/active_support/multibyte/unicode.rb', line 119

def pack_graphemes(unpacked)
  unpacked.flatten.pack("U*")
end

#reorder_characters(codepoints) ⇒ Object

Re-order codepoints so the string becomes canonical.



124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/active_support/multibyte/unicode.rb', line 124

def reorder_characters(codepoints)
  length = codepoints.length - 1
  pos = 0
  while pos < length do
    cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos + 1]]
    if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
      codepoints[pos..pos + 1] = cp2.code, cp1.code
      pos += (pos > 0 ? -1 : 1)
    else
      pos += 1
    end
  end
  codepoints
end

#swapcase(string) ⇒ Object 



292
293
294
 
# File 'lib/active_support/multibyte/unicode.rb', line 292

def swapcase(string)
  apply_mapping string, :swapcase_mapping
end

#tidy_bytes(string, force = false) ⇒ Object

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing true will forcibly tidy all bytes, assuming that the string’s encoding is entirely CP1252 or ISO-8859-1.



226
227
228
229
230
 
# File 'lib/active_support/multibyte/unicode.rb', line 226

def tidy_bytes(string, force = false)
  return string if string.empty?
  return recode_windows1252_chars(string) if force
  string.scrub { |bad| recode_windows1252_chars(bad) }
end

#unpack_graphemes(string) ⇒ Object

Unpack the string at grapheme boundaries. Returns a list of character lists.

Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/active_support/multibyte/unicode.rb', line 51

def unpack_graphemes(string)
  codepoints = string.codepoints.to_a
  unpacked = []
  pos = 0
  marker = 0
  eoc = codepoints.length
  while (pos < eoc)
    pos += 1
    previous = codepoints[pos - 1]
    current = codepoints[pos]

    # See http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
    should_break =
      if pos == eoc
        true
      # GB3. CR X LF
      elsif previous == database.boundary[:cr] && current == database.boundary[:lf]
        false
      # GB4. (Control|CR|LF) ÷
      elsif previous && in_char_class?(previous, [:control, :cr, :lf])
        true
      # GB5. ÷ (Control|CR|LF)
      elsif in_char_class?(current, [:control, :cr, :lf])
        true
      # GB6. L X (L|V|LV|LVT)
      elsif database.boundary[:l] === previous && in_char_class?(current, [:l, :v, :lv, :lvt])
        false
      # GB7. (LV|V) X (V|T)
      elsif in_char_class?(previous, [:lv, :v]) && in_char_class?(current, [:v, :t])
        false
      # GB8. (LVT|T) X (T)
      elsif in_char_class?(previous, [:lvt, :t]) && database.boundary[:t] === current
        false
      # GB9. X (Extend | ZWJ)
      elsif in_char_class?(current, [:extend, :zwj])
        false
      # GB9a. X SpacingMark
      elsif database.boundary[:spacingmark] === current
        false
      # GB9b. Prepend X
      elsif database.boundary[:prepend] === previous
        false
      # GB10. (E_Base | EBG) Extend* X E_Modifier
      elsif (marker...pos).any? { |i| in_char_class?(codepoints[i], [:e_base, :e_base_gaz]) && codepoints[i + 1...pos].all? { |c| database.boundary[:extend] === c } } && database.boundary[:e_modifier] === current
        false
      # GB11. ZWJ X (Glue_After_Zwj | EBG)
      elsif database.boundary[:zwj] === previous && in_char_class?(current, [:glue_after_zwj, :e_base_gaz])
        false
      # GB12. ^ (RI RI)* RI X RI
      # GB13. [^RI] (RI RI)* RI X RI
      elsif codepoints[marker..pos].all? { |c| database.boundary[:regional_indicator] === c } && codepoints[marker..pos].count { |c| database.boundary[:regional_indicator] === c }.even?
        false
      # GB999. Any ÷ Any
      else
        true
      end

    if should_break
      unpacked << codepoints[marker..pos - 1]
      marker = pos
    end
  end
  unpacked
end

#upcase(string) ⇒ Object 



288
289
290
 
# File 'lib/active_support/multibyte/unicode.rb', line 288

def upcase(string)
  apply_mapping string, :uppercase_mapping
end