Class: TinyJapaneseSegmenter

Inherits:

Object

• Object
• TinyJapaneseSegmenter

Defined in:

lib/tiny_japanese_segmenter.rb

Overview

Ruby port of chasen.org/~taku/software/TinySegmenter/tiny_segmenter-0.2.js This is esstentially a trained machine learning model used to segment words in Japanese. Discourse core uses it for “best effort” segmentation of Japanese text for search.

Constant Summary

CHARTYPE =

{
  "[一二三四五六七八九十百千万億兆]" => "M",
  "[一-龠々〆ヵヶ]" => "H",
  "[ぁ-ん]" => "I",
  "[ァ-ヴーｱ-ﾝﾞｰ]" => "K",
  "[a-zA-Zａ-ｚＡ-Ｚ]" => "A",
  "[0-9０-９]" => "N",
}.map { |pattern, value| [Regexp.compile(pattern), value] }

BIAS =

-322

BC1 =

{ "HH" => 6, "II" => 2461, "KH" => 406, "OH" => -1378 }

BC2 =

{
  "AA" => -3267,
  "AI" => 2744,
  "AN" => -878,
  "HH" => -4070,
  "HM" => -1711,
  "HN" => 4012,
  "HO" => 3761,
  "IA" => 1327,
  "IH" => -1184,
  "II" => -1332,
  "IK" => 1721,
  "IO" => 5492,
  "KI" => 3831,
  "KK" => -8741,
  "MH" => -3132,
  "MK" => 3334,
  "OO" => -2920,
}

BC3 =

{
  "HH" => 996,
  "HI" => 626,
  "HK" => -721,
  "HN" => -1307,
  "HO" => -836,
  "IH" => -301,
  "KK" => 2762,
  "MK" => 1079,
  "MM" => 4034,
  "OA" => -1652,
  "OH" => 266,
}

BP1 =

{ "BB" => 295, "OB" => 304, "OO" => -125, "UB" => 352 }

BP2 =

{ "BO" => 60, "OO" => -1762 }

BQ1 =

{
  "BHH" => 1150,
  "BHM" => 1521,
  "BII" => -1158,
  "BIM" => 886,
  "BMH" => 1208,
  "BNH" => 449,
  "BOH" => -91,
  "BOO" => -2597,
  "OHI" => 451,
  "OIH" => -296,
  "OKA" => 1851,
  "OKH" => -1020,
  "OKK" => 904,
  "OOO" => 2965,
}

BQ2 =

{
  "BHH" => 118,
  "BHI" => -1159,
  "BHM" => 466,
  "BIH" => -919,
  "BKK" => -1720,
  "BKO" => 864,
  "OHH" => -1139,
  "OHM" => -181,
  "OIH" => 153,
  "UHI" => -1146,
}

BQ3 =

{
  "BHH" => -792,
  "BHI" => 2664,
  "BII" => -299,
  "BKI" => 419,
  "BMH" => 937,
  "BMM" => 8335,
  "BNN" => 998,
  "BOH" => 775,
  "OHH" => 2174,
  "OHM" => 439,
  "OII" => 280,
  "OKH" => 1798,
  "OKI" => -793,
  "OKO" => -2242,
  "OMH" => -2402,
  "OOO" => 11_699,
}

BQ4 =

{
  "BHH" => -3895,
  "BIH" => 3761,
  "BII" => -4654,
  "BIK" => 1348,
  "BKK" => -1806,
  "BMI" => -3385,
  "BOO" => -12_396,
  "OAH" => 926,
  "OHH" => 266,
  "OHK" => -2036,
  "ONN" => -973,
}

BW1 =

{
  ",と" => 660,
  ",同" => 727,
  "B1あ" => 1404,
  "B1同" => 542,
  "、と" => 660,
  "、同" => 727,
  "」と" => 1682,
  "あっ" => 1505,
  "いう" => 1743,
  "いっ" => -2055,
  "いる" => 672,
  "うし" => -4817,
  "うん" => 665,
  "から" => 3472,
  "がら" => 600,
  "こう" => -790,
  "こと" => 2083,
  "こん" => -1262,
  "さら" => -4143,
  "さん" => 4573,
  "した" => 2641,
  "して" => 1104,
  "すで" => -3399,
  "そこ" => 1977,
  "それ" => -871,
  "たち" => 1122,
  "ため" => 601,
  "った" => 3463,
  "つい" => -802,
  "てい" => 805,
  "てき" => 1249,
  "でき" => 1127,
  "です" => 3445,
  "では" => 844,
  "とい" => -4915,
  "とみ" => 1922,
  "どこ" => 3887,
  "ない" => 5713,
  "なっ" => 3015,
  "など" => 7379,
  "なん" => -1113,
  "にし" => 2468,
  "には" => 1498,
  "にも" => 1671,
  "に対" => -912,
  "の一" => -501,
  "の中" => 741,
  "ませ" => 2448,
  "まで" => 1711,
  "まま" => 2600,
  "まる" => -2155,
  "やむ" => -1947,
  "よっ" => -2565,
  "れた" => 2369,
  "れで" => -913,
  "をし" => 1860,
  "を見" => 731,
  "亡く" => -1886,
  "京都" => 2558,
  "取り" => -2784,
  "大き" => -2604,
  "大阪" => 1497,
  "平方" => -2314,
  "引き" => -1336,
  "日本" => -195,
  "本当" => -2423,
  "毎日" => -2113,
  "目指" => -724,
  "Ｂ１あ" => 1404,
  "Ｂ１同" => 542,
  "｣と" => 1682,
}

BW2 =

{
  ".." => -11_822,
  "11" => -669,
  "――" => -5730,
  "−−" => -13_175,
  "いう" => -1609,
  "うか" => 2490,
  "かし" => -1350,
  "かも" => -602,
  "から" => -7194,
  "かれ" => 4612,
  "がい" => 853,
  "がら" => -3198,
  "きた" => 1941,
  "くな" => -1597,
  "こと" => -8392,
  "この" => -4193,
  "させ" => 4533,
  "され" => 13_168,
  "さん" => -3977,
  "しい" => -1819,
  "しか" => -545,
  "した" => 5078,
  "して" => 972,
  "しな" => 939,
  "その" => -3744,
  "たい" => -1253,
  "たた" => -662,
  "ただ" => -3857,
  "たち" => -786,
  "たと" => 1224,
  "たは" => -939,
  "った" => 4589,
  "って" => 1647,
  "っと" => -2094,
  "てい" => 6144,
  "てき" => 3640,
  "てく" => 2551,
  "ては" => -3110,
  "ても" => -3065,
  "でい" => 2666,
  "でき" => -1528,
  "でし" => -3828,
  "です" => -4761,
  "でも" => -4203,
  "とい" => 1890,
  "とこ" => -1746,
  "とと" => -2279,
  "との" => 720,
  "とみ" => 5168,
  "とも" => -3941,
  "ない" => -2488,
  "なが" => -1313,
  "など" => -6509,
  "なの" => 2614,
  "なん" => 3099,
  "にお" => -1615,
  "にし" => 2748,
  "にな" => 2454,
  "によ" => -7236,
  "に対" => -14_943,
  "に従" => -4688,
  "に関" => -11_388,
  "のか" => 2093,
  "ので" => -7059,
  "のに" => -6041,
  "のの" => -6125,
  "はい" => 1073,
  "はが" => -1033,
  "はず" => -2532,
  "ばれ" => 1813,
  "まし" => -1316,
  "まで" => -6621,
  "まれ" => 5409,
  "めて" => -3153,
  "もい" => 2230,
  "もの" => -10_713,
  "らか" => -944,
  "らし" => -1611,
  "らに" => -1897,
  "りし" => 651,
  "りま" => 1620,
  "れた" => 4270,
  "れて" => 849,
  "れば" => 4114,
  "ろう" => 6067,
  "われ" => 7901,
  "を通" => -11_877,
  "んだ" => 728,
  "んな" => -4115,
  "一人" => 602,
  "一方" => -1375,
  "一日" => 970,
  "一部" => -1051,
  "上が" => -4479,
  "会社" => -1116,
  "出て" => 2163,
  "分の" => -7758,
  "同党" => 970,
  "同日" => -913,
  "大阪" => -2471,
  "委員" => -1250,
  "少な" => -1050,
  "年度" => -8669,
  "年間" => -1626,
  "府県" => -2363,
  "手権" => -1982,
  "新聞" => -4066,
  "日新" => -722,
  "日本" => -7068,
  "日米" => 3372,
  "曜日" => -601,
  "朝鮮" => -2355,
  "本人" => -2697,
  "東京" => -1543,
  "然と" => -1384,
  "社会" => -1276,
  "立て" => -990,
  "第に" => -1612,
  "米国" => -4268,
  "１１" => -669,
}

BW3 =

{
  "あた" => -2194,
  "あり" => 719,
  "ある" => 3846,
  "い." => -1185,
  "い。" => -1185,
  "いい" => 5308,
  "いえ" => 2079,
  "いく" => 3029,
  "いた" => 2056,
  "いっ" => 1883,
  "いる" => 5600,
  "いわ" => 1527,
  "うち" => 1117,
  "うと" => 4798,
  "えと" => 1454,
  "か." => 2857,
  "か。" => 2857,
  "かけ" => -743,
  "かっ" => -4098,
  "かに" => -669,
  "から" => 6520,
  "かり" => -2670,
  "が," => 1816,
  "が、" => 1816,
  "がき" => -4855,
  "がけ" => -1127,
  "がっ" => -913,
  "がら" => -4977,
  "がり" => -2064,
  "きた" => 1645,
  "けど" => 1374,
  "こと" => 7397,
  "この" => 1542,
  "ころ" => -2757,
  "さい" => -714,
  "さを" => 976,
  "し," => 1557,
  "し、" => 1557,
  "しい" => -3714,
  "した" => 3562,
  "して" => 1449,
  "しな" => 2608,
  "しま" => 1200,
  "す." => -1310,
  "す。" => -1310,
  "する" => 6521,
  "ず," => 3426,
  "ず、" => 3426,
  "ずに" => 841,
  "そう" => 428,
  "た." => 8875,
  "た。" => 8875,
  "たい" => -594,
  "たの" => 812,
  "たり" => -1183,
  "たる" => -853,
  "だ." => 4098,
  "だ。" => 4098,
  "だっ" => 1004,
  "った" => -4748,
  "って" => 300,
  "てい" => 6240,
  "てお" => 855,
  "ても" => 302,
  "です" => 1437,
  "でに" => -1482,
  "では" => 2295,
  "とう" => -1387,
  "とし" => 2266,
  "との" => 541,
  "とも" => -3543,
  "どう" => 4664,
  "ない" => 1796,
  "なく" => -903,
  "など" => 2135,
  "に," => -1021,
  "に、" => -1021,
  "にし" => 1771,
  "にな" => 1906,
  "には" => 2644,
  "の," => -724,
  "の、" => -724,
  "の子" => -1000,
  "は," => 1337,
  "は、" => 1337,
  "べき" => 2181,
  "まし" => 1113,
  "ます" => 6943,
  "まっ" => -1549,
  "まで" => 6154,
  "まれ" => -793,
  "らし" => 1479,
  "られ" => 6820,
  "るる" => 3818,
  "れ," => 854,
  "れ、" => 854,
  "れた" => 1850,
  "れて" => 1375,
  "れば" => -3246,
  "れる" => 1091,
  "われ" => -605,
  "んだ" => 606,
  "んで" => 798,
  "カ月" => 990,
  "会議" => 860,
  "入り" => 1232,
  "大会" => 2217,
  "始め" => 1681,
  "市" => 965,
  "新聞" => -5055,
  "日," => 974,
  "日、" => 974,
  "社会" => 2024,
  "ｶ月" => 990,
}

TC1 =

{
  "AAA" => 1093,
  "HHH" => 1029,
  "HHM" => 580,
  "HII" => 998,
  "HOH" => -390,
  "HOM" => -331,
  "IHI" => 1169,
  "IOH" => -142,
  "IOI" => -1015,
  "IOM" => 467,
  "MMH" => 187,
  "OOI" => -1832,
}

TC2 =

{
  "HHO" => 2088,
  "HII" => -1023,
  "HMM" => -1154,
  "IHI" => -1965,
  "KKH" => 703,
  "OII" => -2649,
}

TC3 =

{
  "AAA" => -294,
  "HHH" => 346,
  "HHI" => -341,
  "HII" => -1088,
  "HIK" => 731,
  "HOH" => -1486,
  "IHH" => 128,
  "IHI" => -3041,
  "IHO" => -1935,
  "IIH" => -825,
  "IIM" => -1035,
  "IOI" => -542,
  "KHH" => -1216,
  "KKA" => 491,
  "KKH" => -1217,
  "KOK" => -1009,
  "MHH" => -2694,
  "MHM" => -457,
  "MHO" => 123,
  "MMH" => -471,
  "NNH" => -1689,
  "NNO" => 662,
  "OHO" => -3393,
}

TC4 =

{
  "HHH" => -203,
  "HHI" => 1344,
  "HHK" => 365,
  "HHM" => -122,
  "HHN" => 182,
  "HHO" => 669,
  "HIH" => 804,
  "HII" => 679,
  "HOH" => 446,
  "IHH" => 695,
  "IHO" => -2324,
  "IIH" => 321,
  "III" => 1497,
  "IIO" => 656,
  "IOO" => 54,
  "KAK" => 4845,
  "KKA" => 3386,
  "KKK" => 3065,
  "MHH" => -405,
  "MHI" => 201,
  "MMH" => -241,
  "MMM" => 661,
  "MOM" => 841,
}

TQ1 =

{
  "BHHH" => -227,
  "BHHI" => 316,
  "BHIH" => -132,
  "BIHH" => 60,
  "BIII" => 1595,
  "BNHH" => -744,
  "BOHH" => 225,
  "BOOO" => -908,
  "OAKK" => 482,
  "OHHH" => 281,
  "OHIH" => 249,
  "OIHI" => 200,
  "OIIH" => -68,
}

TQ2 =

{ "BIHH" => -1401, "BIII" => -1033, "BKAK" => -543, "BOOO" => -5591 }

TQ3 =

{
  "BHHH" => 478,
  "BHHM" => -1073,
  "BHIH" => 222,
  "BHII" => -504,
  "BIIH" => -116,
  "BIII" => -105,
  "BMHI" => -863,
  "BMHM" => -464,
  "BOMH" => 620,
  "OHHH" => 346,
  "OHHI" => 1729,
  "OHII" => 997,
  "OHMH" => 481,
  "OIHH" => 623,
  "OIIH" => 1344,
  "OKAK" => 2792,
  "OKHH" => 587,
  "OKKA" => 679,
  "OOHH" => 110,
  "OOII" => -685,
}

TQ4 =

{
  "BHHH" => -721,
  "BHHM" => -3604,
  "BHII" => -966,
  "BIIH" => -607,
  "BIII" => -2181,
  "OAAA" => -2763,
  "OAKK" => 180,
  "OHHH" => -294,
  "OHHI" => 2446,
  "OHHO" => 480,
  "OHIH" => -1573,
  "OIHH" => 1935,
  "OIHI" => -493,
  "OIIH" => 626,
  "OIII" => -4007,
  "OKAK" => -8156,
}

TW1 =

{ "につい" => -4681, "東京都" => 2026 }

TW2 =

{
  "ある程" => -2049,
  "いった" => -1256,
  "ころが" => -2434,
  "しょう" => 3873,
  "その後" => -4430,
  "だって" => -1049,
  "ていた" => 1833,
  "として" => -4657,
  "ともに" => -4517,
  "もので" => 1882,
  "一気に" => -792,
  "初めて" => -1512,
  "同時に" => -8097,
  "大きな" => -1255,
  "対して" => -2721,
  "社会党" => -3216,
}

TW3 =

{
  "いただ" => -1734,
  "してい" => 1314,
  "として" => -4314,
  "につい" => -5483,
  "にとっ" => -5989,
  "に当た" => -6247,
  "ので," => -727,
  "ので、" => -727,
  "のもの" => -600,
  "れから" => -3752,
  "十二月" => -2287,
}

TW4 =

{
  "いう." => 8576,
  "いう。" => 8576,
  "からな" => -2348,
  "してい" => 2958,
  "たが," => 1516,
  "たが、" => 1516,
  "ている" => 1538,
  "という" => 1349,
  "ました" => 5543,
  "ません" => 1097,
  "ようと" => -4258,
  "よると" => 5865,
}

UC1 =

{ "A" => 484, "K" => 93, "M" => 645, "O" => -505 }

UC2 =

{ "A" => 819, "H" => 1059, "I" => 409, "M" => 3987, "N" => 5775, "O" => 646 }

UC3 =

{ "A" => -1370, "I" => 2311 }

UC4 =

{
  "A" => -2643,
  "H" => 1809,
  "I" => -1032,
  "K" => -3450,
  "M" => 3565,
  "N" => 3876,
  "O" => 6646,
}

UC5 =

{ "H" => 313, "I" => -1238, "K" => -799, "M" => 539, "O" => -831 }

UC6 =

{ "H" => -506, "I" => -253, "K" => 87, "M" => 247, "O" => -387 }

UP1 =

{ "O" => -214 }

UP2 =

{ "B" => 69, "O" => 935 }

UP3 =

{ "B" => 189 }

UQ1 =

{
  "BH" => 21,
  "BI" => -12,
  "BK" => -99,
  "BN" => 142,
  "BO" => -56,
  "OH" => -95,
  "OI" => 477,
  "OK" => 410,
  "OO" => -2422,
}

UQ2 =

{ "BH" => 216, "BI" => 113, "OK" => 1759 }

UQ3 =

{
  "BA" => -479,
  "BH" => 42,
  "BI" => 1913,
  "BK" => -7198,
  "BM" => 3160,
  "BN" => 6427,
  "BO" => 14_761,
  "OI" => -827,
  "ON" => -3212,
}

UW1 =

{
  "," => 156,
  "、" => 156,
  "「" => -463,
  "あ" => -941,
  "う" => -127,
  "が" => -553,
  "き" => 121,
  "こ" => 505,
  "で" => -201,
  "と" => -547,
  "ど" => -123,
  "に" => -789,
  "の" => -185,
  "は" => -847,
  "も" => -466,
  "や" => -470,
  "よ" => 182,
  "ら" => -292,
  "り" => 208,
  "れ" => 169,
  "を" => -446,
  "ん" => -137,
  "・" => -135,
  "主" => -402,
  "京" => -268,
  "区" => -912,
  "午" => 871,
  "国" => -460,
  "大" => 561,
  "委" => 729,
  "市" => -411,
  "日" => -141,
  "理" => 361,
  "生" => -408,
  "県" => -386,
  "都" => -718,
  "｢" => -463,
  "･" => -135,
}

UW2 =

{
  "," => -829,
  "、" => -829,
  "〇" => 892,
  "「" => -645,
  "」" => 3145,
  "あ" => -538,
  "い" => 505,
  "う" => 134,
  "お" => -502,
  "か" => 1454,
  "が" => -856,
  "く" => -412,
  "こ" => 1141,
  "さ" => 878,
  "ざ" => 540,
  "し" => 1529,
  "す" => -675,
  "せ" => 300,
  "そ" => -1011,
  "た" => 188,
  "だ" => 1837,
  "つ" => -949,
  "て" => -291,
  "で" => -268,
  "と" => -981,
  "ど" => 1273,
  "な" => 1063,
  "に" => -1764,
  "の" => 130,
  "は" => -409,
  "ひ" => -1273,
  "べ" => 1261,
  "ま" => 600,
  "も" => -1263,
  "や" => -402,
  "よ" => 1639,
  "り" => -579,
  "る" => -694,
  "れ" => 571,
  "を" => -2516,
  "ん" => 2095,
  "ア" => -587,
  "カ" => 306,
  "キ" => 568,
  "ッ" => 831,
  "三" => -758,
  "不" => -2150,
  "世" => -302,
  "中" => -968,
  "主" => -861,
  "事" => 492,
  "人" => -123,
  "会" => 978,
  "保" => 362,
  "入" => 548,
  "初" => -3025,
  "副" => -1566,
  "北" => -3414,
  "区" => -422,
  "大" => -1769,
  "天" => -865,
  "太" => -483,
  "子" => -1519,
  "学" => 760,
  "実" => 1023,
  "小" => -2009,
  "市" => -813,
  "年" => -1060,
  "強" => 1067,
  "手" => -1519,
  "揺" => -1033,
  "政" => 1522,
  "文" => -1355,
  "新" => -1682,
  "日" => -1815,
  "明" => -1462,
  "最" => -630,
  "朝" => -1843,
  "本" => -1650,
  "東" => -931,
  "果" => -665,
  "次" => -2378,
  "民" => -180,
  "気" => -1740,
  "理" => 752,
  "発" => 529,
  "目" => -1584,
  "相" => -242,
  "県" => -1165,
  "立" => -763,
  "第" => 810,
  "米" => 509,
  "自" => -1353,
  "行" => 838,
  "西" => -744,
  "見" => -3874,
  "調" => 1010,
  "議" => 1198,
  "込" => 3041,
  "開" => 1758,
  "間" => -1257,
  "｢" => -645,
  "｣" => 3145,
  "ｯ" => 831,
  "ｱ" => -587,
  "ｶ" => 306,
  "ｷ" => 568,
}

UW3 =

{
  "," => 4889,
  "1" => -800,
  "−" => -1723,
  "、" => 4889,
  "々" => -2311,
  "〇" => 5827,
  "」" => 2670,
  "〓" => -3573,
  "あ" => -2696,
  "い" => 1006,
  "う" => 2342,
  "え" => 1983,
  "お" => -4864,
  "か" => -1163,
  "が" => 3271,
  "く" => 1004,
  "け" => 388,
  "げ" => 401,
  "こ" => -3552,
  "ご" => -3116,
  "さ" => -1058,
  "し" => -395,
  "す" => 584,
  "せ" => 3685,
  "そ" => -5228,
  "た" => 842,
  "ち" => -521,
  "っ" => -1444,
  "つ" => -1081,
  "て" => 6167,
  "で" => 2318,
  "と" => 1691,
  "ど" => -899,
  "な" => -2788,
  "に" => 2745,
  "の" => 4056,
  "は" => 4555,
  "ひ" => -2171,
  "ふ" => -1798,
  "へ" => 1199,
  "ほ" => -5516,
  "ま" => -4384,
  "み" => -120,
  "め" => 1205,
  "も" => 2323,
  "や" => -788,
  "よ" => -202,
  "ら" => 727,
  "り" => 649,
  "る" => 5905,
  "れ" => 2773,
  "わ" => -1207,
  "を" => 6620,
  "ん" => -518,
  "ア" => 551,
  "グ" => 1319,
  "ス" => 874,
  "ッ" => -1350,
  "ト" => 521,
  "ム" => 1109,
  "ル" => 1591,
  "ロ" => 2201,
  "ン" => 278,
  "・" => -3794,
  "一" => -1619,
  "下" => -1759,
  "世" => -2087,
  "両" => 3815,
  "中" => 653,
  "主" => -758,
  "予" => -1193,
  "二" => 974,
  "人" => 2742,
  "今" => 792,
  "他" => 1889,
  "以" => -1368,
  "低" => 811,
  "何" => 4265,
  "作" => -361,
  "保" => -2439,
  "元" => 4858,
  "党" => 3593,
  "全" => 1574,
  "公" => -3030,
  "六" => 755,
  "共" => -1880,
  "円" => 5807,
  "再" => 3095,
  "分" => 457,
  "初" => 2475,
  "別" => 1129,
  "前" => 2286,
  "副" => 4437,
  "力" => 365,
  "動" => -949,
  "務" => -1872,
  "化" => 1327,
  "北" => -1038,
  "区" => 4646,
  "千" => -2309,
  "午" => -783,
  "協" => -1006,
  "口" => 483,
  "右" => 1233,
  "各" => 3588,
  "合" => -241,
  "同" => 3906,
  "和" => -837,
  "員" => 4513,
  "国" => 642,
  "型" => 1389,
  "場" => 1219,
  "外" => -241,
  "妻" => 2016,
  "学" => -1356,
  "安" => -423,
  "実" => -1008,
  "家" => 1078,
  "小" => -513,
  "少" => -3102,
  "州" => 1155,
  "市" => 3197,
  "平" => -1804,
  "年" => 2416,
  "広" => -1030,
  "府" => 1605,
  "度" => 1452,
  "建" => -2352,
  "当" => -3885,
  "得" => 1905,
  "思" => -1291,
  "性" => 1822,
  "戸" => -488,
  "指" => -3973,
  "政" => -2013,
  "教" => -1479,
  "数" => 3222,
  "文" => -1489,
  "新" => 1764,
  "日" => 2099,
  "旧" => 5792,
  "昨" => -661,
  "時" => -1248,
  "曜" => -951,
  "最" => -937,
  "月" => 4125,
  "期" => 360,
  "李" => 3094,
  "村" => 364,
  "東" => -805,
  "核" => 5156,
  "森" => 2438,
  "業" => 484,
  "氏" => 2613,
  "民" => -1694,
  "決" => -1073,
  "法" => 1868,
  "海" => -495,
  "無" => 979,
  "物" => 461,
  "特" => -3850,
  "生" => -273,
  "用" => 914,
  "町" => 1215,
  "的" => 7313,
  "直" => -1835,
  "省" => 792,
  "県" => 6293,
  "知" => -1528,
  "私" => 4231,
  "税" => 401,
  "立" => -960,
  "第" => 1201,
  "米" => 7767,
  "系" => 3066,
  "約" => 3663,
  "級" => 1384,
  "統" => -4229,
  "総" => 1163,
  "線" => 1255,
  "者" => 6457,
  "能" => 725,
  "自" => -2869,
  "英" => 785,
  "見" => 1044,
  "調" => -562,
  "財" => -733,
  "費" => 1777,
  "車" => 1835,
  "軍" => 1375,
  "込" => -1504,
  "通" => -1136,
  "選" => -681,
  "郎" => 1026,
  "郡" => 4404,
  "部" => 1200,
  "金" => 2163,
  "長" => 421,
  "開" => -1432,
  "間" => 1302,
  "関" => -1282,
  "雨" => 2009,
  "電" => -1045,
  "非" => 2066,
  "駅" => 1620,
  "１" => -800,
  "｣" => 2670,
  "･" => -3794,
  "ｯ" => -1350,
  "ｱ" => 551,
  "ｸﾞ" => 1319,
  "ｽ" => 874,
  "ﾄ" => 521,
  "ﾑ" => 1109,
  "ﾙ" => 1591,
  "ﾛ" => 2201,
  "ﾝ" => 278,
}

UW4 =

{
  "," => 3930,
  "." => 3508,
  "―" => -4841,
  "、" => 3930,
  "。" => 3508,
  "〇" => 4999,
  "「" => 1895,
  "」" => 3798,
  "〓" => -5156,
  "あ" => 4752,
  "い" => -3435,
  "う" => -640,
  "え" => -2514,
  "お" => 2405,
  "か" => 530,
  "が" => 6006,
  "き" => -4482,
  "ぎ" => -3821,
  "く" => -3788,
  "け" => -4376,
  "げ" => -4734,
  "こ" => 2255,
  "ご" => 1979,
  "さ" => 2864,
  "し" => -843,
  "じ" => -2506,
  "す" => -731,
  "ず" => 1251,
  "せ" => 181,
  "そ" => 4091,
  "た" => 5034,
  "だ" => 5408,
  "ち" => -3654,
  "っ" => -5882,
  "つ" => -1659,
  "て" => 3994,
  "で" => 7410,
  "と" => 4547,
  "な" => 5433,
  "に" => 6499,
  "ぬ" => 1853,
  "ね" => 1413,
  "の" => 7396,
  "は" => 8578,
  "ば" => 1940,
  "ひ" => 4249,
  "び" => -4134,
  "ふ" => 1345,
  "へ" => 6665,
  "べ" => -744,
  "ほ" => 1464,
  "ま" => 1051,
  "み" => -2082,
  "む" => -882,
  "め" => -5046,
  "も" => 4169,
  "ゃ" => -2666,
  "や" => 2795,
  "ょ" => -1544,
  "よ" => 3351,
  "ら" => -2922,
  "り" => -9726,
  "る" => -14_896,
  "れ" => -2613,
  "ろ" => -4570,
  "わ" => -1783,
  "を" => 13_150,
  "ん" => -2352,
  "カ" => 2145,
  "コ" => 1789,
  "セ" => 1287,
  "ッ" => -724,
  "ト" => -403,
  "メ" => -1635,
  "ラ" => -881,
  "リ" => -541,
  "ル" => -856,
  "ン" => -3637,
  "・" => -4371,
  "ー" => -11_870,
  "一" => -2069,
  "中" => 2210,
  "予" => 782,
  "事" => -190,
  "井" => -1768,
  "人" => 1036,
  "以" => 544,
  "会" => 950,
  "体" => -1286,
  "作" => 530,
  "側" => 4292,
  "先" => 601,
  "党" => -2006,
  "共" => -1212,
  "内" => 584,
  "円" => 788,
  "初" => 1347,
  "前" => 1623,
  "副" => 3879,
  "力" => -302,
  "動" => -740,
  "務" => -2715,
  "化" => 776,
  "区" => 4517,
  "協" => 1013,
  "参" => 1555,
  "合" => -1834,
  "和" => -681,
  "員" => -910,
  "器" => -851,
  "回" => 1500,
  "国" => -619,
  "園" => -1200,
  "地" => 866,
  "場" => -1410,
  "塁" => -2094,
  "士" => -1413,
  "多" => 1067,
  "大" => 571,
  "子" => -4802,
  "学" => -1397,
  "定" => -1057,
  "寺" => -809,
  "小" => 1910,
  "屋" => -1328,
  "山" => -1500,
  "島" => -2056,
  "川" => -2667,
  "市" => 2771,
  "年" => 374,
  "庁" => -4556,
  "後" => 456,
  "性" => 553,
  "感" => 916,
  "所" => -1566,
  "支" => 856,
  "改" => 787,
  "政" => 2182,
  "教" => 704,
  "文" => 522,
  "方" => -856,
  "日" => 1798,
  "時" => 1829,
  "最" => 845,
  "月" => -9066,
  "木" => -485,
  "来" => -442,
  "校" => -360,
  "業" => -1043,
  "氏" => 5388,
  "民" => -2716,
  "気" => -910,
  "沢" => -939,
  "済" => -543,
  "物" => -735,
  "率" => 672,
  "球" => -1267,
  "生" => -1286,
  "産" => -1101,
  "田" => -2900,
  "町" => 1826,
  "的" => 2586,
  "目" => 922,
  "省" => -3485,
  "県" => 2997,
  "空" => -867,
  "立" => -2112,
  "第" => 788,
  "米" => 2937,
  "系" => 786,
  "約" => 2171,
  "経" => 1146,
  "統" => -1169,
  "総" => 940,
  "線" => -994,
  "署" => 749,
  "者" => 2145,
  "能" => -730,
  "般" => -852,
  "行" => -792,
  "規" => 792,
  "警" => -1184,
  "議" => -244,
  "谷" => -1000,
  "賞" => 730,
  "車" => -1481,
  "軍" => 1158,
  "輪" => -1433,
  "込" => -3370,
  "近" => 929,
  "道" => -1291,
  "選" => 2596,
  "郎" => -4866,
  "都" => 1192,
  "野" => -1100,
  "銀" => -2213,
  "長" => 357,
  "間" => -2344,
  "院" => -2297,
  "際" => -2604,
  "電" => -878,
  "領" => -1659,
  "題" => -792,
  "館" => -1984,
  "首" => 1749,
  "高" => 2120,
  "｢" => 1895,
  "｣" => 3798,
  "･" => -4371,
  "ｯ" => -724,
  "ｰ" => -11_870,
  "ｶ" => 2145,
  "ｺ" => 1789,
  "ｾ" => 1287,
  "ﾄ" => -403,
  "ﾒ" => -1635,
  "ﾗ" => -881,
  "ﾘ" => -541,
  "ﾙ" => -856,
  "ﾝ" => -3637,
}

UW5 =

{
  "," => 465,
  "." => -299,
  "1" => -514,
  "E2" => -32_768,
  "]" => -2762,
  "、" => 465,
  "。" => -299,
  "「" => 363,
  "あ" => 1655,
  "い" => 331,
  "う" => -503,
  "え" => 1199,
  "お" => 527,
  "か" => 647,
  "が" => -421,
  "き" => 1624,
  "ぎ" => 1971,
  "く" => 312,
  "げ" => -983,
  "さ" => -1537,
  "し" => -1371,
  "す" => -852,
  "だ" => -1186,
  "ち" => 1093,
  "っ" => 52,
  "つ" => 921,
  "て" => -18,
  "で" => -850,
  "と" => -127,
  "ど" => 1682,
  "な" => -787,
  "に" => -1224,
  "の" => -635,
  "は" => -578,
  "べ" => 1001,
  "み" => 502,
  "め" => 865,
  "ゃ" => 3350,
  "ょ" => 854,
  "り" => -208,
  "る" => 429,
  "れ" => 504,
  "わ" => 419,
  "を" => -1264,
  "ん" => 327,
  "イ" => 241,
  "ル" => 451,
  "ン" => -343,
  "中" => -871,
  "京" => 722,
  "会" => -1153,
  "党" => -654,
  "務" => 3519,
  "区" => -901,
  "告" => 848,
  "員" => 2104,
  "大" => -1296,
  "学" => -548,
  "定" => 1785,
  "嵐" => -1304,
  "市" => -2991,
  "席" => 921,
  "年" => 1763,
  "思" => 872,
  "所" => -814,
  "挙" => 1618,
  "新" => -1682,
  "日" => 218,
  "月" => -4353,
  "査" => 932,
  "格" => 1356,
  "機" => -1508,
  "氏" => -1347,
  "田" => 240,
  "町" => -3912,
  "的" => -3149,
  "相" => 1319,
  "省" => -1052,
  "県" => -4003,
  "研" => -997,
  "社" => -278,
  "空" => -813,
  "統" => 1955,
  "者" => -2233,
  "表" => 663,
  "語" => -1073,
  "議" => 1219,
  "選" => -1018,
  "郎" => -368,
  "長" => 786,
  "間" => 1191,
  "題" => 2368,
  "館" => -689,
  "１" => -514,
  "Ｅ２" => -32_768,
  "｢" => 363,
  "ｲ" => 241,
  "ﾙ" => 451,
  "ﾝ" => -343,
}

UW6 =

{
  "," => 227,
  "." => 808,
  "1" => -270,
  "E1" => 306,
  "、" => 227,
  "。" => 808,
  "あ" => -307,
  "う" => 189,
  "か" => 241,
  "が" => -73,
  "く" => -121,
  "こ" => -200,
  "じ" => 1782,
  "す" => 383,
  "た" => -428,
  "っ" => 573,
  "て" => -1014,
  "で" => 101,
  "と" => -105,
  "な" => -253,
  "に" => -149,
  "の" => -417,
  "は" => -236,
  "も" => -206,
  "り" => 187,
  "る" => -135,
  "を" => 195,
  "ル" => -673,
  "ン" => -496,
  "一" => -277,
  "中" => 201,
  "件" => -800,
  "会" => 624,
  "前" => 302,
  "区" => 1792,
  "員" => -1212,
  "委" => 798,
  "学" => -960,
  "市" => 887,
  "広" => -695,
  "後" => 535,
  "業" => -697,
  "相" => 753,
  "社" => -507,
  "福" => 974,
  "空" => -822,
  "者" => 1811,
  "連" => 463,
  "郎" => 1082,
  "１" => -270,
  "Ｅ１" => 306,
  "ﾙ" => -673,
  "ﾝ" => -496,
}

Class Method Summary

• .segment(text) ⇒ Object

Class Method Details

.segment(text) ⇒ Object

# File 'lib/tiny_japanese_segmenter.rb', line 1385

def segment(text)
  return [] if text.nil? || text.strip.length == 0

  result = []

  segments = %w[B3 B2 B1]
  ctypes = %w[O O O]

  text.chars.each do |char|
    segments << char
    ctypes << ctype(char)
  end

  segments.concat(%w[E1 E2 E3])
  ctypes.concat(%w[O O O])

  word = segments[3]
  p1 = "U"
  p2 = "U"
  p3 = "U"

  4.upto(segments.size - 4) do |i|
    score = BIAS
    w1 = segments[i - 3]
    w2 = segments[i - 2]
    w3 = segments[i - 1]
    w4 = segments[i]
    w5 = segments[i + 1]
    w6 = segments[i + 2]
    c1 = ctypes[i - 3]
    c2 = ctypes[i - 2]
    c3 = ctypes[i - 1]
    c4 = ctypes[i]
    c5 = ctypes[i + 1]
    c6 = ctypes[i + 2]
    score += UP1[p1].to_i
    score += UP2[p2].to_i
    score += UP3[p3].to_i
    score += BP1[p1 + p2].to_i
    score += BP2[p2 + p3].to_i
    score += UW1[w1].to_i
    score += UW2[w2].to_i
    score += UW3[w3].to_i
    score += UW4[w4].to_i
    score += UW5[w5].to_i
    score += UW6[w6].to_i
    score += BW1[w2 + w3].to_i
    score += BW2[w3 + w4].to_i
    score += BW3[w4 + w5].to_i
    score += TW1[w1 + w2 + w3].to_i
    score += TW2[w2 + w3 + w4].to_i
    score += TW3[w3 + w4 + w5].to_i
    score += TW4[w4 + w5 + w6].to_i
    score += UC1[c1].to_i
    score += UC2[c2].to_i
    score += UC3[c3].to_i
    score += UC4[c4].to_i
    score += UC5[c5].to_i
    score += UC6[c6].to_i
    score += BC1[c2 + c3].to_i
    score += BC2[c3 + c4].to_i
    score += BC3[c4 + c5].to_i
    score += TC1[c1 + c2 + c3].to_i
    score += TC2[c2 + c3 + c4].to_i
    score += TC3[c3 + c4 + c5].to_i
    score += TC4[c4 + c5 + c6].to_i
    # score += TC5[c4 + c5 + c6].to_i
    score += UQ1[p1 + c1].to_i
    score += UQ2[p2 + c2].to_i
    score += UQ3[p3 + c3].to_i
    score += BQ1[p2 + c2 + c3].to_i
    score += BQ2[p2 + c3 + c4].to_i
    score += BQ3[p3 + c2 + c3].to_i
    score += BQ4[p3 + c3 + c4].to_i
    score += TQ1[p2 + c1 + c2 + c3].to_i
    score += TQ2[p2 + c2 + c3 + c4].to_i
    score += TQ3[p3 + c1 + c2 + c3].to_i
    score += TQ4[p3 + c2 + c3 + c4].to_i

    p = "O"

    if score > 0
      result.push(word)
      word = ""
      p = "B"
    end

    p1 = p2
    p2 = p3
    p3 = p
    word += segments[i]
  end

  result.push(word)

  result
end