# coding: utf-8 # Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp) require 'unicode_normalize/tables.rb' module UnicodeNormalize ## Constant for max hash capacity to avoid DoS attack MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow ## Regular Expressions and Hash Constants REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED) REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED) REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED) NF_HASH_D = Hash.new do |hash, key| hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack hash[key] = nfd_one(key) end NF_HASH_C = Hash.new do |hash, key| hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack hash[key] = nfc_one(key) end NF_HASH_K = Hash.new do |hash, key| hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack hash[key] = nfkd_one(key) end ## Constants For Hangul SBASE = 0xAC00 LBASE = 0x1100 VBASE = 0x1161 TBASE = 0x11A7 LCOUNT = 19 VCOUNT = 21 TCOUNT = 28 NCOUNT = VCOUNT * TCOUNT SCOUNT = LCOUNT * NCOUNT # Unicode-based encodings (except UTF-8) UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE, Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE] ## Hangul Algorithm def self.hangul_decomp_one(target) sIndex = target.ord - SBASE return target if sIndex < 0 || sIndex >= SCOUNT l = LBASE + sIndex / NCOUNT v = VBASE + (sIndex % NCOUNT) / TCOUNT t = TBASE + sIndex % TCOUNT (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1] end def self.hangul_comp_one(string) length = string.length if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead < LCOUNT and 0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1] else lead_vowel.chr(Encoding::UTF_8) + string[2..-1] end else string end end ## Canonical Ordering def self.canonical_ordering_one(string) sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] } (sorting.length-2).downto(0) do |i| # bubble sort (0..i).each do |j| later_class = sorting[j+1].last if 0