Unicode Basics

# Unicode code points
char = "A"
puts char.ord           # => 65 (Unicode code point)
puts 65.chr(Encoding::UTF_8)  # => "A"

# Multi-byte characters
japanese = "こ"
puts japanese.ord       # => 12371
puts 12371.chr(Encoding::UTF_8)  # => "こ"

# Emoji and special characters
emoji = "🌟"
puts emoji.ord          # => 127775
puts emoji.bytesize     # => 4 (bytes)
puts emoji.length       # => 1 (character)

# Unicode escape sequences
unicode_str = "\u3053\u3093\u306b\u3061\u306f"  # こんにちは
puts unicode_str        # => "こんにちは"

# Hexadecimal Unicode literals
hex_unicode = "\u{1F31F}"  # 🌟
puts hex_unicode        # => "🌟"

Normalization Forms

require 'unicode_normalize'

# Different ways to represent the same character
# é can be: single character (é) or e + combining accent (e + ́)
single_char = "\u00e9"      # é (single character)
composed = "\u0065\u0301"   # e + ́ (base + combining)

puts single_char == composed  # => false (different byte representation)
puts single_char.bytesize    # => 2
puts composed.bytesize       # => 3

# Normalize to compare correctly
puts single_char.unicode_normalize(:nfc) == composed.unicode_normalize(:nfc)  # => true

# Different normalization forms
text = "José"

# NFC (Canonical Decomposition, then Canonical Composition)
nfc = text.unicode_normalize(:nfc)
puts "NFC: #{nfc.inspect}"

# NFD (Canonical Decomposition)
nfd = text.unicode_normalize(:nfd)
puts "NFD: #{nfd.inspect}"

# NFKC (Compatibility Decomposition, then Canonical Composition)
nfkc = text.unicode_normalize(:nfkc)
puts "NFKC: #{nfkc.inspect}"

# NFKD (Compatibility Decomposition)
nfkd = text.unicode_normalize(:nfkd)
puts "NFKD: #{nfkd.inspect}"

# Check if string is normalized
puts text.unicode_normalized?(:nfc)  # Check if already in NFC form

Safe Text Comparison

# Create a Unicode-safe comparison method
def unicode_compare(str1, str2, case_sensitive: true)
  # Normalize both strings
  norm1 = str1.unicode_normalize(:nfc)
  norm2 = str2.unicode_normalize(:nfc)

  unless case_sensitive
    norm1 = norm1.downcase
    norm2 = norm2.downcase
  end

  norm1 == norm2
end

# Test with different representations
text1 = "café"           # é as single character
text2 = "cafe\u0301"     # é as e + combining accent

puts unicode_compare(text1, text2)  # => true
puts unicode_compare("CAFÉ", "café", case_sensitive: false)  # => true

# Case conversion with Unicode
mixed_text = "İstanbul"  # Turkish capital İ
puts mixed_text.downcase            # => "i̇stanbul" (not correct for Turkish)
puts mixed_text.downcase(:turkic)   # => "istanbul" (correct for Turkish)

# Unicode-aware string operations
class UnicodeString
  def initialize(str)
    @str = str.unicode_normalize(:nfc)
  end

  def caseless_include?(substring)
    normalized_str = @str.downcase
    normalized_substring = substring.unicode_normalize(:nfc).downcase
    normalized_str.include?(normalized_substring)
  end

  def character_count
    @str.length  # Returns number of Unicode characters
  end

  def byte_count
    @str.bytesize  # Returns number of bytes
  end

  def to_s
    @str
  end
end

# Example usage
unicode_str = UnicodeString.new("Hello 世界 🌟")
puts unicode_str.character_count  # => 9
puts unicode_str.byte_count      # => 15

Simple I18n Implementation

# Simple internationalization class
class SimpleI18n
  def initialize
    @translations = {}
    @current_locale = :en
  end

  def load_translations(locale, translations)
    @translations[locale] = translations
  end

  def locale=(locale)
    @current_locale = locale
  end

  def t(key, **options)
    translation = dig_translation(@current_locale, key)
    return key.to_s unless translation

    interpolate(translation, options)
  end

  private

  def dig_translation(locale, key)
    keys = key.to_s.split('.')
    keys.reduce(@translations[locale]) do |hash, k|
      hash&.dig(k.to_sym)
    end
  end

  def interpolate(text, options)
    options.reduce(text) do |result, (key, value)|
      result.gsub("%{#{key}}", value.to_s)
    end
  end
end

# Usage example
i18n = SimpleI18n.new

# Load English translations
i18n.load_translations(:en, {
  greeting: "Hello, %{name}!",
  messages: {
    welcome: "Welcome to our application",
    goodbye: "See you later"
  }
})

# Load Japanese translations
i18n.load_translations(:ja, {
  greeting: "こんにちは、%{name}さん！",
  messages: {
    welcome: "アプリケーションへようこそ",
    goodbye: "また後で"
  }
})

# Use translations
i18n.locale = :en
puts i18n.t(:greeting, name: "Alice")  # => "Hello, Alice!"
puts i18n.t('messages.welcome')        # => "Welcome to our application"

i18n.locale = :ja
puts i18n.t(:greeting, name: "Alice")  # => "こんにちは、Aliceさん！"
puts i18n.t('messages.welcome')        # => "アプリケーションへようこそ"

Language-Specific Operations

# Collation (sorting) by locale
class LocaleAwareSort
  def self.sort_by_locale(strings, locale = :en)
    case locale
    when :en
      strings.sort
    when :ja
      # For Japanese, might want to sort by reading (hiragana)
      strings.sort_by { |s| s.tr('ァ-ヾ', 'ぁ-ゞ') }  # Katakana to Hiragana
    when :de
      # German ß sorting
      strings.sort_by { |s| s.gsub('ß', 'ss') }
    else
      strings.sort
    end
  end
end

# Text direction handling
class TextDirection
  RTL_SCRIPTS = %w[Arab Hebr Thaa Nkoo Syrc].freeze

  def self.direction(text)
    # Simple script detection (in real apps, use proper Unicode script detection)
    return :rtl if text.match?(/[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]/)  # Arabic
    return :rtl if text.match?(/[\u0590-\u05FF]/)  # Hebrew
    :ltr
  end

  def self.display_text(text)
    direction = direction(text)
    if direction == :rtl
      "#{text} ← RTL"
    else
      "LTR → #{text}"
    end
  end
end

# Example usage
arabic_text = "مرحبا"
english_text = "Hello"

puts TextDirection.display_text(arabic_text)  # => "مرحبا ← RTL"
puts TextDirection.display_text(english_text) # => "LTR → Hello"

# Pluralization rules
class Pluralization
  RULES = {
    en: ->(n) { n == 1 ? :one : :other },
    ja: ->(n) { :other },  # Japanese doesn't have plural forms
    ru: ->(n) {
      case n % 10
      when 1
        n % 100 == 11 ? :many : :one
      when 2..4
        n % 100 == 12..14 ? :many : :few
      else
        :many
      end
    }
  }.freeze

  def self.pluralize(count, locale, translations)
    rule = RULES[locale] || RULES[:en]
    form = rule.call(count)
    translations[form] || translations[:other]
  end
end

# Usage
count = 5
puts Pluralization.pluralize(count, :en, {
  one: "%{count} item",
  other: "%{count} items"
}) % { count: count }  # => "5 items"

Multi-Script Text Processing

# Script detection and processing
class ScriptProcessor
  # Unicode script ranges (simplified)
  SCRIPT_RANGES = {
    latin: /[A-Za-z\u00C0-\u017F\u1E00-\u1EFF]/,
    cyrillic: /[\u0400-\u04FF]/,
    arabic: /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]/,
    chinese: /[\u4E00-\u9FFF]/,
    japanese_hiragana: /[\u3040-\u309F]/,
    japanese_katakana: /[\u30A0-\u30FF]/,
    korean: /[\uAC00-\uD7AF]/
  }.freeze

  def self.detect_scripts(text)
    scripts = []
    SCRIPT_RANGES.each do |script, pattern|
      scripts << script if text.match?(pattern)
    end
    scripts
  end

  def self.split_by_script(text)
    segments = []
    current_segment = ""
    current_script = nil

    text.each_char do |char|
      char_scripts = detect_scripts(char)
      main_script = char_scripts.first

      if main_script != current_script && !current_segment.empty?
        segments << { text: current_segment, script: current_script }
        current_segment = ""
      end

      current_segment += char
      current_script = main_script
    end

    segments << { text: current_segment, script: current_script } unless current_segment.empty?
    segments
  end
end

# Example usage
mixed_text = "Hello こんにちは 世界 مرحبا World!"
scripts = ScriptProcessor.detect_scripts(mixed_text)
puts "Detected scripts: #{scripts}"
# => Detected scripts: [:latin, :japanese_hiragana, :chinese, :arabic]

segments = ScriptProcessor.split_by_script(mixed_text)
segments.each do |segment|
  puts "#{segment[:script]}: #{segment[:text]}"
end

# Font fallback system
class FontManager
  FONT_MAPPINGS = {
    latin: "Arial, Helvetica, sans-serif",
    chinese: "SimSun, serif",
    japanese_hiragana: "Hiragino Sans, serif",
    japanese_katakana: "Hiragino Sans, serif",
    arabic: "Tahoma, serif",
    korean: "Malgun Gothic, serif"
  }.freeze

  def self.font_for_script(script)
    FONT_MAPPINGS[script] || FONT_MAPPINGS[:latin]
  end

  def self.css_for_text(text)
    scripts = ScriptProcessor.detect_scripts(text)
    primary_script = scripts.first || :latin

    font_family = font_for_script(primary_script)
    direction = text.match?(/[\u0600-\u06FF\u0590-\u05FF]/) ? "rtl" : "ltr"

    "font-family: #{font_family}; direction: #{direction};"
  end
end

Unicode Property Handling

# Grapheme cluster handling (what users see as "characters")
def count_graphemes(text)
  # In Ruby 2.5+, String#each_grapheme_cluster is available
  if text.respond_to?(:each_grapheme_cluster)
    text.each_grapheme_cluster.count
  else
    # Fallback for older Ruby versions
    text.length
  end
end

# Example with combining characters
text_with_combining = "e\u0301"  # e + ́ = é
puts text_with_combining.length         # => 2 (code points)
puts count_graphemes(text_with_combining)  # => 1 (what user sees)

# Emoji with skin tone modifiers
emoji_with_modifier = "👋🏽"  # Waving hand + medium skin tone
puts emoji_with_modifier.length                    # => 2 (code points)
puts count_graphemes(emoji_with_modifier)          # => 1 (what user sees)

# Text width calculation for display
class DisplayWidth
  def self.calculate_width(text)
    width = 0
    text.each_char do |char|
      case char.ord
      when 0x0000..0x001F, 0x007F..0x009F
        # Control characters - no width
        width += 0
      when 0x1100..0x115F, 0x2329..0x232A, 0x2E80..0xA4CF, 0xAC00..0xD7A3, 0xF900..0xFAFF, 0xFE10..0xFE19, 0xFE30..0xFE6F, 0xFF00..0xFF60, 0xFFE0..0xFFE6, 0x20000..0x2FFFD, 0x30000..0x3FFFD
        # Wide characters (CJK, etc.) - 2 units width
        width += 2
      else
        # Regular characters - 1 unit width
        width += 1
      end
    end
    width
  end
end

# Example usage
puts DisplayWidth.calculate_width("Hello")     # => 5
puts DisplayWidth.calculate_width("こんにちは")  # => 10
puts DisplayWidth.calculate_width("Hi 世界")   # => 7

# Bidirectional text handling (simplified)
class BidiText
  def self.process(text)
    # This is a very simplified version
    # Real bidi processing requires the Unicode Bidirectional Algorithm
    segments = []
    current_segment = ""
    current_direction = detect_direction(text[0])

    text.each_char do |char|
      char_direction = detect_direction(char)

      if char_direction != current_direction && !current_segment.empty?
        segments << { text: current_segment, direction: current_direction }
        current_segment = ""
      end

      current_segment += char
      current_direction = char_direction
    end

    segments << { text: current_segment, direction: current_direction } unless current_segment.empty?
    segments
  end

  private

  def self.detect_direction(char)
    case char.ord
    when 0x0590..0x05FF, 0x0600..0x06FF, 0x0750..0x077F, 0x08A0..0x08FF
      :rtl
    else
      :ltr
    end
  end
end

Unicode & I18n Best Practices

• Always normalize Unicode text before comparison or storage
• Use NFC normalization for most applications (canonical composed form)
• Consider locale when implementing sorting, case conversion, and text processing
• Test with real international data early and often
• Separate content from code - externalize all user-facing strings
• Design for text expansion - translations can be 30-50% longer
• Handle RTL languages and bidirectional text properly
• Use proper fonts that support the scripts in your application