DrupalAPI.de

1. drupal
   1. drupal6
   2. drupal7

Basic CJK tokenizer. Simply splits a string into consecutive, overlapping sequences of characters ('minimum_word_size' long).

Code

modules/search/search.module, line 343

<?php
function search_expand_cjk($matches) {
  $min = variable_get('minimum_word_size', 3);
  $str = $matches[0];
  $l = drupal_strlen($str);
  // Passthrough short words
  if ($l <= $min) {
    return ' ' . $str . ' ';
  }
  $tokens = ' ';
  // FIFO queue of characters
  $chars = array();
  // Begin loop
  for ($i = 0; $i < $l; ++$i) {
    // Grab next character
    $current = drupal_substr($str, 0, 1);
    $str = substr($str, strlen($current));
    $chars[] = $current;
    if ($i >= $min - 1) {
      $tokens .= implode('', $chars) . ' ';
      array_shift($chars);
    }
  }
  return $tokens;
}
?>