From 31b40448fd4a08032f562ce171a66c6952370f60 Mon Sep 17 00:00:00 2001 From: tschuettler Date: Thu, 2 Aug 2018 17:23:14 +0200 Subject: Update af_lang_detect plugin with changes from upstream Reapplied downstream changes for phpmd ruleset --- plugins/af_lang_detect/init.php | 4 +- .../languagedetect/LanguageDetect.php | 1708 -------------------- .../languagedetect/Text/LanguageDetect.php | 1678 +++++++++++++++++++ .../Text/LanguageDetect/Exception.php | 24 + .../languagedetect/Text/LanguageDetect/ISO639.php | 3 +- .../languagedetect/Text/LanguageDetect/Parser.php | 189 +-- .../languagedetect/data/build-unicode_blocks.php | 7 + .../languagedetect/data/unicode_blocks.dat | 2 +- .../languagedetect/data/unicode_blocks.php | 873 ++++++++++ 9 files changed, 2685 insertions(+), 1803 deletions(-) delete mode 100644 plugins/af_lang_detect/languagedetect/LanguageDetect.php create mode 100644 plugins/af_lang_detect/languagedetect/Text/LanguageDetect.php create mode 100644 plugins/af_lang_detect/languagedetect/data/build-unicode_blocks.php create mode 100644 plugins/af_lang_detect/languagedetect/data/unicode_blocks.php diff --git a/plugins/af_lang_detect/init.php b/plugins/af_lang_detect/init.php index e78741319..3ec0023b6 100644 --- a/plugins/af_lang_detect/init.php +++ b/plugins/af_lang_detect/init.php @@ -4,7 +4,7 @@ class Af_Lang_Detect extends Plugin { private $lang; function about() { - return array(1.0, + return array(1.1, "Detect article language", "fox"); } @@ -14,7 +14,7 @@ class Af_Lang_Detect extends Plugin { $host->add_hook($host::HOOK_ARTICLE_FILTER, $this); - require_once __DIR__ . "/languagedetect/LanguageDetect.php"; + require_once __DIR__ . "/languagedetect/Text/LanguageDetect.php"; $this->lang = new Text_LanguageDetect(); $this->lang->setNameMode(2); diff --git a/plugins/af_lang_detect/languagedetect/LanguageDetect.php b/plugins/af_lang_detect/languagedetect/LanguageDetect.php deleted file mode 100644 index 6c1863508..000000000 --- a/plugins/af_lang_detect/languagedetect/LanguageDetect.php +++ /dev/null @@ -1,1708 +0,0 @@ - - * @copyright 2005-2006 Nicholas Pisarro - * @license http://www.debian.org/misc/bsd.license BSD - * @version SVN: $Id: LanguageDetect.php 322353 2012-01-16 08:41:43Z cweiske $ - * @link http://pear.php.net/package/Text_LanguageDetect/ - * @link http://langdetect.blogspot.com/ - */ - -require_once __DIR__ . '/Text/LanguageDetect/Exception.php'; -require_once __DIR__ . '/Text/LanguageDetect/Parser.php'; -require_once __DIR__ . '/Text/LanguageDetect/ISO639.php'; - -/** - * Language detection class - * - * Requires the langauge model database (lang.dat) that should have - * accompanied this class definition in order to be instantiated. - * - * Example usage: - * - * - * require_once 'Text/LanguageDetect.php'; - * - * $l = new Text_LanguageDetect; - * - * $stdin = fopen('php://stdin', 'r'); - * - * echo "Supported languages:\n"; - * - * try { - * $langs = $l->getLanguages(); - * } catch (Text_LanguageDetect_Exception $e) { - * die($e->getMessage()); - * } - * - * sort($langs); - * echo join(', ', $langs); - * - * while ($line = fgets($stdin)) { - * print_r($l->detect($line, 4)); - * } - * - * - * @category Text - * @package Text_LanguageDetect - * @author Nicholas Pisarro - * @copyright 2005 Nicholas Pisarro - * @license http://www.debian.org/misc/bsd.license BSD - * @version Release: @package_version@ - * @link http://pear.php.net/package/Text_LanguageDetect/ - * @todo allow users to generate their own language models - * - * @SuppressWarnings(PHPMD) - */ -class Text_LanguageDetect -{ - /** - * The filename that stores the trigram data for the detector - * - * If this value starts with a slash (/) or a dot (.) the value of - * $this->_data_dir will be ignored - * - * @var string - * @access private - */ - var $_db_filename = 'lang.dat'; - - /** - * The filename that stores the unicode block definitions - * - * If this value starts with a slash (/) or a dot (.) the value of - * $this->_data_dir will be ignored - * - * @var string - * @access private - */ - var $_unicode_db_filename = 'unicode_blocks.dat'; - - /** - * The data directory - * - * Should be set by PEAR installer - * - * @var string - * @access private - */ - var $_data_dir = '@data_dir@'; - - /** - * The trigram data for comparison - * - * Will be loaded on start from $this->_db_filename - * - * @var array - * @access private - */ - var $_lang_db = array(); - - /** - * stores the map of the trigram data to unicode characters - * - * @access private - * @var array - */ - var $_unicode_map; - - /** - * The size of the trigram data arrays - * - * @var int - * @access private - */ - var $_threshold = 300; - - /** - * the maximum possible score. - * - * needed for score normalization. Different depending on the - * perl compatibility setting - * - * @access private - * @var int - * @see setPerlCompatible() - */ - var $_max_score = 0; - - /** - * Whether or not to simulate perl's Language::Guess exactly - * - * @access private - * @var bool - * @see setPerlCompatible() - */ - var $_perl_compatible = false; - - /** - * Whether to use the unicode block detection to speed up processing - * - * @access private - * @var bool - */ - var $_use_unicode_narrowing = true; - - /** - * stores the result of the clustering operation - * - * @access private - * @var array - * @see clusterLanguages() - */ - var $_clusters; - - /** - * Which type of "language names" are accepted and returned: - * - * 0 - language name ("english") - * 2 - 2-letter ISO 639-1 code ("en") - * 3 - 3-letter ISO 639-2 code ("eng") - */ - var $_name_mode = 0; - - /** - * Constructor - * - * Will attempt to load the language database. If it fails, you will get - * an exception. - */ - function __construct() - { - $data = $this->_readdb($this->_db_filename); - $this->_checkTrigram($data['trigram']); - $this->_lang_db = $data['trigram']; - - if (isset($data['trigram-unicodemap'])) { - $this->_unicode_map = $data['trigram-unicodemap']; - } - - // Not yet implemented: - if (isset($data['trigram-clusters'])) { - $this->_clusters = $data['trigram-clusters']; - } - } - - /** - * Returns the path to the location of the database - * - * @param string $fname File name to load - * - * @return string expected path to the language model database - * @access private - */ - function _get_data_loc($fname) - { - if ($fname{0} == '/' || $fname{0} == '.') { - // if filename starts with a slash, assume it's an absolute pathname - // and skip whatever is in $this->_data_dir - return $fname; - - } elseif ($this->_data_dir != '@' . 'data_dir' . '@') { - // if the data dir was set by the PEAR installer, use that - return $this->_data_dir . '/Text_LanguageDetect/' . $fname; - - } else { - // assume this was just unpacked somewhere - // try the local working directory if otherwise - return __DIR__ . '/data/' . $fname; - } - } - - /** - * Loads the language trigram database from filename - * - * Trigram datbase should be a serialize()'d array - * - * @param string $fname the filename where the data is stored - * - * @return array the language model data - * @throws Text_LanguageDetect_Exception - * @access private - */ - function _readdb($fname) - { - // finds the correct data dir - $fname = $this->_get_data_loc($fname); - - // input check - if (!file_exists($fname)) { - throw new Text_LanguageDetect_Exception( - 'Language database does not exist: ' . $fname, - Text_LanguageDetect_Exception::DB_NOT_FOUND - ); - } elseif (!is_readable($fname)) { - throw new Text_LanguageDetect_Exception( - 'Language database is not readable: ' . $fname, - Text_LanguageDetect_Exception::DB_NOT_READABLE - ); - } - - return unserialize(file_get_contents($fname)); - } - - - /** - * Checks if this object is ready to detect languages - * - * @param array $trigram Trigram data from database - * - * @return void - * @access private - */ - function _checkTrigram($trigram) - { - if (!is_array($trigram)) { - if (ini_get('magic_quotes_runtime')) { - throw new Text_LanguageDetect_Exception( - 'Error loading database. Try turning magic_quotes_runtime off.', - Text_LanguageDetect_Exception::MAGIC_QUOTES - ); - } - throw new Text_LanguageDetect_Exception( - 'Language database is not an array.', - Text_LanguageDetect_Exception::DB_NOT_ARRAY - ); - } elseif (empty($trigram)) { - throw new Text_LanguageDetect_Exception( - 'Language database has no elements.', - Text_LanguageDetect_Exception::DB_EMPTY - ); - } - } - - /** - * Omits languages - * - * Pass this function the name of or an array of names of - * languages that you don't want considered - * - * If you're only expecting a limited set of languages, this can greatly - * speed up processing - * - * @param mixed $omit_list language name or array of names to omit - * @param bool $include_only if true will include (rather than - * exclude) only those in the list - * - * @return int number of languages successfully deleted - * @throws Text_LanguageDetect_Exception - */ - public function omitLanguages($omit_list, $include_only = false) - { - $deleted = 0; - - $omit_list = $this->_convertFromNameMode($omit_list); - - if (!$include_only) { - // deleting the given languages - if (!is_array($omit_list)) { - $omit_list = strtolower($omit_list); // case desensitize - if (isset($this->_lang_db[$omit_list])) { - unset($this->_lang_db[$omit_list]); - $deleted++; - } - } else { - foreach ($omit_list as $omit_lang) { - if (isset($this->_lang_db[$omit_lang])) { - unset($this->_lang_db[$omit_lang]); - $deleted++; - } - } - } - - } else { - // deleting all except the given languages - if (!is_array($omit_list)) { - $omit_list = array($omit_list); - } - - // case desensitize - foreach ($omit_list as $key => $omit_lang) { - $omit_list[$key] = strtolower($omit_lang); - } - - foreach (array_keys($this->_lang_db) as $lang) { - if (!in_array($lang, $omit_list)) { - unset($this->_lang_db[$lang]); - $deleted++; - } - } - } - - // reset the cluster cache if the number of languages changes - // this will then have to be recalculated - if (isset($this->_clusters) && $deleted > 0) { - $this->_clusters = null; - } - - return $deleted; - } - - - /** - * Returns the number of languages that this object can detect - * - * @access public - * @return int the number of languages - * @throws Text_LanguageDetect_Exception - */ - function getLanguageCount() - { - return count($this->_lang_db); - } - - /** - * Checks if the language with the given name exists in the database - * - * @param mixed $lang Language name or array of language names - * - * @return bool true if language model exists - */ - public function languageExists($lang) - { - $lang = $this->_convertFromNameMode($lang); - - if (is_string($lang)) { - return isset($this->_lang_db[strtolower($lang)]); - - } elseif (is_array($lang)) { - foreach ($lang as $test_lang) { - if (!isset($this->_lang_db[strtolower($test_lang)])) { - return false; - } - } - return true; - - } else { - throw new Text_LanguageDetect_Exception( - 'Unsupported parameter type passed to languageExists()', - Text_LanguageDetect_Exception::PARAM_TYPE - ); - } - } - - /** - * Returns the list of detectable languages - * - * @access public - * @return array the names of the languages known to this object<<<<<<< - * @throws Text_LanguageDetect_Exception - */ - function getLanguages() - { - return $this->_convertToNameMode( - array_keys($this->_lang_db) - ); - } - - /** - * Make this object behave like Language::Guess - * - * @param bool $setting false to turn off perl compatibility - * - * @return void - */ - public function setPerlCompatible($setting = true) - { - if (is_bool($setting)) { // input check - $this->_perl_compatible = $setting; - - if ($setting == true) { - $this->_max_score = $this->_threshold; - } else { - $this->_max_score = 0; - } - } - - } - - /** - * Sets the way how language names are accepted and returned. - * - * @param integer $name_mode One of the following modes: - * 0 - language name ("english") - * 2 - 2-letter ISO 639-1 code ("en") - * 3 - 3-letter ISO 639-2 code ("eng") - * - * @return void - */ - function setNameMode($name_mode) - { - $this->_name_mode = $name_mode; - } - - /** - * Whether to use unicode block ranges in detection - * - * Should speed up most detections if turned on (detault is on). In some - * circumstances it may be slower, such as for large text samples (> 10K) - * in languages that use latin scripts. In other cases it should speed up - * detection noticeably. - * - * @param bool $setting false to turn off - * - * @return void - */ - public function useUnicodeBlocks($setting = true) - { - if (is_bool($setting)) { - $this->_use_unicode_narrowing = $setting; - } - } - - /** - * Converts a piece of text into trigrams - * - * @param string $text text to convert - * - * @return array array of trigram frequencies - * @access private - * @deprecated Superceded by the Text_LanguageDetect_Parser class - */ - function _trigram($text) - { - $s = new Text_LanguageDetect_Parser($text); - $s->prepareTrigram(); - $s->prepareUnicode(false); - $s->setPadStart(!$this->_perl_compatible); - $s->analyze(); - return $s->getTrigramFreqs(); - } - - /** - * Converts a set of trigrams from frequencies to ranks - * - * Thresholds (cuts off) the list at $this->_threshold - * - * @param array $arr array of trigram - * - * @return array ranks of trigrams - * @access protected - */ - function _arr_rank($arr) - { - - // sorts alphabetically first as a standard way of breaking rank ties - $this->_bub_sort($arr); - - // below might also work, but seemed to introduce errors in testing - //ksort($arr); - //asort($arr); - - $rank = array(); - - $i = 0; - foreach ($arr as $key => $value) { - $rank[$key] = $i++; - - // cut off at a standard threshold - if ($i >= $this->_threshold) { - break; - } - } - - return $rank; - } - - /** - * Sorts an array by value breaking ties alphabetically - * - * @param array &$arr the array to sort - * - * @return void - * @access private - */ - function _bub_sort(&$arr) - { - // should do the same as this perl statement: - // sort { $trigrams{$b} == $trigrams{$a} - // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} } - - // needs to sort by both key and value at once - // using the key to break ties for the value - - // converts array into an array of arrays of each key and value - // may be a better way of doing this - $combined = array(); - - foreach ($arr as $key => $value) { - $combined[] = array($key, $value); - } - - usort($combined, array($this, '_sort_func')); - - $replacement = array(); - foreach ($combined as $key => $value) { - list($new_key, $new_value) = $value; - $replacement[$new_key] = $new_value; - } - - $arr = $replacement; - } - - /** - * Sort function used by bubble sort - * - * Callback function for usort(). - * - * @param array $a first param passed by usort() - * @param array $b second param passed by usort() - * - * @return int 1 if $a is greater, -1 if not - * @see _bub_sort() - * @access private - */ - function _sort_func($a, $b) - { - // each is actually a key/value pair, so that it can compare using both - list($a_key, $a_value) = $a; - list($b_key, $b_value) = $b; - - if ($a_value == $b_value) { - // if the values are the same, break ties using the key - return strcmp($a_key, $b_key); - - } else { - // if not, just sort normally - if ($a_value > $b_value) { - return -1; - } else { - return 1; - } - } - - // 0 should not be possible because keys must be unique - } - - /** - * Calculates a linear rank-order distance statistic between two sets of - * ranked trigrams - * - * Sums the differences in rank for each trigram. If the trigram does not - * appear in both, consider it a difference of $this->_threshold. - * - * This distance measure was proposed by Cavnar & Trenkle (1994). Despite - * its simplicity it has been shown to be highly accurate for language - * identification tasks. - * - * @param array $arr1 the reference set of trigram ranks - * @param array $arr2 the target set of trigram ranks - * - * @return int the sum of the differences between the ranks of - * the two trigram sets - * @access private - */ - function _distance($arr1, $arr2) - { - $sumdist = 0; - - foreach ($arr2 as $key => $value) { - if (isset($arr1[$key])) { - $distance = abs($value - $arr1[$key]); - } else { - // $this->_threshold sets the maximum possible distance value - // for any one pair of trigrams - $distance = $this->_threshold; - } - $sumdist += $distance; - } - - return $sumdist; - - // todo: there are other distance statistics to try, e.g. relative - // entropy, but they're probably more costly to compute - } - - /** - * Normalizes the score returned by _distance() - * - * Different if perl compatible or not - * - * @param int $score the score from _distance() - * @param int $base_count the number of trigrams being considered - * - * @return float the normalized score - * @see _distance() - * @access private - */ - function _normalize_score($score, $base_count = null) - { - if ($base_count === null) { - $base_count = $this->_threshold; - } - - if (!$this->_perl_compatible) { - return 1 - ($score / $base_count / $this->_threshold); - } else { - return floor($score / $base_count); - } - } - - - /** - * Detects the closeness of a sample of text to the known languages - * - * Calculates the statistical difference between the text and - * the trigrams for each language, normalizes the score then - * returns results for all languages in sorted order - * - * If perl compatible, the score is 300-0, 0 being most similar. - * Otherwise, it's 0-1 with 1 being most similar. - * - * The $sample text should be at least a few sentences in length; - * should be ascii-7 or utf8 encoded, if another and the mbstring extension - * is present it will try to detect and convert. However, experience has - * shown that mb_detect_encoding() *does not work very well* with at least - * some types of encoding. - * - * @param string $sample a sample of text to compare. - * @param int $limit if specified, return an array of the most likely - * $limit languages and their scores. - * - * @return mixed sorted array of language scores, blank array if no - * useable text was found - * @see _distance() - * @throws Text_LanguageDetect_Exception - */ - public function detect($sample, $limit = 0) - { - // input check - if (!Text_LanguageDetect_Parser::validateString($sample)) { - return array(); - } - - // check char encoding - // (only if mbstring extension is compiled and PHP > 4.0.6) - if (function_exists('mb_detect_encoding') - && function_exists('mb_convert_encoding') - ) { - // mb_detect_encoding isn't very reliable, to say the least - // detection should still work with a sufficient sample - // of ascii characters - $encoding = mb_detect_encoding($sample); - - // mb_detect_encoding() will return FALSE if detection fails - // don't attempt conversion if that's the case - if ($encoding != 'ASCII' && $encoding != 'UTF-8' - && $encoding !== false - ) { - // verify the encoding exists in mb_list_encodings - if (in_array($encoding, mb_list_encodings())) { - $sample = mb_convert_encoding($sample, 'UTF-8', $encoding); - } - } - } - - $sample_obj = new Text_LanguageDetect_Parser($sample); - $sample_obj->prepareTrigram(); - if ($this->_use_unicode_narrowing) { - $sample_obj->prepareUnicode(); - } - $sample_obj->setPadStart(!$this->_perl_compatible); - $sample_obj->analyze(); - - $trigram_freqs =& $sample_obj->getTrigramRanks(); - $trigram_count = count($trigram_freqs); - - if ($trigram_count == 0) { - return array(); - } - - $scores = array(); - - // use unicode block detection to narrow down the possibilities - if ($this->_use_unicode_narrowing) { - $blocks =& $sample_obj->getUnicodeBlocks(); - - if (is_array($blocks)) { - $present_blocks = array_keys($blocks); - } else { - throw new Text_LanguageDetect_Exception( - 'Error during block detection', - Text_LanguageDetect_Exception::BLOCK_DETECTION - ); - } - - $possible_langs = array(); - - foreach ($present_blocks as $blockname) { - if (isset($this->_unicode_map[$blockname])) { - - $possible_langs = array_merge( - $possible_langs, - array_keys($this->_unicode_map[$blockname]) - ); - - // todo: faster way to do this? - } - } - - // could also try an intersect operation rather than a union - // in other words, choose languages whose trigrams contain - // ALL of the unicode blocks found in this sample - // would improve speed but would be completely thrown off by an - // unexpected character, like an umlaut appearing in english text - - $possible_langs = array_intersect( - array_keys($this->_lang_db), - array_unique($possible_langs) - ); - - // needs to intersect it with the keys of _lang_db in case - // languages have been omitted - - } else { - // or just try 'em all - $possible_langs = array_keys($this->_lang_db); - } - - - foreach ($possible_langs as $lang) { - $scores[$lang] = $this->_normalize_score( - $this->_distance($this->_lang_db[$lang], $trigram_freqs), - $trigram_count - ); - } - - unset($sample_obj); - - if ($this->_perl_compatible) { - asort($scores); - } else { - arsort($scores); - } - - // todo: drop languages with a score of $this->_max_score? - - // limit the number of returned scores - if ($limit && is_numeric($limit)) { - $limited_scores = array(); - - $i = 0; - foreach ($scores as $key => $value) { - if ($i++ >= $limit) { - break; - } - - $limited_scores[$key] = $value; - } - - return $this->_convertToNameMode($limited_scores, true); - } else { - return $this->_convertToNameMode($scores, true); - } - } - - /** - * Returns only the most similar language to the text sample - * - * Calls $this->detect() and returns only the top result - * - * @param string $sample text to detect the language of - * - * @return string the name of the most likely language - * or null if no language is similar - * @see detect() - * @throws Text_LanguageDetect_Exception - */ - public function detectSimple($sample) - { - $scores = $this->detect($sample, 1); - - // if top language has the maximum possible score, - // then the top score will have been picked at random - if (!is_array($scores) || empty($scores) - || current($scores) == $this->_max_score - ) { - return null; - } else { - return key($scores); - } - } - - /** - * Returns an array containing the most similar language and a confidence - * rating - * - * Confidence is a simple measure calculated from the similarity score - * minus the similarity score from the next most similar language - * divided by the highest possible score. Languages that have closely - * related cousins (e.g. Norwegian and Danish) should generally have lower - * confidence scores. - * - * The similarity score answers the question "How likely is the text the - * returned language regardless of the other languages considered?" The - * confidence score is one way of answering the question "how likely is the - * text the detected language relative to the rest of the language model - * set?" - * - * To see how similar languages are a priori, see languageSimilarity() - * - * @param string $sample text for which language will be detected - * - * @return array most similar language, score and confidence rating - * or null if no language is similar - * @see detect() - * @throws Text_LanguageDetect_Exception - */ - public function detectConfidence($sample) - { - $scores = $this->detect($sample, 2); - - // if most similar language has the max score, it - // will have been picked at random - if (!is_array($scores) || empty($scores) - || current($scores) == $this->_max_score - ) { - return null; - } - - $arr['language'] = key($scores); - $arr['similarity'] = current($scores); - if (next($scores) !== false) { // if false then no next element - // the goal is to return a higher value if the distance between - // the similarity of the first score and the second score is high - - if ($this->_perl_compatible) { - $arr['confidence'] = (current($scores) - $arr['similarity']) - / $this->_max_score; - - } else { - $arr['confidence'] = $arr['similarity'] - current($scores); - - } - - } else { - $arr['confidence'] = null; - } - - return $arr; - } - - /** - * Returns the distribution of unicode blocks in a given utf8 string - * - * For the block name of a single char, use unicodeBlockName() - * - * @param string $str input string. Must be ascii or utf8 - * @param bool $skip_symbols if true, skip ascii digits, symbols and - * non-printing characters. Includes spaces, - * newlines and common punctutation characters. - * - * @return array - * @throws Text_LanguageDetect_Exception - */ - public function detectUnicodeBlocks($str, $skip_symbols) - { - $skip_symbols = (bool)$skip_symbols; - $str = (string)$str; - - $sample_obj = new Text_LanguageDetect_Parser($str); - $sample_obj->prepareUnicode(); - $sample_obj->prepareTrigram(false); - $sample_obj->setUnicodeSkipSymbols($skip_symbols); - $sample_obj->analyze(); - $blocks = $sample_obj->getUnicodeBlocks(); - unset($sample_obj); - return $blocks; - } - - /** - * Returns the block name for a given unicode value - * - * If passed a string, will assume it is being passed a UTF8-formatted - * character and will automatically convert. Otherwise it will assume it - * is being passed a numeric unicode value. - * - * Make sure input is of the correct type! - * - * @param mixed $unicode unicode value or utf8 char - * - * @return mixed the block name string or false if not found - * @throws Text_LanguageDetect_Exception - */ - public function unicodeBlockName($unicode) - { - if (is_string($unicode)) { - // assume it is being passed a utf8 char, so convert it - if (self::utf8strlen($unicode) > 1) { - throw new Text_LanguageDetect_Exception( - 'Pass a single char only to this method', - Text_LanguageDetect_Exception::PARAM_TYPE - ); - } - $unicode = $this->_utf8char2unicode($unicode); - - } elseif (!is_int($unicode)) { - throw new Text_LanguageDetect_Exception( - 'Input must be of type string or int.', - Text_LanguageDetect_Exception::PARAM_TYPE - ); - } - - $blocks = $this->_read_unicode_block_db(); - - $result = $this->_unicode_block_name($unicode, $blocks); - - if ($result == -1) { - return false; - } else { - return $result[2]; - } - } - - /** - * Searches the unicode block database - * - * Returns the block name for a given unicode value. unicodeBlockName() is - * the public interface for this function, which does input checks which - * this function omits for speed. - * - * @param int $unicode the unicode value - * @param array $blocks the block database - * @param int $block_count the number of defined blocks in the database - * - * @return mixed Block name, -1 if it failed - * @see unicodeBlockName() - * @access protected - */ - function _unicode_block_name($unicode, $blocks, $block_count = -1) - { - // for a reference, see - // http://www.unicode.org/Public/UNIDATA/Blocks.txt - - // assume that ascii characters are the most common - // so try it first for efficiency - if ($unicode <= $blocks[0][1]) { - return $blocks[0]; - } - - // the optional $block_count param is for efficiency - // so we this function doesn't have to run count() every time - if ($block_count != -1) { - $high = $block_count - 1; - } else { - $high = count($blocks) - 1; - } - - $low = 1; // start with 1 because ascii was 0 - - // your average binary search algorithm - while ($low <= $high) { - $mid = floor(($low + $high) / 2); - - if ($unicode < $blocks[$mid][0]) { - // if it's lower than the lower bound - $high = $mid - 1; - - } elseif ($unicode > $blocks[$mid][1]) { - // if it's higher than the upper bound - $low = $mid + 1; - - } else { - // found it - return $blocks[$mid]; - } - } - - // failed to find the block - return -1; - - // todo: differentiate when it's out of range or when it falls - // into an unassigned range? - } - - /** - * Brings up the unicode block database - * - * @return array the database of unicode block definitions - * @throws Text_LanguageDetect_Exception - * @access protected - */ - function _read_unicode_block_db() - { - // since the unicode definitions are always going to be the same, - // might as well share the memory for the db with all other instances - // of this class - static $data; - - if (!isset($data)) { - $data = $this->_readdb($this->_unicode_db_filename); - } - - return $data; - } - - /** - * Calculate the similarities between the language models - * - * Use this function to see how similar languages are to each other. - * - * If passed 2 language names, will return just those languages compared. - * If passed 1 language name, will return that language compared to - * all others. - * If passed none, will return an array of every language model compared - * to every other one. - * - * @param string $lang1 the name of the first language to be compared - * @param string $lang2 the name of the second language to be compared - * - * @return array scores of every language compared - * or the score of just the provided languages - * or null if one of the supplied languages does not exist - * @throws Text_LanguageDetect_Exception - */ - public function languageSimilarity($lang1 = null, $lang2 = null) - { - $lang1 = $this->_convertFromNameMode($lang1); - $lang2 = $this->_convertFromNameMode($lang2); - if ($lang1 != null) { - $lang1 = strtolower($lang1); - - // check if language model exists - if (!isset($this->_lang_db[$lang1])) { - return null; - } - - if ($lang2 != null) { - if (!isset($this->_lang_db[$lang2])) { - // check if language model exists - return null; - } - - $lang2 = strtolower($lang2); - - // compare just these two languages - return $this->_normalize_score( - $this->_distance( - $this->_lang_db[$lang1], - $this->_lang_db[$lang2] - ) - ); - - } else { - // compare just $lang1 to all languages - $return_arr = array(); - foreach ($this->_lang_db as $key => $value) { - if ($key != $lang1) { - // don't compare a language to itself - $return_arr[$key] = $this->_normalize_score( - $this->_distance($this->_lang_db[$lang1], $value) - ); - } - } - asort($return_arr); - - return $return_arr; - } - - - } else { - // compare all languages to each other - $return_arr = array(); - foreach (array_keys($this->_lang_db) as $lang1) { - foreach (array_keys($this->_lang_db) as $lang2) { - // skip comparing languages to themselves - if ($lang1 != $lang2) { - - if (isset($return_arr[$lang2][$lang1])) { - // don't re-calculate what's already been done - $return_arr[$lang1][$lang2] - = $return_arr[$lang2][$lang1]; - - } else { - // calculate - $return_arr[$lang1][$lang2] - = $this->_normalize_score( - $this->_distance( - $this->_lang_db[$lang1], - $this->_lang_db[$lang2] - ) - ); - - } - } - } - } - return $return_arr; - } - } - - /** - * Cluster known languages according to languageSimilarity() - * - * WARNING: this method is EXPERIMENTAL. It is not recommended for common - * use, and it may disappear or its functionality may change in future - * releases without notice. - * - * Uses a nearest neighbor technique to generate the maximum possible - * number of dendograms from the similarity data. - * - * @access public - * @return array language cluster data - * @throws Text_LanguageDetect_Exception - * @see languageSimilarity() - * @deprecated this function will eventually be removed and placed into - * the model generation class - */ - function clusterLanguages() - { - // todo: set the maximum number of clusters - // return cached result, if any - if (isset($this->_clusters)) { - return $this->_clusters; - } - - $langs = array_keys($this->_lang_db); - - $arr = $this->languageSimilarity(); - - sort($langs); - - foreach ($langs as $lang) { - if (!isset($this->_lang_db[$lang])) { - throw new Text_LanguageDetect_Exception( - "missing $lang!", - Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE - ); - } - } - - // http://www.psychstat.missouristate.edu/multibook/mlt04m.html - foreach ($langs as $old_key => $lang1) { - $langs[$lang1] = $lang1; - unset($langs[$old_key]); - } - - $result_data = $really_map = array(); - - $i = 0; - while (count($langs) > 2 && $i++ < 200) { - $highest_score = -1; - $highest_key1 = ''; - $highest_key2 = ''; - foreach ($langs as $lang1) { - foreach ($langs as $lang2) { - if ($lang1 != $lang2 - && $arr[$lang1][$lang2] > $highest_score - ) { - $highest_score = $arr[$lang1][$lang2]; - $highest_key1 = $lang1; - $highest_key2 = $lang2; - } - } - } - - if (!$highest_key1) { - // should not ever happen - throw new Text_LanguageDetect_Exception( - "no highest key? (step: $i)", - Text_LanguageDetect_Exception::NO_HIGHEST_KEY - ); - } - - if ($highest_score == 0) { - // languages are perfectly dissimilar - break; - } - - // $highest_key1 and $highest_key2 are most similar - $sum1 = array_sum($arr[$highest_key1]); - $sum2 = array_sum($arr[$highest_key2]); - - // use the score for the one that is most similar to the rest of - // the field as the score for the group - // todo: could try averaging or "centroid" method instead - // seems like that might make more sense - // actually nearest neighbor may be better for binary searching - - - // for "Complete Linkage"/"furthest neighbor" - // sign should be < - // for "Single Linkage"/"nearest neighbor" method - // should should be > - // results seem to be pretty much the same with either method - - // figure out which to delete and which to replace - if ($sum1 > $sum2) { - $replaceme = $highest_key1; - $deleteme = $highest_key2; - } else { - $replaceme = $highest_key2; - $deleteme = $highest_key1; - } - - $newkey = $replaceme . ':' . $deleteme; - - // $replaceme is most similar to remaining languages - // replace $replaceme with '$newkey', deleting $deleteme - - // keep a record of which fork is really which language - $really_lang = $replaceme; - while (isset($really_map[$really_lang])) { - $really_lang = $really_map[$really_lang]; - } - $really_map[$newkey] = $really_lang; - - - // replace the best fitting key, delete the other - foreach ($arr as $key1 => $arr2) { - foreach ($arr2 as $key2 => $value2) { - if ($key2 == $replaceme) { - $arr[$key1][$newkey] = $arr[$key1][$key2]; - unset($arr[$key1][$key2]); - // replacing $arr[$key1][$key2] with $arr[$key1][$newkey] - } - - if ($key1 == $replaceme) { - $arr[$newkey][$key2] = $arr[$key1][$key2]; - unset($arr[$key1][$key2]); - // replacing $arr[$key1][$key2] with $arr[$newkey][$key2] - } - - if ($key1 == $deleteme || $key2 == $deleteme) { - // deleting $arr[$key1][$key2] - unset($arr[$key1][$key2]); - } - } - } - - - unset($langs[$highest_key1]); - unset($langs[$highest_key2]); - $langs[$newkey] = $newkey; - - - // some of these may be overkill - $result_data[$newkey] = array( - 'newkey' => $newkey, - 'count' => $i, - 'diff' => abs($sum1 - $sum2), - 'score' => $highest_score, - 'bestfit' => $replaceme, - 'otherfit' => $deleteme, - 'really' => $really_lang, - ); - } - - $return_val = array( - 'open_forks' => $langs, - // the top level of clusters - // clusters that are mutually exclusive - // or specified by a specific maximum - - 'fork_data' => $result_data, - // data for each split - - 'name_map' => $really_map, - // which cluster is really which language - // using the nearest neighbor technique, the cluster - // inherits all of the properties of its most-similar member - // this keeps track - ); - - - // saves the result in the object - $this->_clusters = $return_val; - - return $return_val; - } - - - /** - * Perform an intelligent detection based on clusterLanguages() - * - * WARNING: this method is EXPERIMENTAL. It is not recommended for common - * use, and it may disappear or its functionality may change in future - * releases without notice. - * - * This compares the sample text to top the top level of clusters. If the - * sample is similar to the cluster it will drop down and compare it to the - * languages in the cluster, and so on until it hits a leaf node. - * - * this should find the language in considerably fewer compares - * (the equivalent of a binary search), however clusterLanguages() is costly - * and the loss of accuracy from this technique is significant. - * - * This method may need to be 'fuzzier' in order to become more accurate. - * - * This function could be more useful if the universe of possible languages - * was very large, however in such cases some method of Bayesian inference - * might be more helpful. - * - * @param string $str input string - * - * @return array language scores (only those compared) - * @throws Text_LanguageDetect_Exception - * @see clusterLanguages() - */ - public function clusteredSearch($str) - { - // input check - if (!Text_LanguageDetect_Parser::validateString($str)) { - return array(); - } - - // clusterLanguages() will return a cached result if possible - // so it's safe to call it every time - $result = $this->clusterLanguages(); - - $dendogram_start = $result['open_forks']; - $dendogram_data = $result['fork_data']; - $dendogram_alias = $result['name_map']; - - $sample_obj = new Text_LanguageDetect_Parser($str); - $sample_obj->prepareTrigram(); - $sample_obj->setPadStart(!$this->_perl_compatible); - $sample_obj->analyze(); - $sample_result = $sample_obj->getTrigramRanks(); - $sample_count = count($sample_result); - - // input check - if ($sample_count == 0) { - return array(); - } - - $i = 0; // counts the number of steps - - foreach ($dendogram_start as $lang) { - if (isset($dendogram_alias[$lang])) { - $lang_key = $dendogram_alias[$lang]; - } else { - $lang_key = $lang; - } - - $scores[$lang] = $this->_normalize_score( - $this->_distance($this->_lang_db[$lang_key], $sample_result), - $sample_count - ); - - $i++; - } - - if ($this->_perl_compatible) { - asort($scores); - } else { - arsort($scores); - } - - $top_score = current($scores); - $top_key = key($scores); - - // of starting forks, $top_key is the most similar to the sample - - $cur_key = $top_key; - while (isset($dendogram_data[$cur_key])) { - $lang1 = $dendogram_data[$cur_key]['bestfit']; - $lang2 = $dendogram_data[$cur_key]['otherfit']; - foreach (array($lang1, $lang2) as $lang) { - if (isset($dendogram_alias[$lang])) { - $lang_key = $dendogram_alias[$lang]; - } else { - $lang_key = $lang; - } - - $scores[$lang] = $this->_normalize_score( - $this->_distance($this->_lang_db[$lang_key], $sample_result), - $sample_count - ); - - //todo: does not need to do same comparison again - } - - $i++; - - if ($scores[$lang1] > $scores[$lang2]) { - $cur_key = $lang1; - $loser_key = $lang2; - } else { - $cur_key = $lang2; - $loser_key = $lang1; - } - - $diff = $scores[$cur_key] - $scores[$loser_key]; - - // $cur_key ({$dendogram_alias[$cur_key]}) wins - // over $loser_key ({$dendogram_alias[$loser_key]}) - // with a difference of $diff - } - - // found result in $i compares - - // rather than sorting the result, preserve it so that you can see - // which paths the algorithm decided to take along the tree - - // but sometimes the last item is only the second highest - if (($this->_perl_compatible && (end($scores) > prev($scores))) - || (!$this->_perl_compatible && (end($scores) < prev($scores))) - ) { - $real_last_score = current($scores); - $real_last_key = key($scores); - - // swaps the 2nd-to-last item for the last item - unset($scores[$real_last_key]); - $scores[$real_last_key] = $real_last_score; - } - - - if (!$this->_perl_compatible) { - $scores = array_reverse($scores, true); - // second param requires php > 4.0.3 - } - - return $scores; - } - - /** - * ut8-safe strlen() - * - * Returns the numbers of characters (not bytes) in a utf8 string - * - * @param string $str string to get the length of - * - * @return int number of chars - */ - public static function utf8strlen($str) - { - // utf8_decode() will convert unknown chars to '?', which is actually - // ideal for counting. - - return strlen(utf8_decode($str)); - - // idea stolen from dokuwiki - } - - /** - * Returns the unicode value of a utf8 char - * - * @param string $char a utf8 (possibly multi-byte) char - * - * @return int unicode value - * @access protected - * @link http://en.wikipedia.org/wiki/UTF-8 - */ - function _utf8char2unicode($char) - { - // strlen() here will actually get the binary length of a single char - switch (strlen($char)) { - case 1: - // normal ASCII-7 byte - // 0xxxxxxx --> 0xxxxxxx - return ord($char{0}); - - case 2: - // 2 byte unicode - // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx - $z = (ord($char{0}) & 0x000001F) << 6; - $x = (ord($char{1}) & 0x0000003F); - return ($z | $x); - - case 3: - // 3 byte unicode - // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx - $z = (ord($char{0}) & 0x0000000F) << 12; - $x1 = (ord($char{1}) & 0x0000003F) << 6; - $x2 = (ord($char{2}) & 0x0000003F); - return ($z | $x1 | $x2); - - case 4: - // 4 byte unicode - // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx --> - // 000zzzzz xxxxxxxx xxxxxxxx - $z1 = (ord($char{0}) & 0x00000007) << 18; - $z2 = (ord($char{1}) & 0x0000003F) << 12; - $x1 = (ord($char{2}) & 0x0000003F) << 6; - $x2 = (ord($char{3}) & 0x0000003F); - return ($z1 | $z2 | $x1 | $x2); - } - } - - /** - * utf8-safe fast character iterator - * - * Will get the next character starting from $counter, which will then be - * incremented. If a multi-byte char the bytes will be concatenated and - * $counter will be incremeted by the number of bytes in the char. - * - * @param string $str the string being iterated over - * @param int &$counter the iterator, will increment by reference - * @param bool $special_convert whether to do special conversions - * - * @return char the next (possibly multi-byte) char from $counter - * @access private - */ - static function _next_char($str, &$counter, $special_convert = false) - { - $char = $str{$counter++}; - $ord = ord($char); - - // for a description of the utf8 system see - // http://www.phpclasses.org/browse/file/5131.html - - // normal ascii one byte char - if ($ord <= 127) { - // special conversions needed for this package - // (that only apply to regular ascii characters) - // lower case, and convert all non-alphanumeric characters - // other than "'" to space - if ($special_convert && $char != ' ' && $char != "'") { - if ($ord >= 65 && $ord <= 90) { // A-Z - $char = chr($ord + 32); // lower case - } elseif ($ord < 97 || $ord > 122) { // NOT a-z - $char = ' '; // convert to space - } - } - - return $char; - - } elseif ($ord >> 5 == 6) { // two-byte char - // multi-byte chars - $nextchar = $str{$counter++}; // get next byte - - // lower-casing of non-ascii characters is still incomplete - - if ($special_convert) { - // lower case latin accented characters - if ($ord == 195) { - $nextord = ord($nextchar); - $nextord_adj = $nextord + 64; - // for a reference, see - // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html - - // À - Þ but not × - if ($nextord_adj >= 192 - && $nextord_adj <= 222 - && $nextord_adj != 215 - ) { - $nextchar = chr($nextord + 32); - } - - } elseif ($ord == 208) { - // lower case cyrillic alphabet - $nextord = ord($nextchar); - // if A - Pe - if ($nextord >= 144 && $nextord <= 159) { - // lower case - $nextchar = chr($nextord + 32); - - } elseif ($nextord >= 160 && $nextord <= 175) { - // if Er - Ya - // lower case - $char = chr(209); // == $ord++ - $nextchar = chr($nextord - 32); - } - } - } - - // tag on next byte - return $char . $nextchar; - } elseif ($ord >> 4 == 14) { // three-byte char - - // tag on next 2 bytes - return $char . $str{$counter++} . $str{$counter++}; - - } elseif ($ord >> 3 == 30) { // four-byte char - - // tag on next 3 bytes - return $char . $str{$counter++} . $str{$counter++} . $str{$counter++}; - - } else { - // error? - } - } - - /** - * Converts an $language input parameter from the configured mode - * to the language name that is used internally. - * - * Works for strings and arrays. - * - * @param string|array $lang A language description ("english"/"en"/"eng") - * @param boolean $convertKey If $lang is an array, setting $key - * converts the keys to the language name. - * - * @return string|array Language name - */ - function _convertFromNameMode($lang, $convertKey = false) - { - if ($this->_name_mode == 0) { - return $lang; - } - - if ($this->_name_mode == 2) { - $method = 'code2ToName'; - } else { - $method = 'code3ToName'; - } - - if (is_string($lang)) { - return (string)Text_LanguageDetect_ISO639::$method($lang); - } - - $newlang = array(); - foreach ($lang as $key => $val) { - if ($convertKey) { - $newkey = (string)Text_LanguageDetect_ISO639::$method($key); - $newlang[$newkey] = $val; - } else { - $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val); - } - } - return $newlang; - } - - /** - * Converts an $language output parameter from the language name that is - * used internally to the configured mode. - * - * Works for strings and arrays. - * - * @param string|array $lang A language description ("english"/"en"/"eng") - * @param boolean $convertKey If $lang is an array, setting $key - * converts the keys to the language name. - * - * @return string|array Language name - */ - function _convertToNameMode($lang, $convertKey = false) - { - if ($this->_name_mode == 0) { - return $lang; - } - - if ($this->_name_mode == 2) { - $method = 'nameToCode2'; - } else { - $method = 'nameToCode3'; - } - - if (is_string($lang)) { - return Text_LanguageDetect_ISO639::$method($lang); - } - - $newlang = array(); - foreach ($lang as $key => $val) { - if ($convertKey) { - $newkey = Text_LanguageDetect_ISO639::$method($key); - $newlang[$newkey] = $val; - } else { - $newlang[$key] = Text_LanguageDetect_ISO639::$method($val); - } - } - return $newlang; - } -} - -/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */ diff --git a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect.php b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect.php new file mode 100644 index 000000000..ba1647d0f --- /dev/null +++ b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect.php @@ -0,0 +1,1678 @@ + + * @copyright 2005-2006 Nicholas Pisarro + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @link http://pear.php.net/package/Text_LanguageDetect/ + */ + +require_once __DIR__ . '/LanguageDetect/Exception.php'; +require_once __DIR__ . '/LanguageDetect/Parser.php'; +require_once __DIR__ . '/LanguageDetect/ISO639.php'; + +/** + * Detects the language of a given piece of text. + * + * Attempts to detect the language of a sample of text by correlating ranked + * 3-gram frequencies to a table of 3-gram frequencies of known languages. + * + * Implements a version of a technique originally proposed by Cavnar & Trenkle + * (1994): "N-Gram-Based Text Categorization" + * + * Requires the language model database (lang.dat) that should have + * accompanied this class definition in order to be instantiated. + * + * Example usage: + * + * + * require_once 'Text/LanguageDetect.php'; + * + * $l = new Text_LanguageDetect; + * + * $stdin = fopen('php://stdin', 'r'); + * + * echo "Supported languages:\n"; + * + * try { + * $langs = $l->getLanguages(); + * } catch (Text_LanguageDetect_Exception $e) { + * die($e->getMessage()); + * } + * + * sort($langs); + * echo join(', ', $langs); + * + * while ($line = fgets($stdin)) { + * print_r($l->detect($line, 4)); + * } + * + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro + * @copyright 2005 Nicholas Pisarro + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @version Release: @package_version@ + * @link http://pear.php.net/package/Text_LanguageDetect/ + * + * @SuppressWarnings(PHPMD) + */ +class Text_LanguageDetect +{ + /** + * The filename that stores the trigram data for the detector + * + * If this value starts with a slash (/) or a dot (.) the value of + * $this->_data_dir will be ignored + * + * @var string + */ + protected $_db_filename = 'lang.dat'; + + /** + * The filename that stores the unicode block definitions + * + * If this value starts with a slash (/) or a dot (.) the value of + * $this->_data_dir will be ignored + * + * @var string + */ + protected $_unicode_db_filename = 'unicode_blocks.dat'; + + /** + * The data directory + * + * Should be set by PEAR installer + * + * @var string + */ + protected $_data_dir = '@data_dir@'; + + /** + * The trigram data for comparison + * + * Will be loaded on start from $this->_db_filename + * + * @var array + */ + protected $_lang_db = array(); + + /** + * Stores the map of the trigram data to unicode characters + * + * @var array + */ + protected $_unicode_map; + + /** + * The size of the trigram data arrays + * + * @var int + */ + protected $_threshold = 300; + + /** + * The maximum possible score. + * + * Needed for score normalization. Different depending on the + * perl compatibility setting + * + * @var int + * @see setPerlCompatible() + */ + protected $_max_score = 0; + + /** + * Whether or not to simulate perl's Language::Guess exactly + * + * @var bool + * @see setPerlCompatible() + */ + protected $_perl_compatible = false; + + /** + * Whether to use the unicode block detection to speed up processing + * + * @var bool + */ + protected $_use_unicode_narrowing = true; + + /** + * Stores the result of the clustering operation + * + * @var array + * @see clusterLanguages() + */ + protected $_clusters; + + /** + * Which type of "language names" are accepted and returned: + * + * 0 - language name ("english") + * 2 - 2-letter ISO 639-1 code ("en") + * 3 - 3-letter ISO 639-2 code ("eng") + */ + protected $_name_mode = 0; + + /** + * Constructor + * + * Will attempt to load the language database. If it fails, you will get + * an exception. + */ + public function __construct() + { + $data = $this->_readdb($this->_db_filename); + $this->_checkTrigram($data['trigram']); + $this->_lang_db = $data['trigram']; + + if (isset($data['trigram-unicodemap'])) { + $this->_unicode_map = $data['trigram-unicodemap']; + } + + // Not yet implemented: + if (isset($data['trigram-clusters'])) { + $this->_clusters = $data['trigram-clusters']; + } + } + + /** + * Returns the path to the location of the database + * + * @param string $fname File name to load + * + * @return string expected path to the language model database + */ + protected function _get_data_loc($fname) + { + if ($fname{0} == '/' || $fname{0} == '.') { + // if filename starts with a slash, assume it's an absolute pathname + // and skip whatever is in $this->_data_dir + return $fname; + + } elseif ($this->_data_dir != '@' . 'data_dir' . '@') { + // if the data dir was set by the PEAR installer, use that + return $this->_data_dir . '/Text_LanguageDetect/' . $fname; + + } else { + // assume this was just unpacked somewhere + // try the local working directory if otherwise + return __DIR__ . '/../data/' . $fname; + } + } + + /** + * Loads the language trigram database from filename + * + * Trigram datbase should be a serialize()'d array + * + * @param string $fname the filename where the data is stored + * + * @return array the language model data + * @throws Text_LanguageDetect_Exception + */ + protected function _readdb($fname) + { + // finds the correct data dir + $fname = $this->_get_data_loc($fname); + + // input check + if (!file_exists($fname)) { + throw new Text_LanguageDetect_Exception( + 'Language database does not exist: ' . $fname, + Text_LanguageDetect_Exception::DB_NOT_FOUND + ); + } elseif (!is_readable($fname)) { + throw new Text_LanguageDetect_Exception( + 'Language database is not readable: ' . $fname, + Text_LanguageDetect_Exception::DB_NOT_READABLE + ); + } + + return unserialize(file_get_contents($fname)); + } + + + /** + * Checks if this object is ready to detect languages + * + * @param array $trigram Trigram data from database + * + * @return void + */ + protected function _checkTrigram($trigram) + { + if (!is_array($trigram)) { + if (ini_get('magic_quotes_runtime')) { + throw new Text_LanguageDetect_Exception( + 'Error loading database. Try turning magic_quotes_runtime off.', + Text_LanguageDetect_Exception::MAGIC_QUOTES + ); + } + throw new Text_LanguageDetect_Exception( + 'Language database is not an array.', + Text_LanguageDetect_Exception::DB_NOT_ARRAY + ); + } elseif (empty($trigram)) { + throw new Text_LanguageDetect_Exception( + 'Language database has no elements.', + Text_LanguageDetect_Exception::DB_EMPTY + ); + } + } + + /** + * Omits languages + * + * Pass this function the name of or an array of names of + * languages that you don't want considered + * + * If you're only expecting a limited set of languages, this can greatly + * speed up processing + * + * @param mixed $omit_list language name or array of names to omit + * @param bool $include_only if true will include (rather than + * exclude) only those in the list + * + * @return int number of languages successfully deleted + * @throws Text_LanguageDetect_Exception + */ + public function omitLanguages($omit_list, $include_only = false) + { + $deleted = 0; + + $omit_list = $this->_convertFromNameMode($omit_list); + + if (!$include_only) { + // deleting the given languages + if (!is_array($omit_list)) { + $omit_list = strtolower($omit_list); // case desensitize + if (isset($this->_lang_db[$omit_list])) { + unset($this->_lang_db[$omit_list]); + $deleted++; + } + } else { + foreach ($omit_list as $omit_lang) { + if (isset($this->_lang_db[$omit_lang])) { + unset($this->_lang_db[$omit_lang]); + $deleted++; + } + } + } + + } else { + // deleting all except the given languages + if (!is_array($omit_list)) { + $omit_list = array($omit_list); + } + + // case desensitize + foreach ($omit_list as $key => $omit_lang) { + $omit_list[$key] = strtolower($omit_lang); + } + + foreach (array_keys($this->_lang_db) as $lang) { + if (!in_array($lang, $omit_list)) { + unset($this->_lang_db[$lang]); + $deleted++; + } + } + } + + // reset the cluster cache if the number of languages changes + // this will then have to be recalculated + if (isset($this->_clusters) && $deleted > 0) { + $this->_clusters = null; + } + + return $deleted; + } + + + /** + * Returns the number of languages that this object can detect + * + * @return int the number of languages + * @throws Text_LanguageDetect_Exception + */ + public function getLanguageCount() + { + return count($this->_lang_db); + } + + /** + * Checks if the language with the given name exists in the database + * + * @param mixed $lang Language name or array of language names + * + * @return bool true if language model exists + */ + public function languageExists($lang) + { + $lang = $this->_convertFromNameMode($lang); + + if (is_string($lang)) { + return isset($this->_lang_db[strtolower($lang)]); + + } elseif (is_array($lang)) { + foreach ($lang as $test_lang) { + if (!isset($this->_lang_db[strtolower($test_lang)])) { + return false; + } + } + return true; + + } else { + throw new Text_LanguageDetect_Exception( + 'Unsupported parameter type passed to languageExists()', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + } + + /** + * Returns the list of detectable languages + * + * @return array the names of the languages known to this object<<<<<<< + * @throws Text_LanguageDetect_Exception + */ + public function getLanguages() + { + return $this->_convertToNameMode( + array_keys($this->_lang_db) + ); + } + + /** + * Make this object behave like Language::Guess + * + * @param bool $setting false to turn off perl compatibility + * + * @return void + */ + public function setPerlCompatible($setting = true) + { + if (is_bool($setting)) { // input check + $this->_perl_compatible = $setting; + + if ($setting == true) { + $this->_max_score = $this->_threshold; + } else { + $this->_max_score = 0; + } + } + + } + + /** + * Sets the way how language names are accepted and returned. + * + * @param integer $name_mode One of the following modes: + * 0 - language name ("english") + * 2 - 2-letter ISO 639-1 code ("en") + * 3 - 3-letter ISO 639-2 code ("eng") + * + * @return void + */ + public function setNameMode($name_mode) + { + $this->_name_mode = $name_mode; + } + + /** + * Whether to use unicode block ranges in detection + * + * Should speed up most detections if turned on (detault is on). In some + * circumstances it may be slower, such as for large text samples (> 10K) + * in languages that use latin scripts. In other cases it should speed up + * detection noticeably. + * + * @param bool $setting false to turn off + * + * @return void + */ + public function useUnicodeBlocks($setting = true) + { + if (is_bool($setting)) { + $this->_use_unicode_narrowing = $setting; + } + } + + /** + * Converts a piece of text into trigrams + * + * @param string $text text to convert + * + * @return array array of trigram frequencies + * @deprecated Superceded by the Text_LanguageDetect_Parser class + */ + protected function _trigram($text) + { + $s = new Text_LanguageDetect_Parser($text); + $s->prepareTrigram(); + $s->prepareUnicode(false); + $s->setPadStart(!$this->_perl_compatible); + $s->analyze(); + return $s->getTrigramFreqs(); + } + + /** + * Converts a set of trigrams from frequencies to ranks + * + * Thresholds (cuts off) the list at $this->_threshold + * + * @param array $arr array of trigram + * + * @return array ranks of trigrams + */ + protected function _arr_rank($arr) + { + + // sorts alphabetically first as a standard way of breaking rank ties + $this->_bub_sort($arr); + + // below might also work, but seemed to introduce errors in testing + //ksort($arr); + //asort($arr); + + $rank = array(); + + $i = 0; + foreach ($arr as $key => $value) { + $rank[$key] = $i++; + + // cut off at a standard threshold + if ($i >= $this->_threshold) { + break; + } + } + + return $rank; + } + + /** + * Sorts an array by value breaking ties alphabetically + * + * @param array $arr the array to sort + * + * @return void + */ + protected function _bub_sort(&$arr) + { + // should do the same as this perl statement: + // sort { $trigrams{$b} == $trigrams{$a} + // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} } + + // needs to sort by both key and value at once + // using the key to break ties for the value + + // converts array into an array of arrays of each key and value + // may be a better way of doing this + $combined = array(); + + foreach ($arr as $key => $value) { + $combined[] = array($key, $value); + } + + usort($combined, array($this, '_sort_func')); + + $replacement = array(); + foreach ($combined as $key => $value) { + list($new_key, $new_value) = $value; + $replacement[$new_key] = $new_value; + } + + $arr = $replacement; + } + + /** + * Sort function used by bubble sort + * + * Callback function for usort(). + * + * @param array $a first param passed by usort() + * @param array $b second param passed by usort() + * + * @return int 1 if $a is greater, -1 if not + * @see _bub_sort() + */ + protected function _sort_func($a, $b) + { + // each is actually a key/value pair, so that it can compare using both + list($a_key, $a_value) = $a; + list($b_key, $b_value) = $b; + + if ($a_value == $b_value) { + // if the values are the same, break ties using the key + return strcmp($a_key, $b_key); + + } else { + // if not, just sort normally + if ($a_value > $b_value) { + return -1; + } else { + return 1; + } + } + + // 0 should not be possible because keys must be unique + } + + /** + * Calculates a linear rank-order distance statistic between two sets of + * ranked trigrams + * + * Sums the differences in rank for each trigram. If the trigram does not + * appear in both, consider it a difference of $this->_threshold. + * + * This distance measure was proposed by Cavnar & Trenkle (1994). Despite + * its simplicity it has been shown to be highly accurate for language + * identification tasks. + * + * @param array $arr1 the reference set of trigram ranks + * @param array $arr2 the target set of trigram ranks + * + * @return int the sum of the differences between the ranks of + * the two trigram sets + */ + protected function _distance($arr1, $arr2) + { + $sumdist = 0; + + foreach ($arr2 as $key => $value) { + if (isset($arr1[$key])) { + $distance = abs($value - $arr1[$key]); + } else { + // $this->_threshold sets the maximum possible distance value + // for any one pair of trigrams + $distance = $this->_threshold; + } + $sumdist += $distance; + } + + return $sumdist; + + // todo: there are other distance statistics to try, e.g. relative + // entropy, but they're probably more costly to compute + } + + /** + * Normalizes the score returned by _distance() + * + * Different if perl compatible or not + * + * @param int $score the score from _distance() + * @param int $base_count the number of trigrams being considered + * + * @return float the normalized score + * @see _distance() + */ + protected function _normalize_score($score, $base_count = null) + { + if ($base_count === null) { + $base_count = $this->_threshold; + } + + if (!$this->_perl_compatible) { + return 1 - ($score / $base_count / $this->_threshold); + } else { + return floor($score / $base_count); + } + } + + + /** + * Detects the closeness of a sample of text to the known languages + * + * Calculates the statistical difference between the text and + * the trigrams for each language, normalizes the score then + * returns results for all languages in sorted order + * + * If perl compatible, the score is 300-0, 0 being most similar. + * Otherwise, it's 0-1 with 1 being most similar. + * + * The $sample text should be at least a few sentences in length; + * should be ascii-7 or utf8 encoded, if another and the mbstring extension + * is present it will try to detect and convert. However, experience has + * shown that mb_detect_encoding() *does not work very well* with at least + * some types of encoding. + * + * @param string $sample a sample of text to compare. + * @param int $limit if specified, return an array of the most likely + * $limit languages and their scores. + * + * @return mixed sorted array of language scores, blank array if no + * useable text was found + * @see _distance() + * @throws Text_LanguageDetect_Exception + */ + public function detect($sample, $limit = 0) + { + // input check + if (!Text_LanguageDetect_Parser::validateString($sample)) { + return array(); + } + + // check char encoding + // (only if mbstring extension is compiled and PHP > 4.0.6) + if (function_exists('mb_detect_encoding') + && function_exists('mb_convert_encoding') + ) { + // mb_detect_encoding isn't very reliable, to say the least + // detection should still work with a sufficient sample + // of ascii characters + $encoding = mb_detect_encoding($sample); + + // mb_detect_encoding() will return FALSE if detection fails + // don't attempt conversion if that's the case + if ($encoding != 'ASCII' && $encoding != 'UTF-8' + && $encoding !== false + ) { + // verify the encoding exists in mb_list_encodings + if (in_array($encoding, mb_list_encodings())) { + $sample = mb_convert_encoding($sample, 'UTF-8', $encoding); + } + } + } + + $sample_obj = new Text_LanguageDetect_Parser($sample); + $sample_obj->prepareTrigram(); + if ($this->_use_unicode_narrowing) { + $sample_obj->prepareUnicode(); + } + $sample_obj->setPadStart(!$this->_perl_compatible); + $sample_obj->analyze(); + + $trigram_freqs = $sample_obj->getTrigramRanks(); + $trigram_count = count($trigram_freqs); + + if ($trigram_count == 0) { + return array(); + } + + $scores = array(); + + // use unicode block detection to narrow down the possibilities + if ($this->_use_unicode_narrowing) { + $blocks = $sample_obj->getUnicodeBlocks(); + + if (is_array($blocks)) { + $present_blocks = array_keys($blocks); + } else { + throw new Text_LanguageDetect_Exception( + 'Error during block detection', + Text_LanguageDetect_Exception::BLOCK_DETECTION + ); + } + + $possible_langs = array(); + + foreach ($present_blocks as $blockname) { + if (isset($this->_unicode_map[$blockname])) { + + $possible_langs = array_merge( + $possible_langs, + array_keys($this->_unicode_map[$blockname]) + ); + + // todo: faster way to do this? + } + } + + // could also try an intersect operation rather than a union + // in other words, choose languages whose trigrams contain + // ALL of the unicode blocks found in this sample + // would improve speed but would be completely thrown off by an + // unexpected character, like an umlaut appearing in english text + + $possible_langs = array_intersect( + array_keys($this->_lang_db), + array_unique($possible_langs) + ); + + // needs to intersect it with the keys of _lang_db in case + // languages have been omitted + + } else { + // or just try 'em all + $possible_langs = array_keys($this->_lang_db); + } + + + foreach ($possible_langs as $lang) { + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang], $trigram_freqs), + $trigram_count + ); + } + + unset($sample_obj); + + if ($this->_perl_compatible) { + asort($scores); + } else { + arsort($scores); + } + + // todo: drop languages with a score of $this->_max_score? + + // limit the number of returned scores + if ($limit && is_numeric($limit)) { + $limited_scores = array(); + + $i = 0; + foreach ($scores as $key => $value) { + if ($i++ >= $limit) { + break; + } + + $limited_scores[$key] = $value; + } + + return $this->_convertToNameMode($limited_scores, true); + } else { + return $this->_convertToNameMode($scores, true); + } + } + + /** + * Returns only the most similar language to the text sample + * + * Calls $this->detect() and returns only the top result + * + * @param string $sample text to detect the language of + * + * @return string the name of the most likely language + * or null if no language is similar + * @see detect() + * @throws Text_LanguageDetect_Exception + */ + public function detectSimple($sample) + { + $scores = $this->detect($sample, 1); + + // if top language has the maximum possible score, + // then the top score will have been picked at random + if (!is_array($scores) || empty($scores) + || current($scores) == $this->_max_score + ) { + return null; + } else { + return key($scores); + } + } + + /** + * Returns an array containing the most similar language and a confidence + * rating + * + * Confidence is a simple measure calculated from the similarity score + * minus the similarity score from the next most similar language + * divided by the highest possible score. Languages that have closely + * related cousins (e.g. Norwegian and Danish) should generally have lower + * confidence scores. + * + * The similarity score answers the question "How likely is the text the + * returned language regardless of the other languages considered?" The + * confidence score is one way of answering the question "how likely is the + * text the detected language relative to the rest of the language model + * set?" + * + * To see how similar languages are a priori, see languageSimilarity() + * + * @param string $sample text for which language will be detected + * + * @return array most similar language, score and confidence rating + * or null if no language is similar + * @see detect() + * @throws Text_LanguageDetect_Exception + */ + public function detectConfidence($sample) + { + $scores = $this->detect($sample, 2); + + // if most similar language has the max score, it + // will have been picked at random + if (!is_array($scores) || empty($scores) + || current($scores) == $this->_max_score + ) { + return null; + } + + $arr['language'] = key($scores); + $arr['similarity'] = current($scores); + if (next($scores) !== false) { // if false then no next element + // the goal is to return a higher value if the distance between + // the similarity of the first score and the second score is high + + if ($this->_perl_compatible) { + $arr['confidence'] = (current($scores) - $arr['similarity']) + / $this->_max_score; + + } else { + $arr['confidence'] = $arr['similarity'] - current($scores); + + } + + } else { + $arr['confidence'] = null; + } + + return $arr; + } + + /** + * Returns the distribution of unicode blocks in a given utf8 string + * + * For the block name of a single char, use unicodeBlockName() + * + * @param string $str input string. Must be ascii or utf8 + * @param bool $skip_symbols if true, skip ascii digits, symbols and + * non-printing characters. Includes spaces, + * newlines and common punctutation characters. + * + * @return array + * @throws Text_LanguageDetect_Exception + */ + public function detectUnicodeBlocks($str, $skip_symbols) + { + $skip_symbols = (bool)$skip_symbols; + $str = (string)$str; + + $sample_obj = new Text_LanguageDetect_Parser($str); + $sample_obj->prepareUnicode(); + $sample_obj->prepareTrigram(false); + $sample_obj->setUnicodeSkipSymbols($skip_symbols); + $sample_obj->analyze(); + $blocks = $sample_obj->getUnicodeBlocks(); + unset($sample_obj); + return $blocks; + } + + /** + * Returns the block name for a given unicode value + * + * If passed a string, will assume it is being passed a UTF8-formatted + * character and will automatically convert. Otherwise it will assume it + * is being passed a numeric unicode value. + * + * Make sure input is of the correct type! + * + * @param mixed $unicode unicode value or utf8 char + * + * @return mixed the block name string or false if not found + * @throws Text_LanguageDetect_Exception + */ + public function unicodeBlockName($unicode) + { + if (is_string($unicode)) { + // assume it is being passed a utf8 char, so convert it + if (self::utf8strlen($unicode) > 1) { + throw new Text_LanguageDetect_Exception( + 'Pass a single char only to this method', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + $unicode = $this->_utf8char2unicode($unicode); + + } elseif (!is_int($unicode)) { + throw new Text_LanguageDetect_Exception( + 'Input must be of type string or int.', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + + $blocks = $this->_read_unicode_block_db(); + + $result = $this->_unicode_block_name($unicode, $blocks); + + if ($result == -1) { + return false; + } else { + return $result[2]; + } + } + + /** + * Searches the unicode block database + * + * Returns the block name for a given unicode value. unicodeBlockName() is + * the public interface for this function, which does input checks which + * this function omits for speed. + * + * @param int $unicode the unicode value + * @param array $blocks the block database + * @param int $block_count the number of defined blocks in the database + * + * @return mixed Block name, -1 if it failed + * @see unicodeBlockName() + */ + protected function _unicode_block_name($unicode, $blocks, $block_count = -1) + { + // for a reference, see + // http://www.unicode.org/Public/UNIDATA/Blocks.txt + + // assume that ascii characters are the most common + // so try it first for efficiency + if ($unicode <= $blocks[0][1]) { + return $blocks[0]; + } + + // the optional $block_count param is for efficiency + // so we this function doesn't have to run count() every time + if ($block_count != -1) { + $high = $block_count - 1; + } else { + $high = count($blocks) - 1; + } + + $low = 1; // start with 1 because ascii was 0 + + // your average binary search algorithm + while ($low <= $high) { + $mid = floor(($low + $high) / 2); + + if ($unicode < $blocks[$mid][0]) { + // if it's lower than the lower bound + $high = $mid - 1; + + } elseif ($unicode > $blocks[$mid][1]) { + // if it's higher than the upper bound + $low = $mid + 1; + + } else { + // found it + return $blocks[$mid]; + } + } + + // failed to find the block + return -1; + + // todo: differentiate when it's out of range or when it falls + // into an unassigned range? + } + + /** + * Brings up the unicode block database + * + * @return array the database of unicode block definitions + * @throws Text_LanguageDetect_Exception + */ + protected function _read_unicode_block_db() + { + // since the unicode definitions are always going to be the same, + // might as well share the memory for the db with all other instances + // of this class + static $data; + + if (!isset($data)) { + $data = $this->_readdb($this->_unicode_db_filename); + } + + return $data; + } + + /** + * Calculate the similarities between the language models + * + * Use this function to see how similar languages are to each other. + * + * If passed 2 language names, will return just those languages compared. + * If passed 1 language name, will return that language compared to + * all others. + * If passed none, will return an array of every language model compared + * to every other one. + * + * @param string $lang1 the name of the first language to be compared + * @param string $lang2 the name of the second language to be compared + * + * @return array scores of every language compared + * or the score of just the provided languages + * or null if one of the supplied languages does not exist + * @throws Text_LanguageDetect_Exception + */ + public function languageSimilarity($lang1 = null, $lang2 = null) + { + $lang1 = $this->_convertFromNameMode($lang1); + $lang2 = $this->_convertFromNameMode($lang2); + if ($lang1 != null) { + $lang1 = strtolower($lang1); + + // check if language model exists + if (!isset($this->_lang_db[$lang1])) { + return null; + } + + if ($lang2 != null) { + if (!isset($this->_lang_db[$lang2])) { + // check if language model exists + return null; + } + + $lang2 = strtolower($lang2); + + // compare just these two languages + return $this->_normalize_score( + $this->_distance( + $this->_lang_db[$lang1], + $this->_lang_db[$lang2] + ) + ); + + } else { + // compare just $lang1 to all languages + $return_arr = array(); + foreach ($this->_lang_db as $key => $value) { + if ($key != $lang1) { + // don't compare a language to itself + $return_arr[$key] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang1], $value) + ); + } + } + asort($return_arr); + + return $return_arr; + } + + + } else { + // compare all languages to each other + $return_arr = array(); + foreach (array_keys($this->_lang_db) as $lang1) { + foreach (array_keys($this->_lang_db) as $lang2) { + // skip comparing languages to themselves + if ($lang1 != $lang2) { + + if (isset($return_arr[$lang2][$lang1])) { + // don't re-calculate what's already been done + $return_arr[$lang1][$lang2] + = $return_arr[$lang2][$lang1]; + + } else { + // calculate + $return_arr[$lang1][$lang2] + = $this->_normalize_score( + $this->_distance( + $this->_lang_db[$lang1], + $this->_lang_db[$lang2] + ) + ); + + } + } + } + } + return $return_arr; + } + } + + /** + * Cluster known languages according to languageSimilarity() + * + * WARNING: this method is EXPERIMENTAL. It is not recommended for common + * use, and it may disappear or its functionality may change in future + * releases without notice. + * + * Uses a nearest neighbor technique to generate the maximum possible + * number of dendograms from the similarity data. + * + * @return array language cluster data + * @throws Text_LanguageDetect_Exception + * @see languageSimilarity() + * @deprecated this function will eventually be removed and placed into + * the model generation class + */ + public function clusterLanguages() + { + // todo: set the maximum number of clusters + // return cached result, if any + if (isset($this->_clusters)) { + return $this->_clusters; + } + + $langs = array_keys($this->_lang_db); + + $arr = $this->languageSimilarity(); + + sort($langs); + + foreach ($langs as $lang) { + if (!isset($this->_lang_db[$lang])) { + throw new Text_LanguageDetect_Exception( + "missing $lang!", + Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE + ); + } + } + + // http://www.psychstat.missouristate.edu/multibook/mlt04m.html + foreach ($langs as $old_key => $lang1) { + $langs[$lang1] = $lang1; + unset($langs[$old_key]); + } + + $result_data = $really_map = array(); + + $i = 0; + while (count($langs) > 2 && $i++ < 200) { + $highest_score = -1; + $highest_key1 = ''; + $highest_key2 = ''; + foreach ($langs as $lang1) { + foreach ($langs as $lang2) { + if ($lang1 != $lang2 + && $arr[$lang1][$lang2] > $highest_score + ) { + $highest_score = $arr[$lang1][$lang2]; + $highest_key1 = $lang1; + $highest_key2 = $lang2; + } + } + } + + if (!$highest_key1) { + // should not ever happen + throw new Text_LanguageDetect_Exception( + "no highest key? (step: $i)", + Text_LanguageDetect_Exception::NO_HIGHEST_KEY + ); + } + + if ($highest_score == 0) { + // languages are perfectly dissimilar + break; + } + + // $highest_key1 and $highest_key2 are most similar + $sum1 = array_sum($arr[$highest_key1]); + $sum2 = array_sum($arr[$highest_key2]); + + // use the score for the one that is most similar to the rest of + // the field as the score for the group + // todo: could try averaging or "centroid" method instead + // seems like that might make more sense + // actually nearest neighbor may be better for binary searching + + + // for "Complete Linkage"/"furthest neighbor" + // sign should be < + // for "Single Linkage"/"nearest neighbor" method + // should should be > + // results seem to be pretty much the same with either method + + // figure out which to delete and which to replace + if ($sum1 > $sum2) { + $replaceme = $highest_key1; + $deleteme = $highest_key2; + } else { + $replaceme = $highest_key2; + $deleteme = $highest_key1; + } + + $newkey = $replaceme . ':' . $deleteme; + + // $replaceme is most similar to remaining languages + // replace $replaceme with '$newkey', deleting $deleteme + + // keep a record of which fork is really which language + $really_lang = $replaceme; + while (isset($really_map[$really_lang])) { + $really_lang = $really_map[$really_lang]; + } + $really_map[$newkey] = $really_lang; + + + // replace the best fitting key, delete the other + foreach ($arr as $key1 => $arr2) { + foreach ($arr2 as $key2 => $value2) { + if ($key2 == $replaceme) { + $arr[$key1][$newkey] = $arr[$key1][$key2]; + unset($arr[$key1][$key2]); + // replacing $arr[$key1][$key2] with $arr[$key1][$newkey] + } + + if ($key1 == $replaceme) { + $arr[$newkey][$key2] = $arr[$key1][$key2]; + unset($arr[$key1][$key2]); + // replacing $arr[$key1][$key2] with $arr[$newkey][$key2] + } + + if ($key1 == $deleteme || $key2 == $deleteme) { + // deleting $arr[$key1][$key2] + unset($arr[$key1][$key2]); + } + } + } + + + unset($langs[$highest_key1]); + unset($langs[$highest_key2]); + $langs[$newkey] = $newkey; + + + // some of these may be overkill + $result_data[$newkey] = array( + 'newkey' => $newkey, + 'count' => $i, + 'diff' => abs($sum1 - $sum2), + 'score' => $highest_score, + 'bestfit' => $replaceme, + 'otherfit' => $deleteme, + 'really' => $really_lang, + ); + } + + $return_val = array( + 'open_forks' => $langs, + // the top level of clusters + // clusters that are mutually exclusive + // or specified by a specific maximum + + 'fork_data' => $result_data, + // data for each split + + 'name_map' => $really_map, + // which cluster is really which language + // using the nearest neighbor technique, the cluster + // inherits all of the properties of its most-similar member + // this keeps track + ); + + + // saves the result in the object + $this->_clusters = $return_val; + + return $return_val; + } + + + /** + * Perform an intelligent detection based on clusterLanguages() + * + * WARNING: this method is EXPERIMENTAL. It is not recommended for common + * use, and it may disappear or its functionality may change in future + * releases without notice. + * + * This compares the sample text to top the top level of clusters. If the + * sample is similar to the cluster it will drop down and compare it to the + * languages in the cluster, and so on until it hits a leaf node. + * + * this should find the language in considerably fewer compares + * (the equivalent of a binary search), however clusterLanguages() is costly + * and the loss of accuracy from this technique is significant. + * + * This method may need to be 'fuzzier' in order to become more accurate. + * + * This function could be more useful if the universe of possible languages + * was very large, however in such cases some method of Bayesian inference + * might be more helpful. + * + * @param string $str input string + * + * @return array language scores (only those compared) + * @throws Text_LanguageDetect_Exception + * @see clusterLanguages() + */ + public function clusteredSearch($str) + { + // input check + if (!Text_LanguageDetect_Parser::validateString($str)) { + return array(); + } + + // clusterLanguages() will return a cached result if possible + // so it's safe to call it every time + $result = $this->clusterLanguages(); + + $dendogram_start = $result['open_forks']; + $dendogram_data = $result['fork_data']; + $dendogram_alias = $result['name_map']; + + $sample_obj = new Text_LanguageDetect_Parser($str); + $sample_obj->prepareTrigram(); + $sample_obj->setPadStart(!$this->_perl_compatible); + $sample_obj->analyze(); + $sample_result = $sample_obj->getTrigramRanks(); + $sample_count = count($sample_result); + + // input check + if ($sample_count == 0) { + return array(); + } + + $i = 0; // counts the number of steps + + foreach ($dendogram_start as $lang) { + if (isset($dendogram_alias[$lang])) { + $lang_key = $dendogram_alias[$lang]; + } else { + $lang_key = $lang; + } + + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang_key], $sample_result), + $sample_count + ); + + $i++; + } + + if ($this->_perl_compatible) { + asort($scores); + } else { + arsort($scores); + } + + $top_score = current($scores); + $top_key = key($scores); + + // of starting forks, $top_key is the most similar to the sample + + $cur_key = $top_key; + while (isset($dendogram_data[$cur_key])) { + $lang1 = $dendogram_data[$cur_key]['bestfit']; + $lang2 = $dendogram_data[$cur_key]['otherfit']; + foreach (array($lang1, $lang2) as $lang) { + if (isset($dendogram_alias[$lang])) { + $lang_key = $dendogram_alias[$lang]; + } else { + $lang_key = $lang; + } + + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang_key], $sample_result), + $sample_count + ); + + //todo: does not need to do same comparison again + } + + $i++; + + if ($scores[$lang1] > $scores[$lang2]) { + $cur_key = $lang1; + $loser_key = $lang2; + } else { + $cur_key = $lang2; + $loser_key = $lang1; + } + + $diff = $scores[$cur_key] - $scores[$loser_key]; + + // $cur_key ({$dendogram_alias[$cur_key]}) wins + // over $loser_key ({$dendogram_alias[$loser_key]}) + // with a difference of $diff + } + + // found result in $i compares + + // rather than sorting the result, preserve it so that you can see + // which paths the algorithm decided to take along the tree + + // but sometimes the last item is only the second highest + if (($this->_perl_compatible && (end($scores) > prev($scores))) + || (!$this->_perl_compatible && (end($scores) < prev($scores))) + ) { + $real_last_score = current($scores); + $real_last_key = key($scores); + + // swaps the 2nd-to-last item for the last item + unset($scores[$real_last_key]); + $scores[$real_last_key] = $real_last_score; + } + + + if (!$this->_perl_compatible) { + $scores = array_reverse($scores, true); + // second param requires php > 4.0.3 + } + + return $scores; + } + + /** + * UTF8-safe strlen() + * + * Returns the numbers of characters (not bytes) in a utf8 string + * + * @param string $str string to get the length of + * + * @return int number of chars + */ + public static function utf8strlen($str) + { + // utf8_decode() will convert unknown chars to '?', which is actually + // ideal for counting. + + return strlen(utf8_decode($str)); + + // idea stolen from dokuwiki + } + + /** + * Returns the unicode value of a utf8 char + * + * @param string $char a utf8 (possibly multi-byte) char + * + * @return int unicode value + * @link http://en.wikipedia.org/wiki/UTF-8 + */ + protected function _utf8char2unicode($char) + { + // strlen() here will actually get the binary length of a single char + switch (strlen($char)) { + case 1: + // normal ASCII-7 byte + // 0xxxxxxx --> 0xxxxxxx + return ord($char{0}); + + case 2: + // 2 byte unicode + // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx + $z = (ord($char{0}) & 0x000001F) << 6; + $x = (ord($char{1}) & 0x0000003F); + return ($z | $x); + + case 3: + // 3 byte unicode + // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx + $z = (ord($char{0}) & 0x0000000F) << 12; + $x1 = (ord($char{1}) & 0x0000003F) << 6; + $x2 = (ord($char{2}) & 0x0000003F); + return ($z | $x1 | $x2); + + case 4: + // 4 byte unicode + // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx --> + // 000zzzzz xxxxxxxx xxxxxxxx + $z1 = (ord($char{0}) & 0x00000007) << 18; + $z2 = (ord($char{1}) & 0x0000003F) << 12; + $x1 = (ord($char{2}) & 0x0000003F) << 6; + $x2 = (ord($char{3}) & 0x0000003F); + return ($z1 | $z2 | $x1 | $x2); + } + } + + /** + * UTF8-safe fast character iterator + * + * Will get the next character starting from $counter, which will then be + * incremented. If a multi-byte char the bytes will be concatenated and + * $counter will be incremeted by the number of bytes in the char. + * + * @param string $str the string being iterated over + * @param int $counter the iterator, will increment by reference + * @param bool $special_convert whether to do special conversions + * + * @return char the next (possibly multi-byte) char from $counter + */ + protected static function _next_char($str, &$counter, $special_convert = false) + { + $char = $str{$counter++}; + $ord = ord($char); + + // for a description of the utf8 system see + // http://www.phpclasses.org/browse/file/5131.html + + // normal ascii one byte char + if ($ord <= 127) { + // special conversions needed for this package + // (that only apply to regular ascii characters) + // lower case, and convert all non-alphanumeric characters + // other than "'" to space + if ($special_convert && $char != ' ' && $char != "'") { + if ($ord >= 65 && $ord <= 90) { // A-Z + $char = chr($ord + 32); // lower case + } elseif ($ord < 97 || $ord > 122) { // NOT a-z + $char = ' '; // convert to space + } + } + + return $char; + + } elseif ($ord >> 5 == 6) { // two-byte char + // multi-byte chars + $nextchar = $str{$counter++}; // get next byte + + // lower-casing of non-ascii characters is still incomplete + + if ($special_convert) { + // lower case latin accented characters + if ($ord == 195) { + $nextord = ord($nextchar); + $nextord_adj = $nextord + 64; + // for a reference, see + // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html + + // À - Þ but not × + if ($nextord_adj >= 192 + && $nextord_adj <= 222 + && $nextord_adj != 215 + ) { + $nextchar = chr($nextord + 32); + } + + } elseif ($ord == 208) { + // lower case cyrillic alphabet + $nextord = ord($nextchar); + // if A - Pe + if ($nextord >= 144 && $nextord <= 159) { + // lower case + $nextchar = chr($nextord + 32); + + } elseif ($nextord >= 160 && $nextord <= 175) { + // if Er - Ya + // lower case + $char = chr(209); // == $ord++ + $nextchar = chr($nextord - 32); + } + } + } + + // tag on next byte + return $char . $nextchar; + } elseif ($ord >> 4 == 14) { // three-byte char + + // tag on next 2 bytes + return $char . $str{$counter++} . $str{$counter++}; + + } elseif ($ord >> 3 == 30) { // four-byte char + + // tag on next 3 bytes + return $char . $str{$counter++} . $str{$counter++} . $str{$counter++}; + + } else { + // error? + } + } + + /** + * Converts an $language input parameter from the configured mode + * to the language name that is used internally. + * + * Works for strings and arrays. + * + * @param string|array $lang A language description ("english"/"en"/"eng") + * @param boolean $convertKey If $lang is an array, setting $key + * converts the keys to the language name. + * + * @return string|array Language name + */ + protected function _convertFromNameMode($lang, $convertKey = false) + { + if ($this->_name_mode == 0) { + return $lang; + } + + if ($this->_name_mode == 2) { + $method = 'code2ToName'; + } else { + $method = 'code3ToName'; + } + + if (is_string($lang)) { + return (string)Text_LanguageDetect_ISO639::$method($lang); + } + + $newlang = array(); + foreach ($lang as $key => $val) { + if ($convertKey) { + $newkey = (string)Text_LanguageDetect_ISO639::$method($key); + $newlang[$newkey] = $val; + } else { + $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val); + } + } + return $newlang; + } + + /** + * Converts an $language output parameter from the language name that is + * used internally to the configured mode. + * + * Works for strings and arrays. + * + * @param string|array $lang A language description ("english"/"en"/"eng") + * @param boolean $convertKey If $lang is an array, setting $key + * converts the keys to the language name. + * + * @return string|array Language name + */ + protected function _convertToNameMode($lang, $convertKey = false) + { + if ($this->_name_mode == 0) { + return $lang; + } + + if ($this->_name_mode == 2) { + $method = 'nameToCode2'; + } else { + $method = 'nameToCode3'; + } + + if (is_string($lang)) { + return Text_LanguageDetect_ISO639::$method($lang); + } + + $newlang = array(); + foreach ($lang as $key => $val) { + if ($convertKey) { + $newkey = Text_LanguageDetect_ISO639::$method($key); + $newlang[$newkey] = $val; + } else { + $newlang[$key] = Text_LanguageDetect_ISO639::$method($val); + } + } + return $newlang; + } +} + +/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */ diff --git a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Exception.php b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Exception.php index 196d994f5..cdbfe13ba 100644 --- a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Exception.php +++ b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Exception.php @@ -1,4 +1,28 @@ + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @link http://pear.php.net/package/Text_LanguageDetect/ + */ + +/** + * Part of the PEAR language detection package + * + * PHP version 5 + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @link http://pear.php.net/package/Text_LanguageDetect/ + * @link http://langdetect.blogspot.com/ + */ class Text_LanguageDetect_Exception extends Exception { /** diff --git a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/ISO639.php b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/ISO639.php index 7caa9794e..388160c49 100644 --- a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/ISO639.php +++ b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/ISO639.php @@ -9,7 +9,6 @@ * @author Christian Weiske * @copyright 2011 Christian Weiske * @license http://www.debian.org/misc/bsd.license BSD - * @version SVN: $Id$ * @link http://pear.php.net/package/Text_LanguageDetect/ */ @@ -23,7 +22,7 @@ * @package Text_LanguageDetect * @author Christian Weiske * @copyright 2011 Christian Weiske - * @license http://www.debian.org/misc/bsd.license BSD + * @license BSD http://www.opensource.org/licenses/bsd-license.php * @link http://www.loc.gov/standards/iso639-2/php/code_list.php * * @SuppressWarnings(PHPMD) diff --git a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Parser.php b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Parser.php index 44eea897f..4f1206d09 100644 --- a/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Parser.php +++ b/plugins/af_lang_detect/languagedetect/Text/LanguageDetect/Parser.php @@ -1,16 +1,15 @@ + * @copyright 2006 Nicholas Pisarro + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @link http://pear.php.net/package/Text_LanguageDetect/ */ /** @@ -20,99 +19,106 @@ * class. After a new profile has been built, the data can be retrieved using * the accessor functions. * - * This class is intended to be used by the Text_LanguageDetect class, not + * This class is intended to be used by the Text_LanguageDetect class, not * end-users. * - * @category Text - * @package Text_LanguageDetect - * @author Nicholas Pisarro - * @copyright 2006 - * @license BSD - * @version release: 0.3.0 + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro + * @copyright 2006 Nicholas Pisarro + * @license BSD http://www.opensource.org/licenses/bsd-license.php + * @version Release: @package_version@ + * @link http://pear.php.net/package/Text_LanguageDetect/ */ class Text_LanguageDetect_Parser extends Text_LanguageDetect { /** - * the piece of text being parsed + * The piece of text being parsed * - * @access private - * @var string + * @var string */ - var $_string; + protected $_string; /** - * stores the trigram frequencies of the sample + * Stores the trigram frequencies of the sample * - * @access private - * @var string + * @var string */ - var $_trigrams = array(); + protected $_trigrams = array(); /** - * stores the trigram ranks of the sample + * Stores the trigram ranks of the sample * - * @access private - * @var array + * @var array */ - var $_trigram_ranks = array(); + protected $_trigram_ranks = array(); /** - * stores the unicode blocks of the sample + * Stores the unicode blocks of the sample * - * @access private - * @var array + * @var array */ - var $_unicode_blocks = array(); - + protected $_unicode_blocks = array(); + /** * Whether the parser should compile the unicode ranges - * - * @access private - * @var bool + * + * @var bool */ - var $_compile_unicode = false; + protected $_compile_unicode = false; /** * Whether the parser should compile trigrams * - * @access private - * @var bool + * @var bool */ - var $_compile_trigram = false; + protected $_compile_trigram = false; /** * Whether the trigram parser should pad the beginning of the string * - * @access private - * @var bool + * @var bool */ - var $_trigram_pad_start = false; + protected $_trigram_pad_start = false; /** * Whether the unicode parser should skip non-alphabetical ascii chars * - * @access private - * @var bool + * @var bool */ - var $_unicode_skip_symbols = true; + protected $_unicode_skip_symbols = true; /** * Constructor * - * @access private - * @param string $string string to be parsed + * @param string $string string to be parsed */ - function Text_LanguageDetect_Parser($string) { + public function __construct($string) + { $this->_string = $string; } + /** + * PHP 4 constructor for backwards compatibility. + * + * @param string $string string to be parsed + * + * @return void + */ + public function Text_LanguageDetect_Parser($string) + { + self::__construct($string); + } + /** * Returns true if a string is suitable for parsing * - * @param string $str input string to test - * @return bool true if acceptable, false if not + * @param string $str input string to test + * + * @return bool true if acceptable, false if not */ - public static function validateString($str) { + public static function validateString($str) + { if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) { return true; } else { @@ -121,34 +127,37 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect } /** - * turn on/off trigram counting + * Turn on/off trigram counting * - * @access public - * @param bool $bool true for on, false for off + * @param bool $bool true for on, false for off + * + * @return void */ - function prepareTrigram($bool = true) + public function prepareTrigram($bool = true) { $this->_compile_trigram = $bool; } /** - * turn on/off unicode block counting + * Turn on/off unicode block counting + * + * @param bool $bool true for on, false for off * - * @access public - * @param bool $bool true for on, false for off + * @return void */ - function prepareUnicode($bool = true) + public function prepareUnicode($bool = true) { $this->_compile_unicode = $bool; } /** - * turn on/off padding the beginning of the sample string + * Turn on/off padding the beginning of the sample string + * + * @param bool $bool true for on, false for off * - * @access public - * @param bool $bool true for on, false for off + * @return void */ - function setPadStart($bool = true) + public function setPadStart($bool = true) { $this->_trigram_pad_start = $bool; } @@ -156,10 +165,11 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect /** * Should the unicode block counter skip non-alphabetical ascii chars? * - * @access public - * @param bool $bool true for on, false for off + * @param bool $bool true for on, false for off + * + * @return void */ - function setUnicodeSkipSymbols($bool = true) + public function setUnicodeSkipSymbols($bool = true) { $this->_unicode_skip_symbols = $bool; } @@ -167,10 +177,9 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect /** * Returns the trigram ranks for the text sample * - * @access public - * @return array trigram ranks in the text sample + * @return array Trigram ranks in the text sample */ - function &getTrigramRanks() + public function getTrigramRanks() { return $this->_trigram_ranks; } @@ -178,39 +187,37 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect /** * Return the trigram freqency table * - * only used in testing to make sure the parser is working + * Only used in testing to make sure the parser is working * - * @access public - * @return array trigram freqencies in the text sample + * @return array Trigram freqencies in the text sample */ - function &getTrigramFreqs() + public function getTrigramFreqs() { return $this->_trigram; } /** - * returns the array of unicode blocks + * Returns the array of unicode blocks * - * @access public - * @return array unicode blocks in the text sample + * @return array Unicode blocks in the text sample */ - function &getUnicodeBlocks() + public function getUnicodeBlocks() { return $this->_unicode_blocks; } /** * Executes the parsing operation - * - * Be sure to call the set*() functions to set options and the + * + * Be sure to call the set*() functions to set options and the * prepare*() functions first to tell it what kind of data to compute * * Afterwards the get*() functions can be used to access the compiled * information. * - * @access public + * @return void */ - function analyze() + public function analyze() { $len = strlen($this->_string); $byte_counter = 0; @@ -258,9 +265,9 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect if ($this->_compile_trigram) { if (!($b == ' ' && ($a == ' ' || $char == ' '))) { if (!isset($this->_trigram[$a . $b . $char])) { - $this->_trigram[$a . $b . $char] = 1; + $this->_trigram[$a . $b . $char] = 1; } else { - $this->_trigram[$a . $b . $char]++; + $this->_trigram[$a . $b . $char]++; } } @@ -271,10 +278,11 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect // unicode block detection if ($this->_compile_unicode) { if ($this->_unicode_skip_symbols - && strlen($char) == 1 - && ($char < 'A' || $char > 'z' - || ($char > 'Z' && $char < 'a')) - && $char != "'") { // does not skip the apostrophe + && strlen($char) == 1 + && ($char < 'A' || $char > 'z' + || ($char > 'Z' && $char < 'a')) + && $char != "'" + ) { // does not skip the apostrophe // since it's included in the language // models @@ -297,7 +305,8 @@ class Text_LanguageDetect_Parser extends Text_LanguageDetect if ($this->_compile_unicode) { foreach ($unicode_chars as $utf8_char => $count) { $search_result = $this->_unicode_block_name( - $this->_utf8char2unicode($utf8_char), $blocks, $block_count); + $this->_utf8char2unicode($utf8_char), $blocks, $block_count + ); if ($search_result != -1) { $block_name = $search_result[2]; diff --git a/plugins/af_lang_detect/languagedetect/data/build-unicode_blocks.php b/plugins/af_lang_detect/languagedetect/data/build-unicode_blocks.php new file mode 100644 index 000000000..128e9ed95 --- /dev/null +++ b/plugins/af_lang_detect/languagedetect/data/build-unicode_blocks.php @@ -0,0 +1,7 @@ + + array ( + 0 => 0x0000, + 1 => 0x007F, + 2 => 'Basic Latin', + ), + 1 => + array ( + 0 => 0x0080, + 1 => 0x00FF, + 2 => 'Latin-1 Supplement', + ), + 2 => + array ( + 0 => 0x0100, + 1 => 0x017F, + 2 => 'Latin Extended-A', + ), + 3 => + array ( + 0 => 0x0180, + 1 => 0x024F, + 2 => 'Latin Extended-B', + ), + 4 => + array ( + 0 => 0x0250, + 1 => 0x02AF, + 2 => 'IPA Extensions', + ), + 5 => + array ( + 0 => 0x02B0, + 1 => 0x02FF, + 2 => 'Spacing Modifier Letters', + ), + 6 => + array ( + 0 => 0x0300, + 1 => 0x036F, + 2 => 'Combining Diacritical Marks', + ), + 7 => + array ( + 0 => 0x0370, + 1 => 0x03FF, + 2 => 'Greek and Coptic', + ), + 8 => + array ( + 0 => 0x0400, + 1 => 0x04FF, + 2 => 'Cyrillic', + ), + 9 => + array ( + 0 => 0x0500, + 1 => 0x052F, + 2 => 'Cyrillic Supplement', + ), + 10 => + array ( + 0 => 0x0530, + 1 => 0x058F, + 2 => 'Armenian', + ), + 11 => + array ( + 0 => 0x0590, + 1 => 0x05FF, + 2 => 'Hebrew', + ), + 12 => + array ( + 0 => 0x0600, + 1 => 0x06FF, + 2 => 'Arabic', + ), + 13 => + array ( + 0 => 0x0700, + 1 => 0x074F, + 2 => 'Syriac', + ), + 14 => + array ( + 0 => 0x0750, + 1 => 0x077F, + 2 => 'Arabic Supplement', + ), + 15 => + array ( + 0 => 0x0780, + 1 => 0x07BF, + 2 => 'Thaana', + ), + 16 => + array ( + 0 => 0x0900, + 1 => 0x097F, + 2 => 'Devanagari', + ), + 17 => + array ( + 0 => 0x0980, + 1 => 0x09FF, + 2 => 'Bengali', + ), + 18 => + array ( + 0 => 0x0A00, + 1 => 0x0A7F, + 2 => 'Gurmukhi', + ), + 19 => + array ( + 0 => 0x0A80, + 1 => 0x0AFF, + 2 => 'Gujarati', + ), + 20 => + array ( + 0 => 0x0B00, + 1 => 0x0B7F, + 2 => 'Oriya', + ), + 21 => + array ( + 0 => 0x0B80, + 1 => 0x0BFF, + 2 => 'Tamil', + ), + 22 => + array ( + 0 => 0x0C00, + 1 => 0x0C7F, + 2 => 'Telugu', + ), + 23 => + array ( + 0 => 0x0C80, + 1 => 0x0CFF, + 2 => 'Kannada', + ), + 24 => + array ( + 0 => 0x0D00, + 1 => 0x0D7F, + 2 => 'Malayalam', + ), + 25 => + array ( + 0 => 0x0D80, + 1 => 0x0DFF, + 2 => 'Sinhala', + ), + 26 => + array ( + 0 => 0x0E00, + 1 => 0x0E7F, + 2 => 'Thai', + ), + 27 => + array ( + 0 => 0x0E80, + 1 => 0x0EFF, + 2 => 'Lao', + ), + 28 => + array ( + 0 => 0x0F00, + 1 => 0x0FFF, + 2 => 'Tibetan', + ), + 29 => + array ( + 0 => 0x1000, + 1 => 0x109F, + 2 => 'Myanmar', + ), + 30 => + array ( + 0 => 0x10A0, + 1 => 0x10FF, + 2 => 'Georgian', + ), + 31 => + array ( + 0 => 0x1100, + 1 => 0x11FF, + 2 => 'Hangul Jamo', + ), + 32 => + array ( + 0 => 0x1200, + 1 => 0x137F, + 2 => 'Ethiopic', + ), + 33 => + array ( + 0 => 0x1380, + 1 => 0x139F, + 2 => 'Ethiopic Supplement', + ), + 34 => + array ( + 0 => 0x13A0, + 1 => 0x13FF, + 2 => 'Cherokee', + ), + 35 => + array ( + 0 => 0x1400, + 1 => 0x167F, + 2 => 'Unified Canadian Aboriginal Syllabics', + ), + 36 => + array ( + 0 => 0x1680, + 1 => 0x169F, + 2 => 'Ogham', + ), + 37 => + array ( + 0 => 0x16A0, + 1 => 0x16FF, + 2 => 'Runic', + ), + 38 => + array ( + 0 => 0x1700, + 1 => 0x171F, + 2 => 'Tagalog', + ), + 39 => + array ( + 0 => 0x1720, + 1 => 0x173F, + 2 => 'Hanunoo', + ), + 40 => + array ( + 0 => 0x1740, + 1 => 0x175F, + 2 => 'Buhid', + ), + 41 => + array ( + 0 => 0x1760, + 1 => 0x177F, + 2 => 'Tagbanwa', + ), + 42 => + array ( + 0 => 0x1780, + 1 => 0x17FF, + 2 => 'Khmer', + ), + 43 => + array ( + 0 => 0x1800, + 1 => 0x18AF, + 2 => 'Mongolian', + ), + 44 => + array ( + 0 => 0x1900, + 1 => 0x194F, + 2 => 'Limbu', + ), + 45 => + array ( + 0 => 0x1950, + 1 => 0x197F, + 2 => 'Tai Le', + ), + 46 => + array ( + 0 => 0x1980, + 1 => 0x19DF, + 2 => 'New Tai Lue', + ), + 47 => + array ( + 0 => 0x19E0, + 1 => 0x19FF, + 2 => 'Khmer Symbols', + ), + 48 => + array ( + 0 => 0x1A00, + 1 => 0x1A1F, + 2 => 'Buginese', + ), + 49 => + array ( + 0 => 0x1D00, + 1 => 0x1D7F, + 2 => 'Phonetic Extensions', + ), + 50 => + array ( + 0 => 0x1D80, + 1 => 0x1DBF, + 2 => 'Phonetic Extensions Supplement', + ), + 51 => + array ( + 0 => 0x1DC0, + 1 => 0x1DFF, + 2 => 'Combining Diacritical Marks Supplement', + ), + 52 => + array ( + 0 => 0x1E00, + 1 => 0x1EFF, + 2 => 'Latin Extended Additional', + ), + 53 => + array ( + 0 => 0x1F00, + 1 => 0x1FFF, + 2 => 'Greek Extended', + ), + 54 => + array ( + 0 => 0x2000, + 1 => 0x206F, + 2 => 'General Punctuation', + ), + 55 => + array ( + 0 => 0x2070, + 1 => 0x209F, + 2 => 'Superscripts and Subscripts', + ), + 56 => + array ( + 0 => 0x20A0, + 1 => 0x20CF, + 2 => 'Currency Symbols', + ), + 57 => + array ( + 0 => 0x20D0, + 1 => 0x20FF, + 2 => 'Combining Diacritical Marks for Symbols', + ), + 58 => + array ( + 0 => 0x2100, + 1 => 0x214F, + 2 => 'Letterlike Symbols', + ), + 59 => + array ( + 0 => 0x2150, + 1 => 0x218F, + 2 => 'Number Forms', + ), + 60 => + array ( + 0 => 0x2190, + 1 => 0x21FF, + 2 => 'Arrows', + ), + 61 => + array ( + 0 => 0x2200, + 1 => 0x22FF, + 2 => 'Mathematical Operators', + ), + 62 => + array ( + 0 => 0x2300, + 1 => 0x23FF, + 2 => 'Miscellaneous Technical', + ), + 63 => + array ( + 0 => 0x2400, + 1 => 0x243F, + 2 => 'Control Pictures', + ), + 64 => + array ( + 0 => 0x2440, + 1 => 0x245F, + 2 => 'Optical Character Recognition', + ), + 65 => + array ( + 0 => 0x2460, + 1 => 0x24FF, + 2 => 'Enclosed Alphanumerics', + ), + 66 => + array ( + 0 => 0x2500, + 1 => 0x257F, + 2 => 'Box Drawing', + ), + 67 => + array ( + 0 => 0x2580, + 1 => 0x259F, + 2 => 'Block Elements', + ), + 68 => + array ( + 0 => 0x25A0, + 1 => 0x25FF, + 2 => 'Geometric Shapes', + ), + 69 => + array ( + 0 => 0x2600, + 1 => 0x26FF, + 2 => 'Miscellaneous Symbols', + ), + 70 => + array ( + 0 => 0x2700, + 1 => 0x27BF, + 2 => 'Dingbats', + ), + 71 => + array ( + 0 => 0x27C0, + 1 => 0x27EF, + 2 => 'Miscellaneous Mathematical Symbols-A', + ), + 72 => + array ( + 0 => 0x27F0, + 1 => 0x27FF, + 2 => 'Supplemental Arrows-A', + ), + 73 => + array ( + 0 => 0x2800, + 1 => 0x28FF, + 2 => 'Braille Patterns', + ), + 74 => + array ( + 0 => 0x2900, + 1 => 0x297F, + 2 => 'Supplemental Arrows-B', + ), + 75 => + array ( + 0 => 0x2980, + 1 => 0x29FF, + 2 => 'Miscellaneous Mathematical Symbols-B', + ), + 76 => + array ( + 0 => 0x2A00, + 1 => 0x2AFF, + 2 => 'Supplemental Mathematical Operators', + ), + 77 => + array ( + 0 => 0x2B00, + 1 => 0x2BFF, + 2 => 'Miscellaneous Symbols and Arrows', + ), + 78 => + array ( + 0 => 0x2C00, + 1 => 0x2C5F, + 2 => 'Glagolitic', + ), + 79 => + array ( + 0 => 0x2C80, + 1 => 0x2CFF, + 2 => 'Coptic', + ), + 80 => + array ( + 0 => 0x2D00, + 1 => 0x2D2F, + 2 => 'Georgian Supplement', + ), + 81 => + array ( + 0 => 0x2D30, + 1 => 0x2D7F, + 2 => 'Tifinagh', + ), + 82 => + array ( + 0 => 0x2D80, + 1 => 0x2DDF, + 2 => 'Ethiopic Extended', + ), + 83 => + array ( + 0 => 0x2E00, + 1 => 0x2E7F, + 2 => 'Supplemental Punctuation', + ), + 84 => + array ( + 0 => 0x2E80, + 1 => 0x2EFF, + 2 => 'CJK Radicals Supplement', + ), + 85 => + array ( + 0 => 0x2F00, + 1 => 0x2FDF, + 2 => 'Kangxi Radicals', + ), + 86 => + array ( + 0 => 0x2FF0, + 1 => 0x2FFF, + 2 => 'Ideographic Description Characters', + ), + 87 => + array ( + 0 => 0x3000, + 1 => 0x303F, + 2 => 'CJK Symbols and Punctuation', + ), + 88 => + array ( + 0 => 0x3040, + 1 => 0x309F, + 2 => 'Hiragana', + ), + 89 => + array ( + 0 => 0x30A0, + 1 => 0x30FF, + 2 => 'Katakana', + ), + 90 => + array ( + 0 => 0x3100, + 1 => 0x312F, + 2 => 'Bopomofo', + ), + 91 => + array ( + 0 => 0x3130, + 1 => 0x318F, + 2 => 'Hangul Compatibility Jamo', + ), + 92 => + array ( + 0 => 0x3190, + 1 => 0x319F, + 2 => 'Kanbun', + ), + 93 => + array ( + 0 => 0x31A0, + 1 => 0x31BF, + 2 => 'Bopomofo Extended', + ), + 94 => + array ( + 0 => 0x31C0, + 1 => 0x31EF, + 2 => 'CJK Strokes', + ), + 95 => + array ( + 0 => 0x31F0, + 1 => 0x31FF, + 2 => 'Katakana Phonetic Extensions', + ), + 96 => + array ( + 0 => 0x3200, + 1 => 0x32FF, + 2 => 'Enclosed CJK Letters and Months', + ), + 97 => + array ( + 0 => 0x3300, + 1 => 0x33FF, + 2 => 'CJK Compatibility', + ), + 98 => + array ( + 0 => 0x3400, + 1 => 0x4DBF, + 2 => 'CJK Unified Ideographs Extension A', + ), + 99 => + array ( + 0 => 0x4DC0, + 1 => 0x4DFF, + 2 => 'Yijing Hexagram Symbols', + ), + 100 => + array ( + 0 => 0x4E00, + 1 => 0x9FFF, + 2 => 'CJK Unified Ideographs', + ), + 101 => + array ( + 0 => 0xA000, + 1 => 0xA48F, + 2 => 'Yi Syllables', + ), + 102 => + array ( + 0 => 0xA490, + 1 => 0xA4CF, + 2 => 'Yi Radicals', + ), + 103 => + array ( + 0 => 0xA700, + 1 => 0xA71F, + 2 => 'Modifier Tone Letters', + ), + 104 => + array ( + 0 => 0xA800, + 1 => 0xA82F, + 2 => 'Syloti Nagri', + ), + 105 => + array ( + 0 => 0xAC00, + 1 => 0xD7AF, + 2 => 'Hangul Syllables', + ), + 106 => + array ( + 0 => 0xD800, + 1 => 0xDB7F, + 2 => 'High Surrogates', + ), + 107 => + array ( + 0 => 0xDB80, + 1 => 0xDBFF, + 2 => 'High Private Use Surrogates', + ), + 108 => + array ( + 0 => 0xDC00, + 1 => 0xDFFF, + 2 => 'Low Surrogates', + ), + 109 => + array ( + 0 => 0xE000, + 1 => 0xF8FF, + 2 => 'Private Use Area', + ), + 110 => + array ( + 0 => 0xF900, + 1 => 0xFAFF, + 2 => 'CJK Compatibility Ideographs', + ), + 111 => + array ( + 0 => 0xFB00, + 1 => 0xFB4F, + 2 => 'Alphabetic Presentation Forms', + ), + 112 => + array ( + 0 => 0xFB50, + 1 => 0xFDFF, + 2 => 'Arabic Presentation Forms-A', + ), + 113 => + array ( + 0 => 0xFE00, + 1 => 0xFE0F, + 2 => 'Variation Selectors', + ), + 114 => + array ( + 0 => 0xFE10, + 1 => 0xFE1F, + 2 => 'Vertical Forms', + ), + 115 => + array ( + 0 => 0xFE20, + 1 => 0xFE2F, + 2 => 'Combining Half Marks', + ), + 116 => + array ( + 0 => 0xFE30, + 1 => 0xFE4F, + 2 => 'CJK Compatibility Forms', + ), + 117 => + array ( + 0 => 0xFE50, + 1 => 0xFE6F, + 2 => 'Small Form Variants', + ), + 118 => + array ( + 0 => 0xFE70, + 1 => 0xFEFF, + 2 => 'Arabic Presentation Forms-B', + ), + 119 => + array ( + 0 => 0xFF00, + 1 => 0xFFEF, + 2 => 'Halfwidth and Fullwidth Forms', + ), + 120 => + array ( + 0 => 0xFFF0, + 1 => 0xFFFF, + 2 => 'Specials', + ), + 121 => + array ( + 0 => 0x10000, + 1 => 0x1007F, + 2 => 'Linear B Syllabary', + ), + 122 => + array ( + 0 => 0x10080, + 1 => 0x100FF, + 2 => 'Linear B Ideograms', + ), + 123 => + array ( + 0 => 0x10100, + 1 => 0x1013F, + 2 => 'Aegean Numbers', + ), + 124 => + array ( + 0 => 0x10140, + 1 => 0x1018F, + 2 => 'Ancient Greek Numbers', + ), + 125 => + array ( + 0 => 0x10300, + 1 => 0x1032F, + 2 => 'Old Italic', + ), + 126 => + array ( + 0 => 0x10330, + 1 => 0x1034F, + 2 => 'Gothic', + ), + 127 => + array ( + 0 => 0x10380, + 1 => 0x1039F, + 2 => 'Ugaritic', + ), + 128 => + array ( + 0 => 0x103A0, + 1 => 0x103DF, + 2 => 'Old Persian', + ), + 129 => + array ( + 0 => 0x10400, + 1 => 0x1044F, + 2 => 'Deseret', + ), + 130 => + array ( + 0 => 0x10450, + 1 => 0x1047F, + 2 => 'Shavian', + ), + 131 => + array ( + 0 => 0x10480, + 1 => 0x104AF, + 2 => 'Osmanya', + ), + 132 => + array ( + 0 => 0x10800, + 1 => 0x1083F, + 2 => 'Cypriot Syllabary', + ), + 133 => + array ( + 0 => 0x10A00, + 1 => 0x10A5F, + 2 => 'Kharoshthi', + ), + 134 => + array ( + 0 => 0x1D000, + 1 => 0x1D0FF, + 2 => 'Byzantine Musical Symbols', + ), + 135 => + array ( + 0 => 0x1D100, + 1 => 0x1D1FF, + 2 => 'Musical Symbols', + ), + 136 => + array ( + 0 => 0x1D200, + 1 => 0x1D24F, + 2 => 'Ancient Greek Musical Notation', + ), + 137 => + array ( + 0 => 0x1D300, + 1 => 0x1D35F, + 2 => 'Tai Xuan Jing Symbols', + ), + 138 => + array ( + 0 => 0x1D400, + 1 => 0x1D7FF, + 2 => 'Mathematical Alphanumeric Symbols', + ), + 139 => + array ( + 0 => 0x20000, + 1 => 0x2A6DF, + 2 => 'CJK Unified Ideographs Extension B', + ), + 140 => + array ( + 0 => 0x2F800, + 1 => 0x2FA1F, + 2 => 'CJK Compatibility Ideographs Supplement', + ), + 141 => + array ( + 0 => 0xE0000, + 1 => 0xE007F, + 2 => 'Tags', + ), + 142 => + array ( + 0 => 0xE0100, + 1 => 0xE01EF, + 2 => 'Variation Selectors Supplement', + ), + 143 => + array ( + 0 => 0xF0000, + 1 => 0xFFFFF, + 2 => 'Supplementary Private Use Area-A', + ), + 144 => + array ( + 0 => 0x100000, + 1 => 0x10FFFF, + 2 => 'Supplementary Private Use Area-B', + ), +); -- cgit v1.2.3