LanguageDetect.php 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678
  1. <?php
  2. /**
  3. * Part of Text_LanguageDetect
  4. *
  5. * PHP version 5
  6. *
  7. * @category Text
  8. * @package Text_LanguageDetect
  9. * @author Nicholas Pisarro <[email protected]>
  10. * @copyright 2005-2006 Nicholas Pisarro
  11. * @license BSD http://www.opensource.org/licenses/bsd-license.php
  12. * @link http://pear.php.net/package/Text_LanguageDetect/
  13. */
  14. require_once __DIR__ . '/LanguageDetect/Exception.php';
  15. require_once __DIR__ . '/LanguageDetect/Parser.php';
  16. require_once __DIR__ . '/LanguageDetect/ISO639.php';
  17. /**
  18. * Detects the language of a given piece of text.
  19. *
  20. * Attempts to detect the language of a sample of text by correlating ranked
  21. * 3-gram frequencies to a table of 3-gram frequencies of known languages.
  22. *
  23. * Implements a version of a technique originally proposed by Cavnar & Trenkle
  24. * (1994): "N-Gram-Based Text Categorization"
  25. *
  26. * Requires the language model database (lang.dat) that should have
  27. * accompanied this class definition in order to be instantiated.
  28. *
  29. * Example usage:
  30. *
  31. * <code>
  32. * require_once 'Text/LanguageDetect.php';
  33. *
  34. * $l = new Text_LanguageDetect;
  35. *
  36. * $stdin = fopen('php://stdin', 'r');
  37. *
  38. * echo "Supported languages:\n";
  39. *
  40. * try {
  41. * $langs = $l->getLanguages();
  42. * } catch (Text_LanguageDetect_Exception $e) {
  43. * die($e->getMessage());
  44. * }
  45. *
  46. * sort($langs);
  47. * echo join(', ', $langs);
  48. *
  49. * while ($line = fgets($stdin)) {
  50. * print_r($l->detect($line, 4));
  51. * }
  52. * </code>
  53. *
  54. * @category Text
  55. * @package Text_LanguageDetect
  56. * @author Nicholas Pisarro <[email protected]>
  57. * @copyright 2005 Nicholas Pisarro
  58. * @license BSD http://www.opensource.org/licenses/bsd-license.php
  59. * @version Release: @[email protected]
  60. * @link http://pear.php.net/package/Text_LanguageDetect/
  61. *
  62. * @SuppressWarnings(PHPMD)
  63. */
  64. class Text_LanguageDetect
  65. {
  66. /**
  67. * The filename that stores the trigram data for the detector
  68. *
  69. * If this value starts with a slash (/) or a dot (.) the value of
  70. * $this->_data_dir will be ignored
  71. *
  72. * @var string
  73. */
  74. protected $_db_filename = 'lang.dat';
  75. /**
  76. * The filename that stores the unicode block definitions
  77. *
  78. * If this value starts with a slash (/) or a dot (.) the value of
  79. * $this->_data_dir will be ignored
  80. *
  81. * @var string
  82. */
  83. protected $_unicode_db_filename = 'unicode_blocks.dat';
  84. /**
  85. * The data directory
  86. *
  87. * Should be set by PEAR installer
  88. *
  89. * @var string
  90. */
  91. protected $_data_dir = '@[email protected]';
  92. /**
  93. * The trigram data for comparison
  94. *
  95. * Will be loaded on start from $this->_db_filename
  96. *
  97. * @var array
  98. */
  99. protected $_lang_db = array();
  100. /**
  101. * Stores the map of the trigram data to unicode characters
  102. *
  103. * @var array
  104. */
  105. protected $_unicode_map;
  106. /**
  107. * The size of the trigram data arrays
  108. *
  109. * @var int
  110. */
  111. protected $_threshold = 300;
  112. /**
  113. * The maximum possible score.
  114. *
  115. * Needed for score normalization. Different depending on the
  116. * perl compatibility setting
  117. *
  118. * @var int
  119. * @see setPerlCompatible()
  120. */
  121. protected $_max_score = 0;
  122. /**
  123. * Whether or not to simulate perl's Language::Guess exactly
  124. *
  125. * @var bool
  126. * @see setPerlCompatible()
  127. */
  128. protected $_perl_compatible = false;
  129. /**
  130. * Whether to use the unicode block detection to speed up processing
  131. *
  132. * @var bool
  133. */
  134. protected $_use_unicode_narrowing = true;
  135. /**
  136. * Stores the result of the clustering operation
  137. *
  138. * @var array
  139. * @see clusterLanguages()
  140. */
  141. protected $_clusters;
  142. /**
  143. * Which type of "language names" are accepted and returned:
  144. *
  145. * 0 - language name ("english")
  146. * 2 - 2-letter ISO 639-1 code ("en")
  147. * 3 - 3-letter ISO 639-2 code ("eng")
  148. */
  149. protected $_name_mode = 0;
  150. /**
  151. * Constructor
  152. *
  153. * Will attempt to load the language database. If it fails, you will get
  154. * an exception.
  155. */
  156. public function __construct()
  157. {
  158. $data = $this->_readdb($this->_db_filename);
  159. $this->_checkTrigram($data['trigram']);
  160. $this->_lang_db = $data['trigram'];
  161. if (isset($data['trigram-unicodemap'])) {
  162. $this->_unicode_map = $data['trigram-unicodemap'];
  163. }
  164. // Not yet implemented:
  165. if (isset($data['trigram-clusters'])) {
  166. $this->_clusters = $data['trigram-clusters'];
  167. }
  168. }
  169. /**
  170. * Returns the path to the location of the database
  171. *
  172. * @param string $fname File name to load
  173. *
  174. * @return string expected path to the language model database
  175. */
  176. protected function _get_data_loc($fname)
  177. {
  178. if ($fname{0} == '/' || $fname{0} == '.') {
  179. // if filename starts with a slash, assume it's an absolute pathname
  180. // and skip whatever is in $this->_data_dir
  181. return $fname;
  182. } elseif ($this->_data_dir != '@' . 'data_dir' . '@') {
  183. // if the data dir was set by the PEAR installer, use that
  184. return $this->_data_dir . '/Text_LanguageDetect/' . $fname;
  185. } else {
  186. // assume this was just unpacked somewhere
  187. // try the local working directory if otherwise
  188. return __DIR__ . '/../data/' . $fname;
  189. }
  190. }
  191. /**
  192. * Loads the language trigram database from filename
  193. *
  194. * Trigram datbase should be a serialize()'d array
  195. *
  196. * @param string $fname the filename where the data is stored
  197. *
  198. * @return array the language model data
  199. * @throws Text_LanguageDetect_Exception
  200. */
  201. protected function _readdb($fname)
  202. {
  203. // finds the correct data dir
  204. $fname = $this->_get_data_loc($fname);
  205. // input check
  206. if (!file_exists($fname)) {
  207. throw new Text_LanguageDetect_Exception(
  208. 'Language database does not exist: ' . $fname,
  209. Text_LanguageDetect_Exception::DB_NOT_FOUND
  210. );
  211. } elseif (!is_readable($fname)) {
  212. throw new Text_LanguageDetect_Exception(
  213. 'Language database is not readable: ' . $fname,
  214. Text_LanguageDetect_Exception::DB_NOT_READABLE
  215. );
  216. }
  217. return unserialize(file_get_contents($fname));
  218. }
  219. /**
  220. * Checks if this object is ready to detect languages
  221. *
  222. * @param array $trigram Trigram data from database
  223. *
  224. * @return void
  225. */
  226. protected function _checkTrigram($trigram)
  227. {
  228. if (!is_array($trigram)) {
  229. if (ini_get('magic_quotes_runtime')) {
  230. throw new Text_LanguageDetect_Exception(
  231. 'Error loading database. Try turning magic_quotes_runtime off.',
  232. Text_LanguageDetect_Exception::MAGIC_QUOTES
  233. );
  234. }
  235. throw new Text_LanguageDetect_Exception(
  236. 'Language database is not an array.',
  237. Text_LanguageDetect_Exception::DB_NOT_ARRAY
  238. );
  239. } elseif (empty($trigram)) {
  240. throw new Text_LanguageDetect_Exception(
  241. 'Language database has no elements.',
  242. Text_LanguageDetect_Exception::DB_EMPTY
  243. );
  244. }
  245. }
  246. /**
  247. * Omits languages
  248. *
  249. * Pass this function the name of or an array of names of
  250. * languages that you don't want considered
  251. *
  252. * If you're only expecting a limited set of languages, this can greatly
  253. * speed up processing
  254. *
  255. * @param mixed $omit_list language name or array of names to omit
  256. * @param bool $include_only if true will include (rather than
  257. * exclude) only those in the list
  258. *
  259. * @return int number of languages successfully deleted
  260. * @throws Text_LanguageDetect_Exception
  261. */
  262. public function omitLanguages($omit_list, $include_only = false)
  263. {
  264. $deleted = 0;
  265. $omit_list = $this->_convertFromNameMode($omit_list);
  266. if (!$include_only) {
  267. // deleting the given languages
  268. if (!is_array($omit_list)) {
  269. $omit_list = strtolower($omit_list); // case desensitize
  270. if (isset($this->_lang_db[$omit_list])) {
  271. unset($this->_lang_db[$omit_list]);
  272. $deleted++;
  273. }
  274. } else {
  275. foreach ($omit_list as $omit_lang) {
  276. if (isset($this->_lang_db[$omit_lang])) {
  277. unset($this->_lang_db[$omit_lang]);
  278. $deleted++;
  279. }
  280. }
  281. }
  282. } else {
  283. // deleting all except the given languages
  284. if (!is_array($omit_list)) {
  285. $omit_list = array($omit_list);
  286. }
  287. // case desensitize
  288. foreach ($omit_list as $key => $omit_lang) {
  289. $omit_list[$key] = strtolower($omit_lang);
  290. }
  291. foreach (array_keys($this->_lang_db) as $lang) {
  292. if (!in_array($lang, $omit_list)) {
  293. unset($this->_lang_db[$lang]);
  294. $deleted++;
  295. }
  296. }
  297. }
  298. // reset the cluster cache if the number of languages changes
  299. // this will then have to be recalculated
  300. if (isset($this->_clusters) && $deleted > 0) {
  301. $this->_clusters = null;
  302. }
  303. return $deleted;
  304. }
  305. /**
  306. * Returns the number of languages that this object can detect
  307. *
  308. * @return int the number of languages
  309. * @throws Text_LanguageDetect_Exception
  310. */
  311. public function getLanguageCount()
  312. {
  313. return count($this->_lang_db);
  314. }
  315. /**
  316. * Checks if the language with the given name exists in the database
  317. *
  318. * @param mixed $lang Language name or array of language names
  319. *
  320. * @return bool true if language model exists
  321. */
  322. public function languageExists($lang)
  323. {
  324. $lang = $this->_convertFromNameMode($lang);
  325. if (is_string($lang)) {
  326. return isset($this->_lang_db[strtolower($lang)]);
  327. } elseif (is_array($lang)) {
  328. foreach ($lang as $test_lang) {
  329. if (!isset($this->_lang_db[strtolower($test_lang)])) {
  330. return false;
  331. }
  332. }
  333. return true;
  334. } else {
  335. throw new Text_LanguageDetect_Exception(
  336. 'Unsupported parameter type passed to languageExists()',
  337. Text_LanguageDetect_Exception::PARAM_TYPE
  338. );
  339. }
  340. }
  341. /**
  342. * Returns the list of detectable languages
  343. *
  344. * @return array the names of the languages known to this object<<<<<<<
  345. * @throws Text_LanguageDetect_Exception
  346. */
  347. public function getLanguages()
  348. {
  349. return $this->_convertToNameMode(
  350. array_keys($this->_lang_db)
  351. );
  352. }
  353. /**
  354. * Make this object behave like Language::Guess
  355. *
  356. * @param bool $setting false to turn off perl compatibility
  357. *
  358. * @return void
  359. */
  360. public function setPerlCompatible($setting = true)
  361. {
  362. if (is_bool($setting)) { // input check
  363. $this->_perl_compatible = $setting;
  364. if ($setting == true) {
  365. $this->_max_score = $this->_threshold;
  366. } else {
  367. $this->_max_score = 0;
  368. }
  369. }
  370. }
  371. /**
  372. * Sets the way how language names are accepted and returned.
  373. *
  374. * @param integer $name_mode One of the following modes:
  375. * 0 - language name ("english")
  376. * 2 - 2-letter ISO 639-1 code ("en")
  377. * 3 - 3-letter ISO 639-2 code ("eng")
  378. *
  379. * @return void
  380. */
  381. public function setNameMode($name_mode)
  382. {
  383. $this->_name_mode = $name_mode;
  384. }
  385. /**
  386. * Whether to use unicode block ranges in detection
  387. *
  388. * Should speed up most detections if turned on (detault is on). In some
  389. * circumstances it may be slower, such as for large text samples (> 10K)
  390. * in languages that use latin scripts. In other cases it should speed up
  391. * detection noticeably.
  392. *
  393. * @param bool $setting false to turn off
  394. *
  395. * @return void
  396. */
  397. public function useUnicodeBlocks($setting = true)
  398. {
  399. if (is_bool($setting)) {
  400. $this->_use_unicode_narrowing = $setting;
  401. }
  402. }
  403. /**
  404. * Converts a piece of text into trigrams
  405. *
  406. * @param string $text text to convert
  407. *
  408. * @return array array of trigram frequencies
  409. * @deprecated Superceded by the Text_LanguageDetect_Parser class
  410. */
  411. protected function _trigram($text)
  412. {
  413. $s = new Text_LanguageDetect_Parser($text);
  414. $s->prepareTrigram();
  415. $s->prepareUnicode(false);
  416. $s->setPadStart(!$this->_perl_compatible);
  417. $s->analyze();
  418. return $s->getTrigramFreqs();
  419. }
  420. /**
  421. * Converts a set of trigrams from frequencies to ranks
  422. *
  423. * Thresholds (cuts off) the list at $this->_threshold
  424. *
  425. * @param array $arr array of trigram
  426. *
  427. * @return array ranks of trigrams
  428. */
  429. protected function _arr_rank($arr)
  430. {
  431. // sorts alphabetically first as a standard way of breaking rank ties
  432. $this->_bub_sort($arr);
  433. // below might also work, but seemed to introduce errors in testing
  434. //ksort($arr);
  435. //asort($arr);
  436. $rank = array();
  437. $i = 0;
  438. foreach ($arr as $key => $value) {
  439. $rank[$key] = $i++;
  440. // cut off at a standard threshold
  441. if ($i >= $this->_threshold) {
  442. break;
  443. }
  444. }
  445. return $rank;
  446. }
  447. /**
  448. * Sorts an array by value breaking ties alphabetically
  449. *
  450. * @param array $arr the array to sort
  451. *
  452. * @return void
  453. */
  454. protected function _bub_sort(&$arr)
  455. {
  456. // should do the same as this perl statement:
  457. // sort { $trigrams{$b} == $trigrams{$a}
  458. // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
  459. // needs to sort by both key and value at once
  460. // using the key to break ties for the value
  461. // converts array into an array of arrays of each key and value
  462. // may be a better way of doing this
  463. $combined = array();
  464. foreach ($arr as $key => $value) {
  465. $combined[] = array($key, $value);
  466. }
  467. usort($combined, array($this, '_sort_func'));
  468. $replacement = array();
  469. foreach ($combined as $key => $value) {
  470. list($new_key, $new_value) = $value;
  471. $replacement[$new_key] = $new_value;
  472. }
  473. $arr = $replacement;
  474. }
  475. /**
  476. * Sort function used by bubble sort
  477. *
  478. * Callback function for usort().
  479. *
  480. * @param array $a first param passed by usort()
  481. * @param array $b second param passed by usort()
  482. *
  483. * @return int 1 if $a is greater, -1 if not
  484. * @see _bub_sort()
  485. */
  486. protected function _sort_func($a, $b)
  487. {
  488. // each is actually a key/value pair, so that it can compare using both
  489. list($a_key, $a_value) = $a;
  490. list($b_key, $b_value) = $b;
  491. if ($a_value == $b_value) {
  492. // if the values are the same, break ties using the key
  493. return strcmp($a_key, $b_key);
  494. } else {
  495. // if not, just sort normally
  496. if ($a_value > $b_value) {
  497. return -1;
  498. } else {
  499. return 1;
  500. }
  501. }
  502. // 0 should not be possible because keys must be unique
  503. }
  504. /**
  505. * Calculates a linear rank-order distance statistic between two sets of
  506. * ranked trigrams
  507. *
  508. * Sums the differences in rank for each trigram. If the trigram does not
  509. * appear in both, consider it a difference of $this->_threshold.
  510. *
  511. * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
  512. * its simplicity it has been shown to be highly accurate for language
  513. * identification tasks.
  514. *
  515. * @param array $arr1 the reference set of trigram ranks
  516. * @param array $arr2 the target set of trigram ranks
  517. *
  518. * @return int the sum of the differences between the ranks of
  519. * the two trigram sets
  520. */
  521. protected function _distance($arr1, $arr2)
  522. {
  523. $sumdist = 0;
  524. foreach ($arr2 as $key => $value) {
  525. if (isset($arr1[$key])) {
  526. $distance = abs($value - $arr1[$key]);
  527. } else {
  528. // $this->_threshold sets the maximum possible distance value
  529. // for any one pair of trigrams
  530. $distance = $this->_threshold;
  531. }
  532. $sumdist += $distance;
  533. }
  534. return $sumdist;
  535. // todo: there are other distance statistics to try, e.g. relative
  536. // entropy, but they're probably more costly to compute
  537. }
  538. /**
  539. * Normalizes the score returned by _distance()
  540. *
  541. * Different if perl compatible or not
  542. *
  543. * @param int $score the score from _distance()
  544. * @param int $base_count the number of trigrams being considered
  545. *
  546. * @return float the normalized score
  547. * @see _distance()
  548. */
  549. protected function _normalize_score($score, $base_count = null)
  550. {
  551. if ($base_count === null) {
  552. $base_count = $this->_threshold;
  553. }
  554. if (!$this->_perl_compatible) {
  555. return 1 - ($score / $base_count / $this->_threshold);
  556. } else {
  557. return floor($score / $base_count);
  558. }
  559. }
  560. /**
  561. * Detects the closeness of a sample of text to the known languages
  562. *
  563. * Calculates the statistical difference between the text and
  564. * the trigrams for each language, normalizes the score then
  565. * returns results for all languages in sorted order
  566. *
  567. * If perl compatible, the score is 300-0, 0 being most similar.
  568. * Otherwise, it's 0-1 with 1 being most similar.
  569. *
  570. * The $sample text should be at least a few sentences in length;
  571. * should be ascii-7 or utf8 encoded, if another and the mbstring extension
  572. * is present it will try to detect and convert. However, experience has
  573. * shown that mb_detect_encoding() *does not work very well* with at least
  574. * some types of encoding.
  575. *
  576. * @param string $sample a sample of text to compare.
  577. * @param int $limit if specified, return an array of the most likely
  578. * $limit languages and their scores.
  579. *
  580. * @return mixed sorted array of language scores, blank array if no
  581. * useable text was found
  582. * @see _distance()
  583. * @throws Text_LanguageDetect_Exception
  584. */
  585. public function detect($sample, $limit = 0)
  586. {
  587. // input check
  588. if (!Text_LanguageDetect_Parser::validateString($sample)) {
  589. return array();
  590. }
  591. // check char encoding
  592. // (only if mbstring extension is compiled and PHP > 4.0.6)
  593. if (function_exists('mb_detect_encoding')
  594. && function_exists('mb_convert_encoding')
  595. ) {
  596. // mb_detect_encoding isn't very reliable, to say the least
  597. // detection should still work with a sufficient sample
  598. // of ascii characters
  599. $encoding = mb_detect_encoding($sample);
  600. // mb_detect_encoding() will return FALSE if detection fails
  601. // don't attempt conversion if that's the case
  602. if ($encoding != 'ASCII' && $encoding != 'UTF-8'
  603. && $encoding !== false
  604. ) {
  605. // verify the encoding exists in mb_list_encodings
  606. if (in_array($encoding, mb_list_encodings())) {
  607. $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
  608. }
  609. }
  610. }
  611. $sample_obj = new Text_LanguageDetect_Parser($sample);
  612. $sample_obj->prepareTrigram();
  613. if ($this->_use_unicode_narrowing) {
  614. $sample_obj->prepareUnicode();
  615. }
  616. $sample_obj->setPadStart(!$this->_perl_compatible);
  617. $sample_obj->analyze();
  618. $trigram_freqs = $sample_obj->getTrigramRanks();
  619. $trigram_count = count($trigram_freqs);
  620. if ($trigram_count == 0) {
  621. return array();
  622. }
  623. $scores = array();
  624. // use unicode block detection to narrow down the possibilities
  625. if ($this->_use_unicode_narrowing) {
  626. $blocks = $sample_obj->getUnicodeBlocks();
  627. if (is_array($blocks)) {
  628. $present_blocks = array_keys($blocks);
  629. } else {
  630. throw new Text_LanguageDetect_Exception(
  631. 'Error during block detection',
  632. Text_LanguageDetect_Exception::BLOCK_DETECTION
  633. );
  634. }
  635. $possible_langs = array();
  636. foreach ($present_blocks as $blockname) {
  637. if (isset($this->_unicode_map[$blockname])) {
  638. $possible_langs = array_merge(
  639. $possible_langs,
  640. array_keys($this->_unicode_map[$blockname])
  641. );
  642. // todo: faster way to do this?
  643. }
  644. }
  645. // could also try an intersect operation rather than a union
  646. // in other words, choose languages whose trigrams contain
  647. // ALL of the unicode blocks found in this sample
  648. // would improve speed but would be completely thrown off by an
  649. // unexpected character, like an umlaut appearing in english text
  650. $possible_langs = array_intersect(
  651. array_keys($this->_lang_db),
  652. array_unique($possible_langs)
  653. );
  654. // needs to intersect it with the keys of _lang_db in case
  655. // languages have been omitted
  656. } else {
  657. // or just try 'em all
  658. $possible_langs = array_keys($this->_lang_db);
  659. }
  660. foreach ($possible_langs as $lang) {
  661. $scores[$lang] = $this->_normalize_score(
  662. $this->_distance($this->_lang_db[$lang], $trigram_freqs),
  663. $trigram_count
  664. );
  665. }
  666. unset($sample_obj);
  667. if ($this->_perl_compatible) {
  668. asort($scores);
  669. } else {
  670. arsort($scores);
  671. }
  672. // todo: drop languages with a score of $this->_max_score?
  673. // limit the number of returned scores
  674. if ($limit && is_numeric($limit)) {
  675. $limited_scores = array();
  676. $i = 0;
  677. foreach ($scores as $key => $value) {
  678. if ($i++ >= $limit) {
  679. break;
  680. }
  681. $limited_scores[$key] = $value;
  682. }
  683. return $this->_convertToNameMode($limited_scores, true);
  684. } else {
  685. return $this->_convertToNameMode($scores, true);
  686. }
  687. }
  688. /**
  689. * Returns only the most similar language to the text sample
  690. *
  691. * Calls $this->detect() and returns only the top result
  692. *
  693. * @param string $sample text to detect the language of
  694. *
  695. * @return string the name of the most likely language
  696. * or null if no language is similar
  697. * @see detect()
  698. * @throws Text_LanguageDetect_Exception
  699. */
  700. public function detectSimple($sample)
  701. {
  702. $scores = $this->detect($sample, 1);
  703. // if top language has the maximum possible score,
  704. // then the top score will have been picked at random
  705. if (!is_array($scores) || empty($scores)
  706. || current($scores) == $this->_max_score
  707. ) {
  708. return null;
  709. } else {
  710. return key($scores);
  711. }
  712. }
  713. /**
  714. * Returns an array containing the most similar language and a confidence
  715. * rating
  716. *
  717. * Confidence is a simple measure calculated from the similarity score
  718. * minus the similarity score from the next most similar language
  719. * divided by the highest possible score. Languages that have closely
  720. * related cousins (e.g. Norwegian and Danish) should generally have lower
  721. * confidence scores.
  722. *
  723. * The similarity score answers the question "How likely is the text the
  724. * returned language regardless of the other languages considered?" The
  725. * confidence score is one way of answering the question "how likely is the
  726. * text the detected language relative to the rest of the language model
  727. * set?"
  728. *
  729. * To see how similar languages are a priori, see languageSimilarity()
  730. *
  731. * @param string $sample text for which language will be detected
  732. *
  733. * @return array most similar language, score and confidence rating
  734. * or null if no language is similar
  735. * @see detect()
  736. * @throws Text_LanguageDetect_Exception
  737. */
  738. public function detectConfidence($sample)
  739. {
  740. $scores = $this->detect($sample, 2);
  741. // if most similar language has the max score, it
  742. // will have been picked at random
  743. if (!is_array($scores) || empty($scores)
  744. || current($scores) == $this->_max_score
  745. ) {
  746. return null;
  747. }
  748. $arr['language'] = key($scores);
  749. $arr['similarity'] = current($scores);
  750. if (next($scores) !== false) { // if false then no next element
  751. // the goal is to return a higher value if the distance between
  752. // the similarity of the first score and the second score is high
  753. if ($this->_perl_compatible) {
  754. $arr['confidence'] = (current($scores) - $arr['similarity'])
  755. / $this->_max_score;
  756. } else {
  757. $arr['confidence'] = $arr['similarity'] - current($scores);
  758. }
  759. } else {
  760. $arr['confidence'] = null;
  761. }
  762. return $arr;
  763. }
  764. /**
  765. * Returns the distribution of unicode blocks in a given utf8 string
  766. *
  767. * For the block name of a single char, use unicodeBlockName()
  768. *
  769. * @param string $str input string. Must be ascii or utf8
  770. * @param bool $skip_symbols if true, skip ascii digits, symbols and
  771. * non-printing characters. Includes spaces,
  772. * newlines and common punctutation characters.
  773. *
  774. * @return array
  775. * @throws Text_LanguageDetect_Exception
  776. */
  777. public function detectUnicodeBlocks($str, $skip_symbols)
  778. {
  779. $skip_symbols = (bool)$skip_symbols;
  780. $str = (string)$str;
  781. $sample_obj = new Text_LanguageDetect_Parser($str);
  782. $sample_obj->prepareUnicode();
  783. $sample_obj->prepareTrigram(false);
  784. $sample_obj->setUnicodeSkipSymbols($skip_symbols);
  785. $sample_obj->analyze();
  786. $blocks = $sample_obj->getUnicodeBlocks();
  787. unset($sample_obj);
  788. return $blocks;
  789. }
  790. /**
  791. * Returns the block name for a given unicode value
  792. *
  793. * If passed a string, will assume it is being passed a UTF8-formatted
  794. * character and will automatically convert. Otherwise it will assume it
  795. * is being passed a numeric unicode value.
  796. *
  797. * Make sure input is of the correct type!
  798. *
  799. * @param mixed $unicode unicode value or utf8 char
  800. *
  801. * @return mixed the block name string or false if not found
  802. * @throws Text_LanguageDetect_Exception
  803. */
  804. public function unicodeBlockName($unicode)
  805. {
  806. if (is_string($unicode)) {
  807. // assume it is being passed a utf8 char, so convert it
  808. if (self::utf8strlen($unicode) > 1) {
  809. throw new Text_LanguageDetect_Exception(
  810. 'Pass a single char only to this method',
  811. Text_LanguageDetect_Exception::PARAM_TYPE
  812. );
  813. }
  814. $unicode = $this->_utf8char2unicode($unicode);
  815. } elseif (!is_int($unicode)) {
  816. throw new Text_LanguageDetect_Exception(
  817. 'Input must be of type string or int.',
  818. Text_LanguageDetect_Exception::PARAM_TYPE
  819. );
  820. }
  821. $blocks = $this->_read_unicode_block_db();
  822. $result = $this->_unicode_block_name($unicode, $blocks);
  823. if ($result == -1) {
  824. return false;
  825. } else {
  826. return $result[2];
  827. }
  828. }
  829. /**
  830. * Searches the unicode block database
  831. *
  832. * Returns the block name for a given unicode value. unicodeBlockName() is
  833. * the public interface for this function, which does input checks which
  834. * this function omits for speed.
  835. *
  836. * @param int $unicode the unicode value
  837. * @param array $blocks the block database
  838. * @param int $block_count the number of defined blocks in the database
  839. *
  840. * @return mixed Block name, -1 if it failed
  841. * @see unicodeBlockName()
  842. */
  843. protected function _unicode_block_name($unicode, $blocks, $block_count = -1)
  844. {
  845. // for a reference, see
  846. // http://www.unicode.org/Public/UNIDATA/Blocks.txt
  847. // assume that ascii characters are the most common
  848. // so try it first for efficiency
  849. if ($unicode <= $blocks[0][1]) {
  850. return $blocks[0];
  851. }
  852. // the optional $block_count param is for efficiency
  853. // so we this function doesn't have to run count() every time
  854. if ($block_count != -1) {
  855. $high = $block_count - 1;
  856. } else {
  857. $high = count($blocks) - 1;
  858. }
  859. $low = 1; // start with 1 because ascii was 0
  860. // your average binary search algorithm
  861. while ($low <= $high) {
  862. $mid = floor(($low + $high) / 2);
  863. if ($unicode < $blocks[$mid][0]) {
  864. // if it's lower than the lower bound
  865. $high = $mid - 1;
  866. } elseif ($unicode > $blocks[$mid][1]) {
  867. // if it's higher than the upper bound
  868. $low = $mid + 1;
  869. } else {
  870. // found it
  871. return $blocks[$mid];
  872. }
  873. }
  874. // failed to find the block
  875. return -1;
  876. // todo: differentiate when it's out of range or when it falls
  877. // into an unassigned range?
  878. }
  879. /**
  880. * Brings up the unicode block database
  881. *
  882. * @return array the database of unicode block definitions
  883. * @throws Text_LanguageDetect_Exception
  884. */
  885. protected function _read_unicode_block_db()
  886. {
  887. // since the unicode definitions are always going to be the same,
  888. // might as well share the memory for the db with all other instances
  889. // of this class
  890. static $data;
  891. if (!isset($data)) {
  892. $data = $this->_readdb($this->_unicode_db_filename);
  893. }
  894. return $data;
  895. }
  896. /**
  897. * Calculate the similarities between the language models
  898. *
  899. * Use this function to see how similar languages are to each other.
  900. *
  901. * If passed 2 language names, will return just those languages compared.
  902. * If passed 1 language name, will return that language compared to
  903. * all others.
  904. * If passed none, will return an array of every language model compared
  905. * to every other one.
  906. *
  907. * @param string $lang1 the name of the first language to be compared
  908. * @param string $lang2 the name of the second language to be compared
  909. *
  910. * @return array scores of every language compared
  911. * or the score of just the provided languages
  912. * or null if one of the supplied languages does not exist
  913. * @throws Text_LanguageDetect_Exception
  914. */
  915. public function languageSimilarity($lang1 = null, $lang2 = null)
  916. {
  917. $lang1 = $this->_convertFromNameMode($lang1);
  918. $lang2 = $this->_convertFromNameMode($lang2);
  919. if ($lang1 != null) {
  920. $lang1 = strtolower($lang1);
  921. // check if language model exists
  922. if (!isset($this->_lang_db[$lang1])) {
  923. return null;
  924. }
  925. if ($lang2 != null) {
  926. if (!isset($this->_lang_db[$lang2])) {
  927. // check if language model exists
  928. return null;
  929. }
  930. $lang2 = strtolower($lang2);
  931. // compare just these two languages
  932. return $this->_normalize_score(
  933. $this->_distance(
  934. $this->_lang_db[$lang1],
  935. $this->_lang_db[$lang2]
  936. )
  937. );
  938. } else {
  939. // compare just $lang1 to all languages
  940. $return_arr = array();
  941. foreach ($this->_lang_db as $key => $value) {
  942. if ($key != $lang1) {
  943. // don't compare a language to itself
  944. $return_arr[$key] = $this->_normalize_score(
  945. $this->_distance($this->_lang_db[$lang1], $value)
  946. );
  947. }
  948. }
  949. asort($return_arr);
  950. return $return_arr;
  951. }
  952. } else {
  953. // compare all languages to each other
  954. $return_arr = array();
  955. foreach (array_keys($this->_lang_db) as $lang1) {
  956. foreach (array_keys($this->_lang_db) as $lang2) {
  957. // skip comparing languages to themselves
  958. if ($lang1 != $lang2) {
  959. if (isset($return_arr[$lang2][$lang1])) {
  960. // don't re-calculate what's already been done
  961. $return_arr[$lang1][$lang2]
  962. = $return_arr[$lang2][$lang1];
  963. } else {
  964. // calculate
  965. $return_arr[$lang1][$lang2]
  966. = $this->_normalize_score(
  967. $this->_distance(
  968. $this->_lang_db[$lang1],
  969. $this->_lang_db[$lang2]
  970. )
  971. );
  972. }
  973. }
  974. }
  975. }
  976. return $return_arr;
  977. }
  978. }
  979. /**
  980. * Cluster known languages according to languageSimilarity()
  981. *
  982. * WARNING: this method is EXPERIMENTAL. It is not recommended for common
  983. * use, and it may disappear or its functionality may change in future
  984. * releases without notice.
  985. *
  986. * Uses a nearest neighbor technique to generate the maximum possible
  987. * number of dendograms from the similarity data.
  988. *
  989. * @return array language cluster data
  990. * @throws Text_LanguageDetect_Exception
  991. * @see languageSimilarity()
  992. * @deprecated this function will eventually be removed and placed into
  993. * the model generation class
  994. */
  995. public function clusterLanguages()
  996. {
  997. // todo: set the maximum number of clusters
  998. // return cached result, if any
  999. if (isset($this->_clusters)) {
  1000. return $this->_clusters;
  1001. }
  1002. $langs = array_keys($this->_lang_db);
  1003. $arr = $this->languageSimilarity();
  1004. sort($langs);
  1005. foreach ($langs as $lang) {
  1006. if (!isset($this->_lang_db[$lang])) {
  1007. throw new Text_LanguageDetect_Exception(
  1008. "missing $lang!",
  1009. Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE
  1010. );
  1011. }
  1012. }
  1013. // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
  1014. foreach ($langs as $old_key => $lang1) {
  1015. $langs[$lang1] = $lang1;
  1016. unset($langs[$old_key]);
  1017. }
  1018. $result_data = $really_map = array();
  1019. $i = 0;
  1020. while (count($langs) > 2 && $i++ < 200) {
  1021. $highest_score = -1;
  1022. $highest_key1 = '';
  1023. $highest_key2 = '';
  1024. foreach ($langs as $lang1) {
  1025. foreach ($langs as $lang2) {
  1026. if ($lang1 != $lang2
  1027. && $arr[$lang1][$lang2] > $highest_score
  1028. ) {
  1029. $highest_score = $arr[$lang1][$lang2];
  1030. $highest_key1 = $lang1;
  1031. $highest_key2 = $lang2;
  1032. }
  1033. }
  1034. }
  1035. if (!$highest_key1) {
  1036. // should not ever happen
  1037. throw new Text_LanguageDetect_Exception(
  1038. "no highest key? (step: $i)",
  1039. Text_LanguageDetect_Exception::NO_HIGHEST_KEY
  1040. );
  1041. }
  1042. if ($highest_score == 0) {
  1043. // languages are perfectly dissimilar
  1044. break;
  1045. }
  1046. // $highest_key1 and $highest_key2 are most similar
  1047. $sum1 = array_sum($arr[$highest_key1]);
  1048. $sum2 = array_sum($arr[$highest_key2]);
  1049. // use the score for the one that is most similar to the rest of
  1050. // the field as the score for the group
  1051. // todo: could try averaging or "centroid" method instead
  1052. // seems like that might make more sense
  1053. // actually nearest neighbor may be better for binary searching
  1054. // for "Complete Linkage"/"furthest neighbor"
  1055. // sign should be <
  1056. // for "Single Linkage"/"nearest neighbor" method
  1057. // should should be >
  1058. // results seem to be pretty much the same with either method
  1059. // figure out which to delete and which to replace
  1060. if ($sum1 > $sum2) {
  1061. $replaceme = $highest_key1;
  1062. $deleteme = $highest_key2;
  1063. } else {
  1064. $replaceme = $highest_key2;
  1065. $deleteme = $highest_key1;
  1066. }
  1067. $newkey = $replaceme . ':' . $deleteme;
  1068. // $replaceme is most similar to remaining languages
  1069. // replace $replaceme with '$newkey', deleting $deleteme
  1070. // keep a record of which fork is really which language
  1071. $really_lang = $replaceme;
  1072. while (isset($really_map[$really_lang])) {
  1073. $really_lang = $really_map[$really_lang];
  1074. }
  1075. $really_map[$newkey] = $really_lang;
  1076. // replace the best fitting key, delete the other
  1077. foreach ($arr as $key1 => $arr2) {
  1078. foreach ($arr2 as $key2 => $value2) {
  1079. if ($key2 == $replaceme) {
  1080. $arr[$key1][$newkey] = $arr[$key1][$key2];
  1081. unset($arr[$key1][$key2]);
  1082. // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
  1083. }
  1084. if ($key1 == $replaceme) {
  1085. $arr[$newkey][$key2] = $arr[$key1][$key2];
  1086. unset($arr[$key1][$key2]);
  1087. // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
  1088. }
  1089. if ($key1 == $deleteme || $key2 == $deleteme) {
  1090. // deleting $arr[$key1][$key2]
  1091. unset($arr[$key1][$key2]);
  1092. }
  1093. }
  1094. }
  1095. unset($langs[$highest_key1]);
  1096. unset($langs[$highest_key2]);
  1097. $langs[$newkey] = $newkey;
  1098. // some of these may be overkill
  1099. $result_data[$newkey] = array(
  1100. 'newkey' => $newkey,
  1101. 'count' => $i,
  1102. 'diff' => abs($sum1 - $sum2),
  1103. 'score' => $highest_score,
  1104. 'bestfit' => $replaceme,
  1105. 'otherfit' => $deleteme,
  1106. 'really' => $really_lang,
  1107. );
  1108. }
  1109. $return_val = array(
  1110. 'open_forks' => $langs,
  1111. // the top level of clusters
  1112. // clusters that are mutually exclusive
  1113. // or specified by a specific maximum
  1114. 'fork_data' => $result_data,
  1115. // data for each split
  1116. 'name_map' => $really_map,
  1117. // which cluster is really which language
  1118. // using the nearest neighbor technique, the cluster
  1119. // inherits all of the properties of its most-similar member
  1120. // this keeps track
  1121. );
  1122. // saves the result in the object
  1123. $this->_clusters = $return_val;
  1124. return $return_val;
  1125. }
  1126. /**
  1127. * Perform an intelligent detection based on clusterLanguages()
  1128. *
  1129. * WARNING: this method is EXPERIMENTAL. It is not recommended for common
  1130. * use, and it may disappear or its functionality may change in future
  1131. * releases without notice.
  1132. *
  1133. * This compares the sample text to top the top level of clusters. If the
  1134. * sample is similar to the cluster it will drop down and compare it to the
  1135. * languages in the cluster, and so on until it hits a leaf node.
  1136. *
  1137. * this should find the language in considerably fewer compares
  1138. * (the equivalent of a binary search), however clusterLanguages() is costly
  1139. * and the loss of accuracy from this technique is significant.
  1140. *
  1141. * This method may need to be 'fuzzier' in order to become more accurate.
  1142. *
  1143. * This function could be more useful if the universe of possible languages
  1144. * was very large, however in such cases some method of Bayesian inference
  1145. * might be more helpful.
  1146. *
  1147. * @param string $str input string
  1148. *
  1149. * @return array language scores (only those compared)
  1150. * @throws Text_LanguageDetect_Exception
  1151. * @see clusterLanguages()
  1152. */
  1153. public function clusteredSearch($str)
  1154. {
  1155. // input check
  1156. if (!Text_LanguageDetect_Parser::validateString($str)) {
  1157. return array();
  1158. }
  1159. // clusterLanguages() will return a cached result if possible
  1160. // so it's safe to call it every time
  1161. $result = $this->clusterLanguages();
  1162. $dendogram_start = $result['open_forks'];
  1163. $dendogram_data = $result['fork_data'];
  1164. $dendogram_alias = $result['name_map'];
  1165. $sample_obj = new Text_LanguageDetect_Parser($str);
  1166. $sample_obj->prepareTrigram();
  1167. $sample_obj->setPadStart(!$this->_perl_compatible);
  1168. $sample_obj->analyze();
  1169. $sample_result = $sample_obj->getTrigramRanks();
  1170. $sample_count = count($sample_result);
  1171. // input check
  1172. if ($sample_count == 0) {
  1173. return array();
  1174. }
  1175. $i = 0; // counts the number of steps
  1176. foreach ($dendogram_start as $lang) {
  1177. if (isset($dendogram_alias[$lang])) {
  1178. $lang_key = $dendogram_alias[$lang];
  1179. } else {
  1180. $lang_key = $lang;
  1181. }
  1182. $scores[$lang] = $this->_normalize_score(
  1183. $this->_distance($this->_lang_db[$lang_key], $sample_result),
  1184. $sample_count
  1185. );
  1186. $i++;
  1187. }
  1188. if ($this->_perl_compatible) {
  1189. asort($scores);
  1190. } else {
  1191. arsort($scores);
  1192. }
  1193. $top_score = current($scores);
  1194. $top_key = key($scores);
  1195. // of starting forks, $top_key is the most similar to the sample
  1196. $cur_key = $top_key;
  1197. while (isset($dendogram_data[$cur_key])) {
  1198. $lang1 = $dendogram_data[$cur_key]['bestfit'];
  1199. $lang2 = $dendogram_data[$cur_key]['otherfit'];
  1200. foreach (array($lang1, $lang2) as $lang) {
  1201. if (isset($dendogram_alias[$lang])) {
  1202. $lang_key = $dendogram_alias[$lang];
  1203. } else {
  1204. $lang_key = $lang;
  1205. }
  1206. $scores[$lang] = $this->_normalize_score(
  1207. $this->_distance($this->_lang_db[$lang_key], $sample_result),
  1208. $sample_count
  1209. );
  1210. //todo: does not need to do same comparison again
  1211. }
  1212. $i++;
  1213. if ($scores[$lang1] > $scores[$lang2]) {
  1214. $cur_key = $lang1;
  1215. $loser_key = $lang2;
  1216. } else {
  1217. $cur_key = $lang2;
  1218. $loser_key = $lang1;
  1219. }
  1220. $diff = $scores[$cur_key] - $scores[$loser_key];
  1221. // $cur_key ({$dendogram_alias[$cur_key]}) wins
  1222. // over $loser_key ({$dendogram_alias[$loser_key]})
  1223. // with a difference of $diff
  1224. }
  1225. // found result in $i compares
  1226. // rather than sorting the result, preserve it so that you can see
  1227. // which paths the algorithm decided to take along the tree
  1228. // but sometimes the last item is only the second highest
  1229. if (($this->_perl_compatible && (end($scores) > prev($scores)))
  1230. || (!$this->_perl_compatible && (end($scores) < prev($scores)))
  1231. ) {
  1232. $real_last_score = current($scores);
  1233. $real_last_key = key($scores);
  1234. // swaps the 2nd-to-last item for the last item
  1235. unset($scores[$real_last_key]);
  1236. $scores[$real_last_key] = $real_last_score;
  1237. }
  1238. if (!$this->_perl_compatible) {
  1239. $scores = array_reverse($scores, true);
  1240. // second param requires php > 4.0.3
  1241. }
  1242. return $scores;
  1243. }
  1244. /**
  1245. * UTF8-safe strlen()
  1246. *
  1247. * Returns the numbers of characters (not bytes) in a utf8 string
  1248. *
  1249. * @param string $str string to get the length of
  1250. *
  1251. * @return int number of chars
  1252. */
  1253. public static function utf8strlen($str)
  1254. {
  1255. // utf8_decode() will convert unknown chars to '?', which is actually
  1256. // ideal for counting.
  1257. return strlen(utf8_decode($str));
  1258. // idea stolen from dokuwiki
  1259. }
  1260. /**
  1261. * Returns the unicode value of a utf8 char
  1262. *
  1263. * @param string $char a utf8 (possibly multi-byte) char
  1264. *
  1265. * @return int unicode value
  1266. * @link http://en.wikipedia.org/wiki/UTF-8
  1267. */
  1268. protected function _utf8char2unicode($char)
  1269. {
  1270. // strlen() here will actually get the binary length of a single char
  1271. switch (strlen($char)) {
  1272. case 1:
  1273. // normal ASCII-7 byte
  1274. // 0xxxxxxx --> 0xxxxxxx
  1275. return ord($char{0});
  1276. case 2:
  1277. // 2 byte unicode
  1278. // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
  1279. $z = (ord($char{0}) & 0x000001F) << 6;
  1280. $x = (ord($char{1}) & 0x0000003F);
  1281. return ($z | $x);
  1282. case 3:
  1283. // 3 byte unicode
  1284. // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
  1285. $z = (ord($char{0}) & 0x0000000F) << 12;
  1286. $x1 = (ord($char{1}) & 0x0000003F) << 6;
  1287. $x2 = (ord($char{2}) & 0x0000003F);
  1288. return ($z | $x1 | $x2);
  1289. case 4:
  1290. // 4 byte unicode
  1291. // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
  1292. // 000zzzzz xxxxxxxx xxxxxxxx
  1293. $z1 = (ord($char{0}) & 0x00000007) << 18;
  1294. $z2 = (ord($char{1}) & 0x0000003F) << 12;
  1295. $x1 = (ord($char{2}) & 0x0000003F) << 6;
  1296. $x2 = (ord($char{3}) & 0x0000003F);
  1297. return ($z1 | $z2 | $x1 | $x2);
  1298. }
  1299. }
  1300. /**
  1301. * UTF8-safe fast character iterator
  1302. *
  1303. * Will get the next character starting from $counter, which will then be
  1304. * incremented. If a multi-byte char the bytes will be concatenated and
  1305. * $counter will be incremeted by the number of bytes in the char.
  1306. *
  1307. * @param string $str the string being iterated over
  1308. * @param int $counter the iterator, will increment by reference
  1309. * @param bool $special_convert whether to do special conversions
  1310. *
  1311. * @return char the next (possibly multi-byte) char from $counter
  1312. */
  1313. protected static function _next_char($str, &$counter, $special_convert = false)
  1314. {
  1315. $char = $str{$counter++};
  1316. $ord = ord($char);
  1317. // for a description of the utf8 system see
  1318. // http://www.phpclasses.org/browse/file/5131.html
  1319. // normal ascii one byte char
  1320. if ($ord <= 127) {
  1321. // special conversions needed for this package
  1322. // (that only apply to regular ascii characters)
  1323. // lower case, and convert all non-alphanumeric characters
  1324. // other than "'" to space
  1325. if ($special_convert && $char != ' ' && $char != "'") {
  1326. if ($ord >= 65 && $ord <= 90) { // A-Z
  1327. $char = chr($ord + 32); // lower case
  1328. } elseif ($ord < 97 || $ord > 122) { // NOT a-z
  1329. $char = ' '; // convert to space
  1330. }
  1331. }
  1332. return $char;
  1333. } elseif ($ord >> 5 == 6) { // two-byte char
  1334. // multi-byte chars
  1335. $nextchar = $str{$counter++}; // get next byte
  1336. // lower-casing of non-ascii characters is still incomplete
  1337. if ($special_convert) {
  1338. // lower case latin accented characters
  1339. if ($ord == 195) {
  1340. $nextord = ord($nextchar);
  1341. $nextord_adj = $nextord + 64;
  1342. // for a reference, see
  1343. // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
  1344. // &Agrave; - &THORN; but not &times;
  1345. if ($nextord_adj >= 192
  1346. && $nextord_adj <= 222
  1347. && $nextord_adj != 215
  1348. ) {
  1349. $nextchar = chr($nextord + 32);
  1350. }
  1351. } elseif ($ord == 208) {
  1352. // lower case cyrillic alphabet
  1353. $nextord = ord($nextchar);
  1354. // if A - Pe
  1355. if ($nextord >= 144 && $nextord <= 159) {
  1356. // lower case
  1357. $nextchar = chr($nextord + 32);
  1358. } elseif ($nextord >= 160 && $nextord <= 175) {
  1359. // if Er - Ya
  1360. // lower case
  1361. $char = chr(209); // == $ord++
  1362. $nextchar = chr($nextord - 32);
  1363. }
  1364. }
  1365. }
  1366. // tag on next byte
  1367. return $char . $nextchar;
  1368. } elseif ($ord >> 4 == 14) { // three-byte char
  1369. // tag on next 2 bytes
  1370. return $char . $str{$counter++} . $str{$counter++};
  1371. } elseif ($ord >> 3 == 30) { // four-byte char
  1372. // tag on next 3 bytes
  1373. return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
  1374. } else {
  1375. // error?
  1376. }
  1377. }
  1378. /**
  1379. * Converts an $language input parameter from the configured mode
  1380. * to the language name that is used internally.
  1381. *
  1382. * Works for strings and arrays.
  1383. *
  1384. * @param string|array $lang A language description ("english"/"en"/"eng")
  1385. * @param boolean $convertKey If $lang is an array, setting $key
  1386. * converts the keys to the language name.
  1387. *
  1388. * @return string|array Language name
  1389. */
  1390. protected function _convertFromNameMode($lang, $convertKey = false)
  1391. {
  1392. if ($this->_name_mode == 0) {
  1393. return $lang;
  1394. }
  1395. if ($this->_name_mode == 2) {
  1396. $method = 'code2ToName';
  1397. } else {
  1398. $method = 'code3ToName';
  1399. }
  1400. if (is_string($lang)) {
  1401. return (string)Text_LanguageDetect_ISO639::$method($lang);
  1402. }
  1403. $newlang = array();
  1404. foreach ($lang as $key => $val) {
  1405. if ($convertKey) {
  1406. $newkey = (string)Text_LanguageDetect_ISO639::$method($key);
  1407. $newlang[$newkey] = $val;
  1408. } else {
  1409. $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val);
  1410. }
  1411. }
  1412. return $newlang;
  1413. }
  1414. /**
  1415. * Converts an $language output parameter from the language name that is
  1416. * used internally to the configured mode.
  1417. *
  1418. * Works for strings and arrays.
  1419. *
  1420. * @param string|array $lang A language description ("english"/"en"/"eng")
  1421. * @param boolean $convertKey If $lang is an array, setting $key
  1422. * converts the keys to the language name.
  1423. *
  1424. * @return string|array Language name
  1425. */
  1426. protected function _convertToNameMode($lang, $convertKey = false)
  1427. {
  1428. if ($this->_name_mode == 0) {
  1429. return $lang;
  1430. }
  1431. if ($this->_name_mode == 2) {
  1432. $method = 'nameToCode2';
  1433. } else {
  1434. $method = 'nameToCode3';
  1435. }
  1436. if (is_string($lang)) {
  1437. return Text_LanguageDetect_ISO639::$method($lang);
  1438. }
  1439. $newlang = array();
  1440. foreach ($lang as $key => $val) {
  1441. if ($convertKey) {
  1442. $newkey = Text_LanguageDetect_ISO639::$method($key);
  1443. $newlang[$newkey] = $val;
  1444. } else {
  1445. $newlang[$key] = Text_LanguageDetect_ISO639::$method($val);
  1446. }
  1447. }
  1448. return $newlang;
  1449. }
  1450. }
  1451. /* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */