3 * Part of Text_LanguageDetect
8 * @package Text_LanguageDetect
9 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
10 * @copyright 2006 Nicholas Pisarro
11 * @license BSD http://www.opensource.org/licenses/bsd-license.php
12 * @link http://pear.php.net/package/Text_LanguageDetect/
16 * This class represents a text sample to be parsed.
18 * This separates the analysis of a text sample from the primary LanguageDetect
19 * class. After a new profile has been built, the data can be retrieved using
20 * the accessor functions.
22 * This class is intended to be used by the Text_LanguageDetect class, not
26 * @package Text_LanguageDetect
27 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
28 * @copyright 2006 Nicholas Pisarro
29 * @license BSD http://www.opensource.org/licenses/bsd-license.php
30 * @version Release: @package_version@
31 * @link http://pear.php.net/package/Text_LanguageDetect/
33 class Text_LanguageDetect_Parser extends Text_LanguageDetect
36 * The piece of text being parsed
43 * Stores the trigram frequencies of the sample
47 protected $_trigrams = array();
50 * Stores the trigram ranks of the sample
54 protected $_trigram_ranks = array();
57 * Stores the unicode blocks of the sample
61 protected $_unicode_blocks = array();
64 * Whether the parser should compile the unicode ranges
68 protected $_compile_unicode = false;
71 * Whether the parser should compile trigrams
75 protected $_compile_trigram = false;
78 * Whether the trigram parser should pad the beginning of the string
82 protected $_trigram_pad_start = false;
85 * Whether the unicode parser should skip non-alphabetical ascii chars
89 protected $_unicode_skip_symbols = true;
94 * @param string $string string to be parsed
96 public function __construct($string)
98 $this->_string = $string;
102 * PHP 4 constructor for backwards compatibility.
104 * @param string $string string to be parsed
108 public function Text_LanguageDetect_Parser($string)
110 self::__construct($string);
114 * Returns true if a string is suitable for parsing
116 * @param string $str input string to test
118 * @return bool true if acceptable, false if not
120 public static function validateString($str)
122 if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) {
130 * Turn on/off trigram counting
132 * @param bool $bool true for on, false for off
136 public function prepareTrigram($bool = true)
138 $this->_compile_trigram = $bool;
142 * Turn on/off unicode block counting
144 * @param bool $bool true for on, false for off
148 public function prepareUnicode($bool = true)
150 $this->_compile_unicode = $bool;
154 * Turn on/off padding the beginning of the sample string
156 * @param bool $bool true for on, false for off
160 public function setPadStart($bool = true)
162 $this->_trigram_pad_start = $bool;
166 * Should the unicode block counter skip non-alphabetical ascii chars?
168 * @param bool $bool true for on, false for off
172 public function setUnicodeSkipSymbols($bool = true)
174 $this->_unicode_skip_symbols = $bool;
178 * Returns the trigram ranks for the text sample
180 * @return array Trigram ranks in the text sample
182 public function getTrigramRanks()
184 return $this->_trigram_ranks;
188 * Return the trigram freqency table
190 * Only used in testing to make sure the parser is working
192 * @return array Trigram freqencies in the text sample
194 public function getTrigramFreqs()
196 return $this->_trigram;
200 * Returns the array of unicode blocks
202 * @return array Unicode blocks in the text sample
204 public function getUnicodeBlocks()
206 return $this->_unicode_blocks;
210 * Executes the parsing operation
212 * Be sure to call the set*() functions to set options and the
213 * prepare*() functions first to tell it what kind of data to compute
215 * Afterwards the get*() functions can be used to access the compiled
220 public function analyze()
222 $len = strlen($this->_string);
227 if ($this->_compile_unicode) {
228 $blocks = $this->_read_unicode_block_db();
229 $block_count = count($blocks);
232 $unicode_chars = array();
236 if ($this->_compile_trigram) {
237 // initialize them as blank so the parser will skip the first two
238 // (since it skips trigrams with more than 2 contiguous spaces)
243 // if it finds a valid trigram to start and the start pad option is
244 // off, then set a variable that will be used to reduce this
245 // trigram after parsing has finished
246 if (!$this->_trigram_pad_start) {
247 $a = $this->_next_char($this->_string, $byte_counter, true);
250 $b = $this->_next_char($this->_string, $byte_counter, true);
260 while ($byte_counter < $len) {
261 $char = $this->_next_char($this->_string, $byte_counter, true);
264 // language trigram detection
265 if ($this->_compile_trigram) {
266 if (!($b == ' ' && ($a == ' ' || $char == ' '))) {
267 if (!isset($this->_trigram[$a . $b . $char])) {
268 $this->_trigram[$a . $b . $char] = 1;
270 $this->_trigram[$a . $b . $char]++;
278 // unicode block detection
279 if ($this->_compile_unicode) {
280 if ($this->_unicode_skip_symbols
281 && strlen($char) == 1
282 && ($char < 'A' || $char > 'z'
283 || ($char > 'Z' && $char < 'a'))
285 ) { // does not skip the apostrophe
286 // since it's included in the language
293 // build an array of all the characters
294 if (isset($unicode_chars[$char])) {
295 $unicode_chars[$char]++;
297 $unicode_chars[$char] = 1;
301 // todo: add byte detection here
305 if ($this->_compile_unicode) {
306 foreach ($unicode_chars as $utf8_char => $count) {
307 $search_result = $this->_unicode_block_name(
308 $this->_utf8char2unicode($utf8_char), $blocks, $block_count
311 if ($search_result != -1) {
312 $block_name = $search_result[2];
314 $block_name = '[Malformatted]';
317 if (isset($this->_unicode_blocks[$block_name])) {
318 $this->_unicode_blocks[$block_name] += $count;
320 $this->_unicode_blocks[$block_name] = $count;
327 if ($this->_compile_trigram) {
330 if (!isset($this->_trigram["$a$b "])) {
331 $this->_trigram["$a$b "] = 1;
333 $this->_trigram["$a$b "]++;
337 // perl compatibility; Language::Guess does not pad the beginning
339 if (isset($dropone)) {
340 if ($this->_trigram[$dropone] == 1) {
341 unset($this->_trigram[$dropone]);
343 $this->_trigram[$dropone]--;
347 if (!empty($this->_trigram)) {
348 $this->_trigram_ranks = $this->_arr_rank($this->_trigram);
350 $this->_trigram_ranks = array();
356 /* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */