[ Index ]

PHP Cross Reference of phpBB-3.3.0-deutsch

title

Body

[close]

/phpbb/search/ -> fulltext_native.php (source)

   1  <?php
   2  /**
   3  *
   4  * This file is part of the phpBB Forum Software package.
   5  *
   6  * @copyright (c) phpBB Limited <https://www.phpbb.com>
   7  * @license GNU General Public License, version 2 (GPL-2.0)
   8  *
   9  * For full copyright and license information, please see
  10  * the docs/CREDITS.txt file.
  11  *
  12  */
  13  
  14  namespace phpbb\search;
  15  
  16  /**
  17  * phpBB's own db driven fulltext search, version 2
  18  */
  19  class fulltext_native extends \phpbb\search\base
  20  {
  21      const UTF8_HANGUL_FIRST = "\xEA\xB0\x80";
  22      const UTF8_HANGUL_LAST = "\xED\x9E\xA3";
  23      const UTF8_CJK_FIRST = "\xE4\xB8\x80";
  24      const UTF8_CJK_LAST = "\xE9\xBE\xBB";
  25      const UTF8_CJK_B_FIRST = "\xF0\xA0\x80\x80";
  26      const UTF8_CJK_B_LAST = "\xF0\xAA\x9B\x96";
  27  
  28      /**
  29       * Associative array holding index stats
  30       * @var array
  31       */
  32      protected $stats = array();
  33  
  34      /**
  35       * Associative array stores the min and max word length to be searched
  36       * @var array
  37       */
  38      protected $word_length = array();
  39  
  40      /**
  41       * Contains tidied search query.
  42       * Operators are prefixed in search query and common words excluded
  43       * @var string
  44       */
  45      protected $search_query;
  46  
  47      /**
  48       * Contains common words.
  49       * Common words are words with length less/more than min/max length
  50       * @var array
  51       */
  52      protected $common_words = array();
  53  
  54      /**
  55       * Post ids of posts containing words that are to be included
  56       * @var array
  57       */
  58      protected $must_contain_ids = array();
  59  
  60      /**
  61       * Post ids of posts containing words that should not be included
  62       * @var array
  63       */
  64      protected $must_not_contain_ids = array();
  65  
  66      /**
  67       * Post ids of posts containing at least one word that needs to be excluded
  68       * @var array
  69       */
  70      protected $must_exclude_one_ids = array();
  71  
  72      /**
  73       * Relative path to board root
  74       * @var string
  75       */
  76      protected $phpbb_root_path;
  77  
  78      /**
  79       * PHP Extension
  80       * @var string
  81       */
  82      protected $php_ext;
  83  
  84      /**
  85       * Config object
  86       * @var \phpbb\config\config
  87       */
  88      protected $config;
  89  
  90      /**
  91       * Database connection
  92       * @var \phpbb\db\driver\driver_interface
  93       */
  94      protected $db;
  95  
  96      /**
  97       * phpBB event dispatcher object
  98       * @var \phpbb\event\dispatcher_interface
  99       */
 100      protected $phpbb_dispatcher;
 101  
 102      /**
 103       * User object
 104       * @var \phpbb\user
 105       */
 106      protected $user;
 107  
 108      /**
 109      * Initialises the fulltext_native search backend with min/max word length
 110      *
 111      * @param    boolean|string    &$error    is passed by reference and should either be set to false on success or an error message on failure
 112      * @param    string    $phpbb_root_path    phpBB root path
 113      * @param    string    $phpEx    PHP file extension
 114      * @param    \phpbb\auth\auth    $auth    Auth object
 115      * @param    \phpbb\config\config    $config    Config object
 116      * @param    \phpbb\db\driver\driver_interface    $db    Database object
 117      * @param    \phpbb\user    $user    User object
 118      * @param    \phpbb\event\dispatcher_interface    $phpbb_dispatcher    Event dispatcher object
 119      */
 120  	public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher)
 121      {
 122          $this->phpbb_root_path = $phpbb_root_path;
 123          $this->php_ext = $phpEx;
 124          $this->config = $config;
 125          $this->db = $db;
 126          $this->phpbb_dispatcher = $phpbb_dispatcher;
 127          $this->user = $user;
 128  
 129          $this->word_length = array('min' => (int) $this->config['fulltext_native_min_chars'], 'max' => (int) $this->config['fulltext_native_max_chars']);
 130  
 131          /**
 132          * Load the UTF tools
 133          */
 134          if (!function_exists('utf8_decode_ncr'))
 135          {
 136              include($this->phpbb_root_path . 'includes/utf/utf_tools.' . $this->php_ext);
 137          }
 138  
 139          $error = false;
 140      }
 141  
 142      /**
 143      * Returns the name of this search backend to be displayed to administrators
 144      *
 145      * @return string Name
 146      */
 147  	public function get_name()
 148      {
 149          return 'phpBB Native Fulltext';
 150      }
 151  
 152      /**
 153       * Returns the search_query
 154       *
 155       * @return string search query
 156       */
 157  	public function get_search_query()
 158      {
 159          return $this->search_query;
 160      }
 161  
 162      /**
 163       * Returns the common_words array
 164       *
 165       * @return array common words that are ignored by search backend
 166       */
 167  	public function get_common_words()
 168      {
 169          return $this->common_words;
 170      }
 171  
 172      /**
 173       * Returns the word_length array
 174       *
 175       * @return array min and max word length for searching
 176       */
 177  	public function get_word_length()
 178      {
 179          return $this->word_length;
 180      }
 181  
 182      /**
 183      * This function fills $this->search_query with the cleaned user search query
 184      *
 185      * If $terms is 'any' then the words will be extracted from the search query
 186      * and combined with | inside brackets. They will afterwards be treated like
 187      * an standard search query.
 188      *
 189      * Then it analyses the query and fills the internal arrays $must_not_contain_ids,
 190      * $must_contain_ids and $must_exclude_one_ids which are later used by keyword_search()
 191      *
 192      * @param    string    $keywords    contains the search query string as entered by the user
 193      * @param    string    $terms        is either 'all' (use search query as entered, default words to 'must be contained in post')
 194      *     or 'any' (find all posts containing at least one of the given words)
 195      * @return    boolean                false if no valid keywords were found and otherwise true
 196      */
 197  	public function split_keywords($keywords, $terms)
 198      {
 199          $tokens = '+-|()* ';
 200  
 201          $keywords = trim($this->cleanup($keywords, $tokens));
 202  
 203          // allow word|word|word without brackets
 204          if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false))
 205          {
 206              $keywords = '(' . $keywords . ')';
 207          }
 208  
 209          $open_bracket = $space = false;
 210          for ($i = 0, $n = strlen($keywords); $i < $n; $i++)
 211          {
 212              if ($open_bracket !== false)
 213              {
 214                  switch ($keywords[$i])
 215                  {
 216                      case ')':
 217                          if ($open_bracket + 1 == $i)
 218                          {
 219                              $keywords[$i - 1] = '|';
 220                              $keywords[$i] = '|';
 221                          }
 222                          $open_bracket = false;
 223                      break;
 224                      case '(':
 225                          $keywords[$i] = '|';
 226                      break;
 227                      case '+':
 228                      case '-':
 229                      case ' ':
 230                          $keywords[$i] = '|';
 231                      break;
 232                      case '*':
 233                          // $i can never be 0 here since $open_bracket is initialised to false
 234                          if (strpos($tokens, $keywords[$i - 1]) !== false && ($i + 1 === $n || strpos($tokens, $keywords[$i + 1]) !== false))
 235                          {
 236                              $keywords[$i] = '|';
 237                          }
 238                      break;
 239                  }
 240              }
 241              else
 242              {
 243                  switch ($keywords[$i])
 244                  {
 245                      case ')':
 246                          $keywords[$i] = ' ';
 247                      break;
 248                      case '(':
 249                          $open_bracket = $i;
 250                          $space = false;
 251                      break;
 252                      case '|':
 253                          $keywords[$i] = ' ';
 254                      break;
 255                      case '-':
 256                      case '+':
 257                          $space = $keywords[$i];
 258                      break;
 259                      case ' ':
 260                          if ($space !== false)
 261                          {
 262                              $keywords[$i] = $space;
 263                          }
 264                      break;
 265                      default:
 266                          $space = false;
 267                  }
 268              }
 269          }
 270  
 271          if ($open_bracket !== false)
 272          {
 273              $keywords .= ')';
 274          }
 275  
 276          $match = array(
 277              '#  +#',
 278              '#\|\|+#',
 279              '#(\+|\-)(?:\+|\-)+#',
 280              '#\(\|#',
 281              '#\|\)#',
 282          );
 283          $replace = array(
 284              ' ',
 285              '|',
 286              '$1',
 287              '(',
 288              ')',
 289          );
 290  
 291          $keywords = preg_replace($match, $replace, $keywords);
 292          $num_keywords = count(explode(' ', $keywords));
 293  
 294          // We limit the number of allowed keywords to minimize load on the database
 295          if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords'])
 296          {
 297              trigger_error($this->user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], $num_keywords));
 298          }
 299  
 300          // $keywords input format: each word separated by a space, words in a bracket are not separated
 301  
 302          // the user wants to search for any word, convert the search query
 303          if ($terms == 'any')
 304          {
 305              $words = array();
 306  
 307              preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words);
 308              if (count($words[1]))
 309              {
 310                  $keywords = '(' . implode('|', $words[1]) . ')';
 311              }
 312          }
 313  
 314          // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index)
 315          $match = '#\*(?!$|\s)#';
 316          $replace = '$1';
 317          $keywords = preg_replace($match, $replace, $keywords);
 318  
 319          // Only allow one wildcard in the search query to limit the database load
 320          $match = '#\*#';
 321          $replace = '$1';
 322          $count_wildcards = substr_count($keywords, '*');
 323  
 324          // Reverse the string to remove all wildcards except the first one
 325          $keywords = strrev(preg_replace($match, $replace, strrev($keywords), $count_wildcards - 1));
 326          unset($count_wildcards);
 327  
 328          // set the search_query which is shown to the user
 329          $this->search_query = $keywords;
 330  
 331          $exact_words = array();
 332          preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words);
 333          $exact_words = $exact_words[1];
 334  
 335          $common_ids = $words = array();
 336  
 337          if (count($exact_words))
 338          {
 339              $sql = 'SELECT word_id, word_text, word_common
 340                  FROM ' . SEARCH_WORDLIST_TABLE . '
 341                  WHERE ' . $this->db->sql_in_set('word_text', $exact_words) . '
 342                  ORDER BY word_count ASC';
 343              $result = $this->db->sql_query($sql);
 344  
 345              // store an array of words and ids, remove common words
 346              while ($row = $this->db->sql_fetchrow($result))
 347              {
 348                  if ($row['word_common'])
 349                  {
 350                      $this->common_words[] = $row['word_text'];
 351                      $common_ids[$row['word_text']] = (int) $row['word_id'];
 352                      continue;
 353                  }
 354  
 355                  $words[$row['word_text']] = (int) $row['word_id'];
 356              }
 357              $this->db->sql_freeresult($result);
 358          }
 359  
 360          // Handle +, - without preceding whitespace character
 361          $match        = array('#(\S)\+#', '#(\S)-#');
 362          $replace    = array('$1 +', '$1 +');
 363  
 364          $keywords = preg_replace($match, $replace, $keywords);
 365  
 366          // now analyse the search query, first split it using the spaces
 367          $query = explode(' ', $keywords);
 368  
 369          $this->must_contain_ids = array();
 370          $this->must_not_contain_ids = array();
 371          $this->must_exclude_one_ids = array();
 372  
 373          foreach ($query as $word)
 374          {
 375              if (empty($word))
 376              {
 377                  continue;
 378              }
 379  
 380              // words which should not be included
 381              if ($word[0] == '-')
 382              {
 383                  $word = substr($word, 1);
 384  
 385                  // a group of which at least one may not be in the resulting posts
 386                  if ($word[0] == '(')
 387                  {
 388                      $word = array_unique(explode('|', substr($word, 1, -1)));
 389                      $mode = 'must_exclude_one';
 390                  }
 391                  // one word which should not be in the resulting posts
 392                  else
 393                  {
 394                      $mode = 'must_not_contain';
 395                  }
 396                  $ignore_no_id = true;
 397              }
 398              // words which have to be included
 399              else
 400              {
 401                  // no prefix is the same as a +prefix
 402                  if ($word[0] == '+')
 403                  {
 404                      $word = substr($word, 1);
 405                  }
 406  
 407                  // a group of words of which at least one word should be in every resulting post
 408                  if ($word[0] == '(')
 409                  {
 410                      $word = array_unique(explode('|', substr($word, 1, -1)));
 411                  }
 412                  $ignore_no_id = false;
 413                  $mode = 'must_contain';
 414              }
 415  
 416              if (empty($word))
 417              {
 418                  continue;
 419              }
 420  
 421              // if this is an array of words then retrieve an id for each
 422              if (is_array($word))
 423              {
 424                  $non_common_words = array();
 425                  $id_words = array();
 426                  foreach ($word as $i => $word_part)
 427                  {
 428                      if (strpos($word_part, '*') !== false)
 429                      {
 430                          $len = utf8_strlen(str_replace('*', '', $word_part));
 431                          if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])
 432                          {
 433                              $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\'';
 434                              $non_common_words[] = $word_part;
 435                          }
 436                          else
 437                          {
 438                              $this->common_words[] = $word_part;
 439                          }
 440                      }
 441                      else if (isset($words[$word_part]))
 442                      {
 443                          $id_words[] = $words[$word_part];
 444                          $non_common_words[] = $word_part;
 445                      }
 446                      else
 447                      {
 448                          $len = utf8_strlen($word_part);
 449                          if ($len < $this->word_length['min'] || $len > $this->word_length['max'])
 450                          {
 451                              $this->common_words[] = $word_part;
 452                          }
 453                      }
 454                  }
 455                  if (count($id_words))
 456                  {
 457                      sort($id_words);
 458                      if (count($id_words) > 1)
 459                      {
 460                          $this->{$mode . '_ids'}[] = $id_words;
 461                      }
 462                      else
 463                      {
 464                          $mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode;
 465                          $this->{$mode . '_ids'}[] = $id_words[0];
 466                      }
 467                  }
 468                  // throw an error if we shall not ignore unexistant words
 469                  else if (!$ignore_no_id && count($non_common_words))
 470                  {
 471                      trigger_error(sprintf($this->user->lang['WORDS_IN_NO_POST'], implode($this->user->lang['COMMA_SEPARATOR'], $non_common_words)));
 472                  }
 473                  unset($non_common_words);
 474              }
 475              // else we only need one id
 476              else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word]))
 477              {
 478                  if ($wildcard)
 479                  {
 480                      $len = utf8_strlen(str_replace('*', '', $word));
 481                      if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])
 482                      {
 483                          $this->{$mode . '_ids'}[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word)) . '\'';
 484                      }
 485                      else
 486                      {
 487                          $this->common_words[] = $word;
 488                      }
 489                  }
 490                  else
 491                  {
 492                      $this->{$mode . '_ids'}[] = $words[$word];
 493                  }
 494              }
 495              else
 496              {
 497                  if (!isset($common_ids[$word]))
 498                  {
 499                      $len = utf8_strlen($word);
 500                      if ($len < $this->word_length['min'] || $len > $this->word_length['max'])
 501                      {
 502                          $this->common_words[] = $word;
 503                      }
 504                  }
 505              }
 506          }
 507  
 508          // Return true if all words are not common words
 509          if (count($exact_words) - count($this->common_words) > 0)
 510          {
 511              return true;
 512          }
 513          return false;
 514      }
 515  
 516      /**
 517      * Performs a search on keywords depending on display specific params. You have to run split_keywords() first
 518      *
 519      * @param    string        $type                contains either posts or topics depending on what should be searched for
 520      * @param    string        $fields                contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
 521      * @param    string        $terms                is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
 522      * @param    array        $sort_by_sql        contains SQL code for the ORDER BY part of a query
 523      * @param    string        $sort_key            is the key of $sort_by_sql for the selected sorting
 524      * @param    string        $sort_dir            is either a or d representing ASC and DESC
 525      * @param    string        $sort_days            specifies the maximum amount of days a post may be old
 526      * @param    array        $ex_fid_ary            specifies an array of forum ids which should not be searched
 527      * @param    string        $post_visibility    specifies which types of posts the user can view in which forums
 528      * @param    int            $topic_id            is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
 529      * @param    array        $author_ary            an array of author ids if the author should be ignored during the search the array is empty
 530      * @param    string        $author_name        specifies the author match, when ANONYMOUS is also a search-match
 531      * @param    array        &$id_ary            passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
 532      * @param    int            $start                indicates the first index of the page
 533      * @param    int            $per_page            number of ids each page is supposed to contain
 534      * @return    boolean|int                        total number of results
 535      */
 536  	public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
 537      {
 538          // No keywords? No posts.
 539          if (empty($this->search_query))
 540          {
 541              return false;
 542          }
 543  
 544          // we can't search for negatives only
 545          if (empty($this->must_contain_ids))
 546          {
 547              return false;
 548          }
 549  
 550          $must_contain_ids = $this->must_contain_ids;
 551          $must_not_contain_ids = $this->must_not_contain_ids;
 552          $must_exclude_one_ids = $this->must_exclude_one_ids;
 553  
 554          sort($must_contain_ids);
 555          sort($must_not_contain_ids);
 556          sort($must_exclude_one_ids);
 557  
 558          // generate a search_key from all the options to identify the results
 559          $search_key_array = array(
 560              serialize($must_contain_ids),
 561              serialize($must_not_contain_ids),
 562              serialize($must_exclude_one_ids),
 563              $type,
 564              $fields,
 565              $terms,
 566              $sort_days,
 567              $sort_key,
 568              $topic_id,
 569              implode(',', $ex_fid_ary),
 570              $post_visibility,
 571              implode(',', $author_ary),
 572              $author_name,
 573          );
 574  
 575          /**
 576          * Allow changing the search_key for cached results
 577          *
 578          * @event core.search_native_by_keyword_modify_search_key
 579          * @var    array    search_key_array    Array with search parameters to generate the search_key
 580          * @var    array    must_contain_ids    Array with post ids of posts containing words that are to be included
 581          * @var    array    must_not_contain_ids    Array with post ids of posts containing words that should not be included
 582          * @var    array    must_exclude_one_ids    Array with post ids of posts containing at least one word that needs to be excluded
 583          * @var    string    type                Searching type ('posts', 'topics')
 584          * @var    string    fields                Searching fields ('titleonly', 'msgonly', 'firstpost', 'all')
 585          * @var    string    terms                Searching terms ('all', 'any')
 586          * @var    int        sort_days            Time, in days, of the oldest possible post to list
 587          * @var    string    sort_key            The sort type used from the possible sort types
 588          * @var    int        topic_id            Limit the search to this topic_id only
 589          * @var    array    ex_fid_ary            Which forums not to search on
 590          * @var    string    post_visibility        Post visibility data
 591          * @var    array    author_ary            Array of user_id containing the users to filter the results to
 592          * @since 3.1.7-RC1
 593          */
 594          $vars = array(
 595              'search_key_array',
 596              'must_contain_ids',
 597              'must_not_contain_ids',
 598              'must_exclude_one_ids',
 599              'type',
 600              'fields',
 601              'terms',
 602              'sort_days',
 603              'sort_key',
 604              'topic_id',
 605              'ex_fid_ary',
 606              'post_visibility',
 607              'author_ary',
 608          );
 609          extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_keyword_modify_search_key', compact($vars)));
 610  
 611          $search_key = md5(implode('#', $search_key_array));
 612  
 613          // try reading the results from cache
 614          $total_results = 0;
 615          if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == SEARCH_RESULT_IN_CACHE)
 616          {
 617              return $total_results;
 618          }
 619  
 620          $id_ary = array();
 621  
 622          $sql_where = array();
 623          $m_num = 0;
 624          $w_num = 0;
 625  
 626          $sql_array = array(
 627              'SELECT'    => ($type == 'posts') ? 'p.post_id' : 'p.topic_id',
 628              'FROM'        => array(
 629                  SEARCH_WORDMATCH_TABLE    => array(),
 630                  SEARCH_WORDLIST_TABLE    => array(),
 631              ),
 632              'LEFT_JOIN' => array(array(
 633                  'FROM'    => array(POSTS_TABLE => 'p'),
 634                  'ON'    => 'm0.post_id = p.post_id',
 635              )),
 636          );
 637  
 638          $title_match = '';
 639          $left_join_topics = false;
 640          $group_by = true;
 641          // Build some display specific sql strings
 642          switch ($fields)
 643          {
 644              case 'titleonly':
 645                  $title_match = 'title_match = 1';
 646                  $group_by = false;
 647              // no break
 648              case 'firstpost':
 649                  $left_join_topics = true;
 650                  $sql_where[] = 'p.post_id = t.topic_first_post_id';
 651              break;
 652  
 653              case 'msgonly':
 654                  $title_match = 'title_match = 0';
 655                  $group_by = false;
 656              break;
 657          }
 658  
 659          if ($type == 'topics')
 660          {
 661              $left_join_topics = true;
 662              $group_by = true;
 663          }
 664  
 665          /**
 666          * @todo Add a query optimizer (handle stuff like "+(4|3) +4")
 667          */
 668  
 669          foreach ($this->must_contain_ids as $subquery)
 670          {
 671              if (is_array($subquery))
 672              {
 673                  $group_by = true;
 674  
 675                  $word_id_sql = array();
 676                  $word_ids = array();
 677                  foreach ($subquery as $id)
 678                  {
 679                      if (is_string($id))
 680                      {
 681                          $sql_array['LEFT_JOIN'][] = array(
 682                              'FROM'    => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
 683                              'ON'    => "w$w_num.word_text LIKE $id"
 684                          );
 685                          $word_ids[] = "w$w_num.word_id";
 686  
 687                          $w_num++;
 688                      }
 689                      else
 690                      {
 691                          $word_ids[] = $id;
 692                      }
 693                  }
 694  
 695                  $sql_where[] = $this->db->sql_in_set("m$m_num.word_id", $word_ids);
 696  
 697                  unset($word_id_sql);
 698                  unset($word_ids);
 699              }
 700              else if (is_string($subquery))
 701              {
 702                  $sql_array['FROM'][SEARCH_WORDLIST_TABLE][] = 'w' . $w_num;
 703  
 704                  $sql_where[] = "w$w_num.word_text LIKE $subquery";
 705                  $sql_where[] = "m$m_num.word_id = w$w_num.word_id";
 706  
 707                  $group_by = true;
 708                  $w_num++;
 709              }
 710              else
 711              {
 712                  $sql_where[] = "m$m_num.word_id = $subquery";
 713              }
 714  
 715              $sql_array['FROM'][SEARCH_WORDMATCH_TABLE][] = 'm' . $m_num;
 716  
 717              if ($title_match)
 718              {
 719                  $sql_where[] = "m$m_num.$title_match";
 720              }
 721  
 722              if ($m_num != 0)
 723              {
 724                  $sql_where[] = "m$m_num.post_id = m0.post_id";
 725              }
 726              $m_num++;
 727          }
 728  
 729          foreach ($this->must_not_contain_ids as $key => $subquery)
 730          {
 731              if (is_string($subquery))
 732              {
 733                  $sql_array['LEFT_JOIN'][] = array(
 734                      'FROM'    => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
 735                      'ON'    => "w$w_num.word_text LIKE $subquery"
 736                  );
 737  
 738                  $this->must_not_contain_ids[$key] = "w$w_num.word_id";
 739  
 740                  $group_by = true;
 741                  $w_num++;
 742              }
 743          }
 744  
 745          if (count($this->must_not_contain_ids))
 746          {
 747              $sql_array['LEFT_JOIN'][] = array(
 748                  'FROM'    => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),
 749                  'ON'    => $this->db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id"
 750              );
 751  
 752              $sql_where[] = "m$m_num.word_id IS NULL";
 753              $m_num++;
 754          }
 755  
 756          foreach ($this->must_exclude_one_ids as $ids)
 757          {
 758              $is_null_joins = array();
 759              foreach ($ids as $id)
 760              {
 761                  if (is_string($id))
 762                  {
 763                      $sql_array['LEFT_JOIN'][] = array(
 764                          'FROM'    => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
 765                          'ON'    => "w$w_num.word_text LIKE $id"
 766                      );
 767                      $id = "w$w_num.word_id";
 768  
 769                      $group_by = true;
 770                      $w_num++;
 771                  }
 772  
 773                  $sql_array['LEFT_JOIN'][] = array(
 774                      'FROM'    => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),
 775                      'ON'    => "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '')
 776                  );
 777                  $is_null_joins[] = "m$m_num.word_id IS NULL";
 778  
 779                  $m_num++;
 780              }
 781              $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')';
 782          }
 783  
 784          $sql_where[] = $post_visibility;
 785  
 786          $search_query = $this->search_query;
 787          $must_exclude_one_ids = $this->must_exclude_one_ids;
 788          $must_not_contain_ids = $this->must_not_contain_ids;
 789          $must_contain_ids = $this->must_contain_ids;
 790  
 791          $sql_sort_table = $sql_sort_join = $sql_match = $sql_match_where = $sql_sort = '';
 792  
 793          /**
 794          * Allow changing the query used for counting for posts using fulltext_native
 795          *
 796          * @event core.search_native_keywords_count_query_before
 797          * @var    string    search_query            The parsed keywords used for this search
 798          * @var    array    must_not_contain_ids    Ids that cannot be taken into account for the results
 799          * @var    array    must_exclude_one_ids    Ids that cannot be on the results
 800          * @var    array    must_contain_ids        Ids that must be on the results
 801          * @var    int        total_results            The previous result count for the format of the query
 802          *                                        Set to 0 to force a re-count
 803          * @var    array    sql_array                The data on how to search in the DB at this point
 804          * @var    bool    left_join_topics        Whether or not TOPICS_TABLE should be CROSS JOIN'ED
 805          * @var    array    author_ary                Array of user_id containing the users to filter the results to
 806          * @var    string    author_name                An extra username to search on (!empty(author_ary) must be true, to be relevant)
 807          * @var    array    ex_fid_ary                Which forums not to search on
 808          * @var    int        topic_id                Limit the search to this topic_id only
 809          * @var    string    sql_sort_table            Extra tables to include in the SQL query.
 810          *                                        Used in conjunction with sql_sort_join
 811          * @var    string    sql_sort_join            SQL conditions to join all the tables used together.
 812          *                                        Used in conjunction with sql_sort_table
 813          * @var    int        sort_days                Time, in days, of the oldest possible post to list
 814          * @var    string    sql_where                An array of the current WHERE clause conditions
 815          * @var    string    sql_match                Which columns to do the search on
 816          * @var    string    sql_match_where            Extra conditions to use to properly filter the matching process
 817          * @var    bool    group_by                Whether or not the SQL query requires a GROUP BY for the elements in the SELECT clause
 818          * @var    string    sort_by_sql                The possible predefined sort types
 819          * @var    string    sort_key                The sort type used from the possible sort types
 820          * @var    string    sort_dir                "a" for ASC or "d" dor DESC for the sort order used
 821          * @var    string    sql_sort                The result SQL when processing sort_by_sql + sort_key + sort_dir
 822          * @var    int        start                    How many posts to skip in the search results (used for pagination)
 823          * @since 3.1.5-RC1
 824          */
 825          $vars = array(
 826              'search_query',
 827              'must_not_contain_ids',
 828              'must_exclude_one_ids',
 829              'must_contain_ids',
 830              'total_results',
 831              'sql_array',
 832              'left_join_topics',
 833              'author_ary',
 834              'author_name',
 835              'ex_fid_ary',
 836              'topic_id',
 837              'sql_sort_table',
 838              'sql_sort_join',
 839              'sort_days',
 840              'sql_where',
 841              'sql_match',
 842              'sql_match_where',
 843              'group_by',
 844              'sort_by_sql',
 845              'sort_key',
 846              'sort_dir',
 847              'sql_sort',
 848              'start',
 849          );
 850          extract($this->phpbb_dispatcher->trigger_event('core.search_native_keywords_count_query_before', compact($vars)));
 851  
 852          if ($topic_id)
 853          {
 854              $sql_where[] = 'p.topic_id = ' . $topic_id;
 855          }
 856  
 857          if (count($author_ary))
 858          {
 859              if ($author_name)
 860              {
 861                  // first one matches post of registered users, second one guests and deleted users
 862                  $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')';
 863              }
 864              else
 865              {
 866                  $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary);
 867              }
 868              $sql_where[] = $sql_author;
 869          }
 870  
 871          if (count($ex_fid_ary))
 872          {
 873              $sql_where[] = $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true);
 874          }
 875  
 876          if ($sort_days)
 877          {
 878              $sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400));
 879          }
 880  
 881          $sql_array['WHERE'] = implode(' AND ', $sql_where);
 882  
 883          $is_mysql = false;
 884          // if the total result count is not cached yet, retrieve it from the db
 885          if (!$total_results)
 886          {
 887              $sql = '';
 888              $sql_array_count = $sql_array;
 889  
 890              if ($left_join_topics)
 891              {
 892                  $sql_array_count['LEFT_JOIN'][] = array(
 893                      'FROM'    => array(TOPICS_TABLE => 't'),
 894                      'ON'    => 'p.topic_id = t.topic_id'
 895                  );
 896              }
 897  
 898              switch ($this->db->get_sql_layer())
 899              {
 900                  case 'mysqli':
 901  
 902                      // 3.x does not support SQL_CALC_FOUND_ROWS
 903                      // $sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];
 904                      $is_mysql = true;
 905  
 906                  break;
 907  
 908                  case 'sqlite3':
 909                      $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id';
 910                      $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results
 911                              FROM (' . $this->db->sql_build_query('SELECT', $sql_array_count) . ')';
 912  
 913                  // no break
 914  
 915                  default:
 916                      $sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results';
 917                      $sql = (!$sql) ? $this->db->sql_build_query('SELECT', $sql_array_count) : $sql;
 918  
 919                      $result = $this->db->sql_query($sql);
 920                      $total_results = (int) $this->db->sql_fetchfield('total_results');
 921                      $this->db->sql_freeresult($result);
 922  
 923                      if (!$total_results)
 924                      {
 925                          return false;
 926                      }
 927                  break;
 928              }
 929  
 930              unset($sql_array_count, $sql);
 931          }
 932  
 933          // Build sql strings for sorting
 934          $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');
 935  
 936          switch ($sql_sort[0])
 937          {
 938              case 'u':
 939                  $sql_array['FROM'][USERS_TABLE] = 'u';
 940                  $sql_where[] = 'u.user_id = p.poster_id ';
 941              break;
 942  
 943              case 't':
 944                  $left_join_topics = true;
 945              break;
 946  
 947              case 'f':
 948                  $sql_array['FROM'][FORUMS_TABLE] = 'f';
 949                  $sql_where[] = 'f.forum_id = p.forum_id';
 950              break;
 951          }
 952  
 953          if ($left_join_topics)
 954          {
 955              $sql_array['LEFT_JOIN'][] = array(
 956                  'FROM'    => array(TOPICS_TABLE => 't'),
 957                  'ON'    => 'p.topic_id = t.topic_id'
 958              );
 959          }
 960  
 961          // if using mysql and the total result count is not calculated yet, get it from the db
 962          if (!$total_results && $is_mysql)
 963          {
 964              // Also count rows for the query as if there was not LIMIT. Add SQL_CALC_FOUND_ROWS to SQL
 965              $sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];
 966          }
 967  
 968          $sql_array['WHERE'] = implode(' AND ', $sql_where);
 969          $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : '';
 970          $sql_array['ORDER_BY'] = $sql_sort;
 971  
 972          unset($sql_where, $sql_sort, $group_by);
 973  
 974          $sql = $this->db->sql_build_query('SELECT', $sql_array);
 975          $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start);
 976  
 977          while ($row = $this->db->sql_fetchrow($result))
 978          {
 979              $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')];
 980          }
 981          $this->db->sql_freeresult($result);
 982  
 983          if (!$total_results && $is_mysql)
 984          {
 985              // Get the number of results as calculated by MySQL
 986              $sql_count = 'SELECT FOUND_ROWS() as total_results';
 987              $result = $this->db->sql_query($sql_count);
 988              $total_results = (int) $this->db->sql_fetchfield('total_results');
 989              $this->db->sql_freeresult($result);
 990  
 991              if (!$total_results)
 992              {
 993                  return false;
 994              }
 995          }
 996  
 997          if ($start >= $total_results)
 998          {
 999              $start = floor(($total_results - 1) / $per_page) * $per_page;
1000  
1001              $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start);
1002  
1003              while ($row = $this->db->sql_fetchrow($result))
1004              {
1005                  $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')];
1006              }
1007              $this->db->sql_freeresult($result);
1008  
1009          }
1010  
1011          // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page
1012          $this->save_ids($search_key, $this->search_query, $author_ary, $total_results, $id_ary, $start, $sort_dir);
1013          $id_ary = array_slice($id_ary, 0, (int) $per_page);
1014  
1015          return $total_results;
1016      }
1017  
1018      /**
1019      * Performs a search on an author's posts without caring about message contents. Depends on display specific params
1020      *
1021      * @param    string        $type                contains either posts or topics depending on what should be searched for
1022      * @param    boolean        $firstpost_only        if true, only topic starting posts will be considered
1023      * @param    array        $sort_by_sql        contains SQL code for the ORDER BY part of a query
1024      * @param    string        $sort_key            is the key of $sort_by_sql for the selected sorting
1025      * @param    string        $sort_dir            is either a or d representing ASC and DESC
1026      * @param    string        $sort_days            specifies the maximum amount of days a post may be old
1027      * @param    array        $ex_fid_ary            specifies an array of forum ids which should not be searched
1028      * @param    string        $post_visibility    specifies which types of posts the user can view in which forums
1029      * @param    int            $topic_id            is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
1030      * @param    array        $author_ary            an array of author ids
1031      * @param    string        $author_name        specifies the author match, when ANONYMOUS is also a search-match
1032      * @param    array        &$id_ary            passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
1033      * @param    int            $start                indicates the first index of the page
1034      * @param    int            $per_page            number of ids each page is supposed to contain
1035      * @return    boolean|int                        total number of results
1036      */
1037  	public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
1038      {
1039          // No author? No posts
1040          if (!count($author_ary))
1041          {
1042              return 0;
1043          }
1044  
1045          // generate a search_key from all the options to identify the results
1046          $search_key_array = array(
1047              '',
1048              $type,
1049              ($firstpost_only) ? 'firstpost' : '',
1050              '',
1051              '',
1052              $sort_days,
1053              $sort_key,
1054              $topic_id,
1055              implode(',', $ex_fid_ary),
1056              $post_visibility,
1057              implode(',', $author_ary),
1058              $author_name,
1059          );
1060  
1061          /**
1062          * Allow changing the search_key for cached results
1063          *
1064          * @event core.search_native_by_author_modify_search_key
1065          * @var    array    search_key_array    Array with search parameters to generate the search_key
1066          * @var    string    type                Searching type ('posts', 'topics')
1067          * @var    boolean    firstpost_only        Flag indicating if only topic starting posts are considered
1068          * @var    int        sort_days            Time, in days, of the oldest possible post to list
1069          * @var    string    sort_key            The sort type used from the possible sort types
1070          * @var    int        topic_id            Limit the search to this topic_id only
1071          * @var    array    ex_fid_ary            Which forums not to search on
1072          * @var    string    post_visibility        Post visibility data
1073          * @var    array    author_ary            Array of user_id containing the users to filter the results to
1074          * @var    string    author_name            The username to search on
1075          * @since 3.1.7-RC1
1076          */
1077          $vars = array(
1078              'search_key_array',
1079              'type',
1080              'firstpost_only',
1081              'sort_days',
1082              'sort_key',
1083              'topic_id',
1084              'ex_fid_ary',
1085              'post_visibility',
1086              'author_ary',
1087              'author_name',
1088          );
1089          extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_author_modify_search_key', compact($vars)));
1090  
1091          $search_key = md5(implode('#', $search_key_array));
1092  
1093          // try reading the results from cache
1094          $total_results = 0;
1095          if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == SEARCH_RESULT_IN_CACHE)
1096          {
1097              return $total_results;
1098          }
1099  
1100          $id_ary = array();
1101  
1102          // Create some display specific sql strings
1103          if ($author_name)
1104          {
1105              // first one matches post of registered users, second one guests and deleted users
1106              $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')';
1107          }
1108          else
1109          {
1110              $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary);
1111          }
1112          $sql_fora        = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : '';
1113          $sql_time        = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : '';
1114          $sql_topic_id    = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : '';
1115          $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : '';
1116          $post_visibility = ($post_visibility) ? ' AND ' . $post_visibility : '';
1117  
1118          // Build sql strings for sorting
1119          $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');
1120          $sql_sort_table = $sql_sort_join = '';
1121          switch ($sql_sort[0])
1122          {
1123              case 'u':
1124                  $sql_sort_table    = USERS_TABLE . ' u, ';
1125                  $sql_sort_join    = ' AND u.user_id = p.poster_id ';
1126              break;
1127  
1128              case 't':
1129                  $sql_sort_table    = ($type == 'posts' && !$firstpost_only) ? TOPICS_TABLE . ' t, ' : '';
1130                  $sql_sort_join    = ($type == 'posts' && !$firstpost_only) ? ' AND t.topic_id = p.topic_id ' : '';
1131              break;
1132  
1133              case 'f':
1134                  $sql_sort_table    = FORUMS_TABLE . ' f, ';
1135                  $sql_sort_join    = ' AND f.forum_id = p.forum_id ';
1136              break;
1137          }
1138  
1139          $select = ($type == 'posts') ? 'p.post_id' : 't.topic_id';
1140          $is_mysql = false;
1141  
1142          /**
1143          * Allow changing the query used to search for posts by author in fulltext_native
1144          *
1145          * @event core.search_native_author_count_query_before
1146          * @var    int        total_results        The previous result count for the format of the query.
1147          *                                    Set to 0 to force a re-count
1148          * @var    string    type                The type of search being made
1149          * @var    string    select                SQL SELECT clause for what to get
1150          * @var    string    sql_sort_table        CROSS JOIN'ed table to allow doing the sort chosen
1151          * @var    string    sql_sort_join        Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table
1152          * @var    array    sql_author            SQL WHERE condition for the post author ids
1153          * @var    int        topic_id            Limit the search to this topic_id only
1154          * @var    string    sort_by_sql            The possible predefined sort types
1155          * @var    string    sort_key            The sort type used from the possible sort types
1156          * @var    string    sort_dir            "a" for ASC or "d" dor DESC for the sort order used
1157          * @var    string    sql_sort            The result SQL when processing sort_by_sql + sort_key + sort_dir
1158          * @var    string    sort_days            Time, in days, that the oldest post showing can have
1159          * @var    string    sql_time            The SQL to search on the time specifyed by sort_days
1160          * @var    bool    firstpost_only        Wether or not to search only on the first post of the topics
1161          * @var    string    sql_firstpost        The SQL used in the WHERE claused to filter by firstpost.
1162          * @var    array    ex_fid_ary            Forum ids that must not be searched on
1163          * @var    array    sql_fora            SQL query for ex_fid_ary
1164          * @var    int        start                How many posts to skip in the search results (used for pagination)
1165          * @since 3.1.5-RC1
1166          */
1167          $vars = array(
1168              'total_results',
1169              'type',
1170              'select',
1171              'sql_sort_table',
1172              'sql_sort_join',
1173              'sql_author',
1174              'topic_id',
1175              'sort_by_sql',
1176              'sort_key',
1177              'sort_dir',
1178              'sql_sort',
1179              'sort_days',
1180              'sql_time',
1181              'firstpost_only',
1182              'sql_firstpost',
1183              'ex_fid_ary',
1184              'sql_fora',
1185              'start',
1186          );
1187          extract($this->phpbb_dispatcher->trigger_event('core.search_native_author_count_query_before', compact($vars)));
1188  
1189          // If the cache was completely empty count the results
1190          if (!$total_results)
1191          {
1192              switch ($this->db->get_sql_layer())
1193              {
1194                  case 'mysqli':
1195  //                    $select = 'SQL_CALC_FOUND_ROWS ' . $select;
1196                      $is_mysql = true;
1197                  break;
1198  
1199                  default:
1200                      if ($type == 'posts')
1201                      {
1202                          $sql = 'SELECT COUNT(p.post_id) as total_results
1203                              FROM ' . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t ' : ' ') . "
1204                              WHERE $sql_author
1205                                  $sql_topic_id
1206                                  $sql_firstpost
1207                                  $post_visibility
1208                                  $sql_fora
1209                                  $sql_time";
1210                      }
1211                      else
1212                      {
1213                          if ($this->db->get_sql_layer() == 'sqlite3')
1214                          {
1215                              $sql = 'SELECT COUNT(topic_id) as total_results
1216                                  FROM (SELECT DISTINCT t.topic_id';
1217                          }
1218                          else
1219                          {
1220                              $sql = 'SELECT COUNT(DISTINCT t.topic_id) as total_results';
1221                          }
1222  
1223                          $sql .= ' FROM ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p
1224                              WHERE $sql_author
1225                                  $sql_topic_id
1226                                  $sql_firstpost
1227                                  $post_visibility
1228                                  $sql_fora
1229                                  AND t.topic_id = p.topic_id
1230                                  $sql_time" . ($this->db->get_sql_layer() == 'sqlite3' ? ')' : '');
1231                      }
1232                      $result = $this->db->sql_query($sql);
1233  
1234                      $total_results = (int) $this->db->sql_fetchfield('total_results');
1235                      $this->db->sql_freeresult($result);
1236  
1237                      if (!$total_results)
1238                      {
1239                          return false;
1240                      }
1241                  break;
1242              }
1243          }
1244  
1245          // Build the query for really selecting the post_ids
1246          if ($type == 'posts')
1247          {
1248              $sql = "SELECT $select
1249                  FROM " . $sql_sort_table . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t' : '') . "
1250                  WHERE $sql_author
1251                      $sql_topic_id
1252                      $sql_firstpost
1253                      $post_visibility
1254                      $sql_fora
1255                      $sql_sort_join
1256                      $sql_time
1257                  ORDER BY $sql_sort";
1258              $field = 'post_id';
1259          }
1260          else
1261          {
1262              $sql = "SELECT $select
1263                  FROM " . $sql_sort_table . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p
1264                  WHERE $sql_author
1265                      $sql_topic_id
1266                      $sql_firstpost
1267                      $post_visibility
1268                      $sql_fora
1269                      AND t.topic_id = p.topic_id
1270                      $sql_sort_join
1271                      $sql_time
1272                  GROUP BY t.topic_id, " . $sort_by_sql[$sort_key] . '
1273                  ORDER BY ' . $sql_sort;
1274              $field = 'topic_id';
1275          }
1276  
1277          // Only read one block of posts from the db and then cache it
1278          $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start);
1279  
1280          while ($row = $this->db->sql_fetchrow($result))
1281          {
1282              $id_ary[] = (int) $row[$field];
1283          }
1284          $this->db->sql_freeresult($result);
1285  
1286          if (!$total_results && $is_mysql)
1287          {
1288              // Count rows for the executed queries. Replace $select within $sql with SQL_CALC_FOUND_ROWS, and run it.
1289              $sql_calc = str_replace('SELECT ' . $select, 'SELECT SQL_CALC_FOUND_ROWS ' . $select, $sql);
1290  
1291              $result = $this->db->sql_query($sql_calc);
1292              $this->db->sql_freeresult($result);
1293  
1294              $sql_count = 'SELECT FOUND_ROWS() as total_results';
1295              $result = $this->db->sql_query($sql_count);
1296              $total_results = (int) $this->db->sql_fetchfield('total_results');
1297              $this->db->sql_freeresult($result);
1298  
1299              if (!$total_results)
1300              {
1301                  return false;
1302              }
1303          }
1304  
1305          if ($start >= $total_results)
1306          {
1307              $start = floor(($total_results - 1) / $per_page) * $per_page;
1308  
1309              $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start);
1310  
1311              while ($row = $this->db->sql_fetchrow($result))
1312              {
1313                  $id_ary[] = (int) $row[$field];
1314              }
1315              $this->db->sql_freeresult($result);
1316          }
1317  
1318          if (count($id_ary))
1319          {
1320              $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir);
1321              $id_ary = array_slice($id_ary, 0, $per_page);
1322  
1323              return $total_results;
1324          }
1325          return false;
1326      }
1327  
1328      /**
1329      * Split a text into words of a given length
1330      *
1331      * The text is converted to UTF-8, cleaned up, and split. Then, words that
1332      * conform to the defined length range are returned in an array.
1333      *
1334      * NOTE: duplicates are NOT removed from the return array
1335      *
1336      * @param    string    $text    Text to split, encoded in UTF-8
1337      * @return    array            Array of UTF-8 words
1338      */
1339  	public function split_message($text)
1340      {
1341          $match = $words = array();
1342  
1343          /**
1344          * Taken from the original code
1345          */
1346          // Do not index code
1347          $match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is';
1348          // BBcode
1349          $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#';
1350  
1351          $min = $this->word_length['min'];
1352  
1353          $isset_min = $min - 1;
1354  
1355          /**
1356          * Clean up the string, remove HTML tags, remove BBCodes
1357          */
1358          $word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' ');
1359  
1360          while (strlen($word))
1361          {
1362              if (strlen($word) > 255 || strlen($word) <= $isset_min)
1363              {
1364                  /**
1365                  * Words longer than 255 bytes are ignored. This will have to be
1366                  * changed whenever we change the length of search_wordlist.word_text
1367                  *
1368                  * Words shorter than $isset_min bytes are ignored, too
1369                  */
1370                  $word = strtok(' ');
1371                  continue;
1372              }
1373  
1374              $len = utf8_strlen($word);
1375  
1376              /**
1377              * Test whether the word is too short to be indexed.
1378              *
1379              * Note that this limit does NOT apply to CJK and Hangul
1380              */
1381              if ($len < $min)
1382              {
1383                  /**
1384                  * Note: this could be optimized. If the codepoint is lower than Hangul's range
1385                  * we know that it will also be lower than CJK ranges
1386                  */
1387                  if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0)
1388                      && (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0)
1389                      && (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0))
1390                  {
1391                      $word = strtok(' ');
1392                      continue;
1393                  }
1394              }
1395  
1396              $words[] = $word;
1397              $word = strtok(' ');
1398          }
1399  
1400          return $words;
1401      }
1402  
1403      /**
1404      * Updates wordlist and wordmatch tables when a message is posted or changed
1405      *
1406      * @param    string    $mode        Contains the post mode: edit, post, reply, quote
1407      * @param    int        $post_id    The id of the post which is modified/created
1408      * @param    string    &$message    New or updated post content
1409      * @param    string    &$subject    New or updated post subject
1410      * @param    int        $poster_id    Post author's user id
1411      * @param    int        $forum_id    The id of the forum in which the post is located
1412      */
1413  	public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
1414      {
1415          if (!$this->config['fulltext_native_load_upd'])
1416          {
1417              /**
1418              * The search indexer is disabled, return
1419              */
1420              return;
1421          }
1422  
1423          // Split old and new post/subject to obtain array of 'words'
1424          $split_text = $this->split_message($message);
1425          $split_title = $this->split_message($subject);
1426  
1427          $cur_words = array('post' => array(), 'title' => array());
1428  
1429          $words = array();
1430          if ($mode == 'edit')
1431          {
1432              $words['add']['post'] = array();
1433              $words['add']['title'] = array();
1434              $words['del']['post'] = array();
1435              $words['del']['title'] = array();
1436  
1437              $sql = 'SELECT w.word_id, w.word_text, m.title_match
1438                  FROM ' . SEARCH_WORDLIST_TABLE . ' w, ' . SEARCH_WORDMATCH_TABLE . " m
1439                  WHERE m.post_id = $post_id
1440                      AND w.word_id = m.word_id";
1441              $result = $this->db->sql_query($sql);
1442  
1443              while ($row = $this->db->sql_fetchrow($result))
1444              {
1445                  $which = ($row['title_match']) ? 'title' : 'post';
1446                  $cur_words[$which][$row['word_text']] = $row['word_id'];
1447              }
1448              $this->db->sql_freeresult($result);
1449  
1450              $words['add']['post'] = array_diff($split_text, array_keys($cur_words['post']));
1451              $words['add']['title'] = array_diff($split_title, array_keys($cur_words['title']));
1452              $words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text);
1453              $words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title);
1454          }
1455          else
1456          {
1457              $words['add']['post'] = $split_text;
1458              $words['add']['title'] = $split_title;
1459              $words['del']['post'] = array();
1460              $words['del']['title'] = array();
1461          }
1462  
1463          /**
1464          * Event to modify method arguments and words before the native search index is updated
1465          *
1466          * @event core.search_native_index_before
1467          * @var string    mode                Contains the post mode: edit, post, reply, quote
1468          * @var int        post_id                The id of the post which is modified/created
1469          * @var string    message                New or updated post content
1470          * @var string    subject                New or updated post subject
1471          * @var int        poster_id            Post author's user id
1472          * @var int        forum_id            The id of the forum in which the post is located
1473          * @var array    words                Grouped lists of words added to or remove from the index
1474          * @var array    split_text            Array of words from the message
1475          * @var array    split_title            Array of words from the title
1476          * @var array    cur_words            Array of words currently in the index for comparing to new words
1477          *                                     when mode is edit. Empty for other modes.
1478          * @since 3.2.3-RC1
1479          */
1480          $vars = array(
1481              'mode',
1482              'post_id',
1483              'message',
1484              'subject',
1485              'poster_id',
1486              'forum_id',
1487              'words',
1488              'split_text',
1489              'split_title',
1490              'cur_words',
1491          );
1492          extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars)));
1493  
1494          unset($split_text);
1495          unset($split_title);
1496  
1497          // Get unique words from the above arrays
1498          $unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title']));
1499  
1500          // We now have unique arrays of all words to be added and removed and
1501          // individual arrays of added and removed words for text and title. What
1502          // we need to do now is add the new words (if they don't already exist)
1503          // and then add (or remove) matches between the words and this post
1504          if (count($unique_add_words))
1505          {
1506              $sql = 'SELECT word_id, word_text
1507                  FROM ' . SEARCH_WORDLIST_TABLE . '
1508                  WHERE ' . $this->db->sql_in_set('word_text', $unique_add_words);
1509              $result = $this->db->sql_query($sql);
1510  
1511              $word_ids = array();
1512              while ($row = $this->db->sql_fetchrow($result))
1513              {
1514                  $word_ids[$row['word_text']] = $row['word_id'];
1515              }
1516              $this->db->sql_freeresult($result);
1517              $new_words = array_diff($unique_add_words, array_keys($word_ids));
1518  
1519              $this->db->sql_transaction('begin');
1520              if (count($new_words))
1521              {
1522                  $sql_ary = array();
1523  
1524                  foreach ($new_words as $word)
1525                  {
1526                      $sql_ary[] = array('word_text' => (string) $word, 'word_count' => 0);
1527                  }
1528                  $this->db->sql_return_on_error(true);
1529                  $this->db->sql_multi_insert(SEARCH_WORDLIST_TABLE, $sql_ary);
1530                  $this->db->sql_return_on_error(false);
1531              }
1532              unset($new_words, $sql_ary);
1533          }
1534          else
1535          {
1536              $this->db->sql_transaction('begin');
1537          }
1538  
1539          // now update the search match table, remove links to removed words and add links to new words
1540          foreach ($words['del'] as $word_in => $word_ary)
1541          {
1542              $title_match = ($word_in == 'title') ? 1 : 0;
1543  
1544              if (count($word_ary))
1545              {
1546                  $sql_in = array();
1547                  foreach ($word_ary as $word)
1548                  {
1549                      $sql_in[] = $cur_words[$word_in][$word];
1550                  }
1551  
1552                  $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1553                      WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . '
1554                          AND post_id = ' . intval($post_id) . "
1555                          AND title_match = $title_match";
1556                  $this->db->sql_query($sql);
1557  
1558                  $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1559                      SET word_count = word_count - 1
1560                      WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . '
1561                          AND word_count > 0';
1562                  $this->db->sql_query($sql);
1563  
1564                  unset($sql_in);
1565              }
1566          }
1567  
1568          $this->db->sql_return_on_error(true);
1569          foreach ($words['add'] as $word_in => $word_ary)
1570          {
1571              $title_match = ($word_in == 'title') ? 1 : 0;
1572  
1573              if (count($word_ary))
1574              {
1575                  $sql = 'INSERT INTO ' . SEARCH_WORDMATCH_TABLE . ' (post_id, word_id, title_match)
1576                      SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . '
1577                      FROM ' . SEARCH_WORDLIST_TABLE . '
1578                      WHERE ' . $this->db->sql_in_set('word_text', $word_ary);
1579                  $this->db->sql_query($sql);
1580  
1581                  $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1582                      SET word_count = word_count + 1
1583                      WHERE ' . $this->db->sql_in_set('word_text', $word_ary);
1584                  $this->db->sql_query($sql);
1585              }
1586          }
1587          $this->db->sql_return_on_error(false);
1588  
1589          $this->db->sql_transaction('commit');
1590  
1591          // destroy cached search results containing any of the words removed or added
1592          $this->destroy_cache(array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['del']['post'], $words['del']['title'])), array($poster_id));
1593  
1594          unset($unique_add_words);
1595          unset($words);
1596          unset($cur_words);
1597      }
1598  
1599      /**
1600      * Removes entries from the wordmatch table for the specified post_ids
1601      */
1602  	public function index_remove($post_ids, $author_ids, $forum_ids)
1603      {
1604          if (count($post_ids))
1605          {
1606              $sql = 'SELECT w.word_id, w.word_text, m.title_match
1607                  FROM ' . SEARCH_WORDMATCH_TABLE . ' m, ' . SEARCH_WORDLIST_TABLE . ' w
1608                  WHERE ' . $this->db->sql_in_set('m.post_id', $post_ids) . '
1609                      AND w.word_id = m.word_id';
1610              $result = $this->db->sql_query($sql);
1611  
1612              $message_word_ids = $title_word_ids = $word_texts = array();
1613              while ($row = $this->db->sql_fetchrow($result))
1614              {
1615                  if ($row['title_match'])
1616                  {
1617                      $title_word_ids[] = $row['word_id'];
1618                  }
1619                  else
1620                  {
1621                      $message_word_ids[] = $row['word_id'];
1622                  }
1623                  $word_texts[] = $row['word_text'];
1624              }
1625              $this->db->sql_freeresult($result);
1626  
1627              if (count($title_word_ids))
1628              {
1629                  $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1630                      SET word_count = word_count - 1
1631                      WHERE ' . $this->db->sql_in_set('word_id', $title_word_ids) . '
1632                          AND word_count > 0';
1633                  $this->db->sql_query($sql);
1634              }
1635  
1636              if (count($message_word_ids))
1637              {
1638                  $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1639                      SET word_count = word_count - 1
1640                      WHERE ' . $this->db->sql_in_set('word_id', $message_word_ids) . '
1641                          AND word_count > 0';
1642                  $this->db->sql_query($sql);
1643              }
1644  
1645              unset($title_word_ids);
1646              unset($message_word_ids);
1647  
1648              $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1649                  WHERE ' . $this->db->sql_in_set('post_id', $post_ids);
1650              $this->db->sql_query($sql);
1651          }
1652  
1653          $this->destroy_cache(array_unique($word_texts), array_unique($author_ids));
1654      }
1655  
1656      /**
1657      * Tidy up indexes: Tag 'common words' and remove
1658      * words no longer referenced in the match table
1659      */
1660  	public function tidy()
1661      {
1662          // Is the fulltext indexer disabled? If yes then we need not
1663          // carry on ... it's okay ... I know when I'm not wanted boo hoo
1664          if (!$this->config['fulltext_native_load_upd'])
1665          {
1666              $this->config->set('search_last_gc', time(), false);
1667              return;
1668          }
1669  
1670          $destroy_cache_words = array();
1671  
1672          // Remove common words
1673          if ($this->config['num_posts'] >= 100 && $this->config['fulltext_native_common_thres'])
1674          {
1675              $common_threshold = ((double) $this->config['fulltext_native_common_thres']) / 100.0;
1676              // First, get the IDs of common words
1677              $sql = 'SELECT word_id, word_text
1678                  FROM ' . SEARCH_WORDLIST_TABLE . '
1679                  WHERE word_count > ' . floor($this->config['num_posts'] * $common_threshold) . '
1680                      OR word_common = 1';
1681              $result = $this->db->sql_query($sql);
1682  
1683              $sql_in = array();
1684              while ($row = $this->db->sql_fetchrow($result))
1685              {
1686                  $sql_in[] = $row['word_id'];
1687                  $destroy_cache_words[] = $row['word_text'];
1688              }
1689              $this->db->sql_freeresult($result);
1690  
1691              if (count($sql_in))
1692              {
1693                  // Flag the words
1694                  $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1695                      SET word_common = 1
1696                      WHERE ' . $this->db->sql_in_set('word_id', $sql_in);
1697                  $this->db->sql_query($sql);
1698  
1699                  // by setting search_last_gc to the new time here we make sure that if a user reloads because the
1700                  // following query takes too long, he won't run into it again
1701                  $this->config->set('search_last_gc', time(), false);
1702  
1703                  // Delete the matches
1704                  $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1705                      WHERE ' . $this->db->sql_in_set('word_id', $sql_in);
1706                  $this->db->sql_query($sql);
1707              }
1708              unset($sql_in);
1709          }
1710  
1711          if (count($destroy_cache_words))
1712          {
1713              // destroy cached search results containing any of the words that are now common or were removed
1714              $this->destroy_cache(array_unique($destroy_cache_words));
1715          }
1716  
1717          $this->config->set('search_last_gc', time(), false);
1718      }
1719  
1720      /**
1721      * Deletes all words from the index
1722      */
1723  	public function delete_index($acp_module, $u_action)
1724      {
1725          $sql_queries = [];
1726  
1727          switch ($this->db->get_sql_layer())
1728          {
1729              case 'sqlite3':
1730                  $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDLIST_TABLE;
1731                  $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE;
1732                  $sql_queries[] = 'DELETE FROM ' . SEARCH_RESULTS_TABLE;
1733              break;
1734  
1735              default:
1736                  $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE;
1737                  $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE;
1738                  $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE;
1739              break;
1740          }
1741  
1742          $stats = $this->stats;
1743  
1744          /**
1745          * Event to modify SQL queries before the native search index is deleted
1746          *
1747          * @event core.search_native_delete_index_before
1748          * @var array    sql_queries            Array with queries for deleting the search index
1749          * @var array    stats                Array with statistics of the current index (read only)
1750          * @since 3.2.3-RC1
1751          */
1752          $vars = array(
1753              'sql_queries',
1754              'stats',
1755          );
1756          extract($this->phpbb_dispatcher->trigger_event('core.search_native_delete_index_before', compact($vars)));
1757  
1758          foreach ($sql_queries as $sql_query)
1759          {
1760              $this->db->sql_query($sql_query);
1761          }
1762      }
1763  
1764      /**
1765      * Returns true if both FULLTEXT indexes exist
1766      */
1767  	public function index_created()
1768      {
1769          if (!count($this->stats))
1770          {
1771              $this->get_stats();
1772          }
1773  
1774          return ($this->stats['total_words'] && $this->stats['total_matches']) ? true : false;
1775      }
1776  
1777      /**
1778      * Returns an associative array containing information about the indexes
1779      */
1780  	public function index_stats()
1781      {
1782          if (!count($this->stats))
1783          {
1784              $this->get_stats();
1785          }
1786  
1787          return array(
1788              $this->user->lang['TOTAL_WORDS']        => $this->stats['total_words'],
1789              $this->user->lang['TOTAL_MATCHES']    => $this->stats['total_matches']);
1790      }
1791  
1792  	protected function get_stats()
1793      {
1794          $this->stats['total_words']        = $this->db->get_estimated_row_count(SEARCH_WORDLIST_TABLE);
1795          $this->stats['total_matches']    = $this->db->get_estimated_row_count(SEARCH_WORDMATCH_TABLE);
1796      }
1797  
1798      /**
1799      * Clean up a text to remove non-alphanumeric characters
1800      *
1801      * This method receives a UTF-8 string, normalizes and validates it, replaces all
1802      * non-alphanumeric characters with strings then returns the result.
1803      *
1804      * Any number of "allowed chars" can be passed as a UTF-8 string in NFC.
1805      *
1806      * @param    string    $text            Text to split, in UTF-8 (not normalized or sanitized)
1807      * @param    string    $allowed_chars    String of special chars to allow
1808      * @param    string    $encoding        Text encoding
1809      * @return    string                    Cleaned up text, only alphanumeric chars are left
1810      */
1811  	protected function cleanup($text, $allowed_chars = null, $encoding = 'utf-8')
1812      {
1813          static $conv = array(), $conv_loaded = array();
1814          $allow = array();
1815  
1816          // Convert the text to UTF-8
1817          $encoding = strtolower($encoding);
1818          if ($encoding != 'utf-8')
1819          {
1820              $text = utf8_recode($text, $encoding);
1821          }
1822  
1823          $utf_len_mask = array(
1824              "\xC0"    =>    2,
1825              "\xD0"    =>    2,
1826              "\xE0"    =>    3,
1827              "\xF0"    =>    4
1828          );
1829  
1830          /**
1831          * Replace HTML entities and NCRs
1832          */
1833          $text = htmlspecialchars_decode(utf8_decode_ncr($text), ENT_QUOTES);
1834  
1835          /**
1836          * Normalize to NFC
1837          */
1838          $text = \Normalizer::normalize($text);
1839  
1840          /**
1841          * The first thing we do is:
1842          *
1843          * - convert ASCII-7 letters to lowercase
1844          * - remove the ASCII-7 non-alpha characters
1845          * - remove the bytes that should not appear in a valid UTF-8 string: 0xC0,
1846          *   0xC1 and 0xF5-0xFF
1847          *
1848          * @todo in theory, the third one is already taken care of during normalization and those chars should have been replaced by Unicode replacement chars
1849          */
1850          $sb_match    = "ISTCPAMELRDOJBNHFGVWUQKYXZ\r\n\t!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\xC0\xC1\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF";
1851          $sb_replace    = 'istcpamelrdojbnhfgvwuqkyxz                                                                              ';
1852  
1853          /**
1854          * This is the list of legal ASCII chars, it is automatically extended
1855          * with ASCII chars from $allowed_chars
1856          */
1857          $legal_ascii = ' eaisntroludcpmghbfvq10xy2j9kw354867z';
1858  
1859          /**
1860          * Prepare an array containing the extra chars to allow
1861          */
1862          if (isset($allowed_chars[0]))
1863          {
1864              $pos = 0;
1865              $len = strlen($allowed_chars);
1866              do
1867              {
1868                  $c = $allowed_chars[$pos];
1869  
1870                  if ($c < "\x80")
1871                  {
1872                      /**
1873                      * ASCII char
1874                      */
1875                      $sb_pos = strpos($sb_match, $c);
1876                      if (is_int($sb_pos))
1877                      {
1878                          /**
1879                          * Remove the char from $sb_match and its corresponding
1880                          * replacement in $sb_replace
1881                          */
1882                          $sb_match = substr($sb_match, 0, $sb_pos) . substr($sb_match, $sb_pos + 1);
1883                          $sb_replace = substr($sb_replace, 0, $sb_pos) . substr($sb_replace, $sb_pos + 1);
1884                          $legal_ascii .= $c;
1885                      }
1886  
1887                      ++$pos;
1888                  }
1889                  else
1890                  {
1891                      /**
1892                      * UTF-8 char
1893                      */
1894                      $utf_len = $utf_len_mask[$c & "\xF0"];
1895                      $allow[substr($allowed_chars, $pos, $utf_len)] = 1;
1896                      $pos += $utf_len;
1897                  }
1898              }
1899              while ($pos < $len);
1900          }
1901  
1902          $text = strtr($text, $sb_match, $sb_replace);
1903          $ret = '';
1904  
1905          $pos = 0;
1906          $len = strlen($text);
1907  
1908          do
1909          {
1910              /**
1911              * Do all consecutive ASCII chars at once
1912              */
1913              if ($spn = strspn($text, $legal_ascii, $pos))
1914              {
1915                  $ret .= substr($text, $pos, $spn);
1916                  $pos += $spn;
1917              }
1918  
1919              if ($pos >= $len)
1920              {
1921                  return $ret;
1922              }
1923  
1924              /**
1925              * Capture the UTF char
1926              */
1927              $utf_len = $utf_len_mask[$text[$pos] & "\xF0"];
1928              $utf_char = substr($text, $pos, $utf_len);
1929              $pos += $utf_len;
1930  
1931              if (($utf_char >= self::UTF8_HANGUL_FIRST && $utf_char <= self::UTF8_HANGUL_LAST)
1932                  || ($utf_char >= self::UTF8_CJK_FIRST && $utf_char <= self::UTF8_CJK_LAST)
1933                  || ($utf_char >= self::UTF8_CJK_B_FIRST && $utf_char <= self::UTF8_CJK_B_LAST))
1934              {
1935                  /**
1936                  * All characters within these ranges are valid
1937                  *
1938                  * We separate them with a space in order to index each character
1939                  * individually
1940                  */
1941                  $ret .= ' ' . $utf_char . ' ';
1942                  continue;
1943              }
1944  
1945              if (isset($allow[$utf_char]))
1946              {
1947                  /**
1948                  * The char is explicitly allowed
1949                  */
1950                  $ret .= $utf_char;
1951                  continue;
1952              }
1953  
1954              if (isset($conv[$utf_char]))
1955              {
1956                  /**
1957                  * The char is mapped to something, maybe to itself actually
1958                  */
1959                  $ret .= $conv[$utf_char];
1960                  continue;
1961              }
1962  
1963              /**
1964              * The char isn't mapped, but did we load its conversion table?
1965              *
1966              * The search indexer table is split into blocks. The block number of
1967              * each char is equal to its codepoint right-shifted for 11 bits. It
1968              * means that out of the 11, 16 or 21 meaningful bits of a 2-, 3- or
1969              * 4- byte sequence we only keep the leftmost 0, 5 or 10 bits. Thus,
1970              * all UTF chars encoded in 2 bytes are in the same first block.
1971              */
1972              if (isset($utf_char[2]))
1973              {
1974                  if (isset($utf_char[3]))
1975                  {
1976                      /**
1977                      * 1111 0nnn 10nn nnnn 10nx xxxx 10xx xxxx
1978                      * 0000 0111 0011 1111 0010 0000
1979                      */
1980                      $idx = ((ord($utf_char[0]) & 0x07) << 7) | ((ord($utf_char[1]) & 0x3F) << 1) | ((ord($utf_char[2]) & 0x20) >> 5);
1981                  }
1982                  else
1983                  {
1984                      /**
1985                      * 1110 nnnn 10nx xxxx 10xx xxxx
1986                      * 0000 0111 0010 0000
1987                      */
1988                      $idx = ((ord($utf_char[0]) & 0x07) << 1) | ((ord($utf_char[1]) & 0x20) >> 5);
1989                  }
1990              }
1991              else
1992              {
1993                  /**
1994                  * 110x xxxx 10xx xxxx
1995                  * 0000 0000 0000 0000
1996                  */
1997                  $idx = 0;
1998              }
1999  
2000              /**
2001              * Check if the required conv table has been loaded already
2002              */
2003              if (!isset($conv_loaded[$idx]))
2004              {
2005                  $conv_loaded[$idx] = 1;
2006                  $file = $this->phpbb_root_path . 'includes/utf/data/search_indexer_' . $idx . '.' . $this->php_ext;
2007  
2008                  if (file_exists($file))
2009                  {
2010                      $conv += include($file);
2011                  }
2012              }
2013  
2014              if (isset($conv[$utf_char]))
2015              {
2016                  $ret .= $conv[$utf_char];
2017              }
2018              else
2019              {
2020                  /**
2021                  * We add an entry to the conversion table so that we
2022                  * don't have to convert to codepoint and perform the checks
2023                  * that are above this block
2024                  */
2025                  $conv[$utf_char] = ' ';
2026                  $ret .= ' ';
2027              }
2028          }
2029          while (1);
2030  
2031          return $ret;
2032      }
2033  
2034      /**
2035      * Returns a list of options for the ACP to display
2036      */
2037  	public function acp()
2038      {
2039          /**
2040          * if we need any options, copied from fulltext_native for now, will have to be adjusted or removed
2041          */
2042  
2043          $tpl = '
2044          <dl>
2045              <dt><label for="fulltext_native_load_upd">' . $this->user->lang['YES_SEARCH_UPDATE'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['YES_SEARCH_UPDATE_EXPLAIN'] . '</span></dt>
2046              <dd><label><input type="radio" id="fulltext_native_load_upd" name="config[fulltext_native_load_upd]" value="1"' . (($this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->user->lang['YES'] . '</label><label><input type="radio" name="config[fulltext_native_load_upd]" value="0"' . ((!$this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->user->lang['NO'] . '</label></dd>
2047          </dl>
2048          <dl>
2049              <dt><label for="fulltext_native_min_chars">' . $this->user->lang['MIN_SEARCH_CHARS'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['MIN_SEARCH_CHARS_EXPLAIN'] . '</span></dt>
2050              <dd><input id="fulltext_native_min_chars" type="number" min="0" max="255" name="config[fulltext_native_min_chars]" value="' . (int) $this->config['fulltext_native_min_chars'] . '" /></dd>
2051          </dl>
2052          <dl>
2053              <dt><label for="fulltext_native_max_chars">' . $this->user->lang['MAX_SEARCH_CHARS'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['MAX_SEARCH_CHARS_EXPLAIN'] . '</span></dt>
2054              <dd><input id="fulltext_native_max_chars" type="number" min="0" max="255" name="config[fulltext_native_max_chars]" value="' . (int) $this->config['fulltext_native_max_chars'] . '" /></dd>
2055          </dl>
2056          <dl>
2057              <dt><label for="fulltext_native_common_thres">' . $this->user->lang['COMMON_WORD_THRESHOLD'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['COMMON_WORD_THRESHOLD_EXPLAIN'] . '</span></dt>
2058              <dd><input id="fulltext_native_common_thres" type="text" name="config[fulltext_native_common_thres]" value="' . (double) $this->config['fulltext_native_common_thres'] . '" /> %</dd>
2059          </dl>
2060          ';
2061  
2062          // These are fields required in the config table
2063          return array(
2064              'tpl'        => $tpl,
2065              'config'    => array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:255', 'fulltext_native_max_chars' => 'integer:0:255', 'fulltext_native_common_thres' => 'double:0:100')
2066          );
2067      }
2068  }


Generated: Tue Apr 7 19:44:41 2020 Cross-referenced by PHPXref 0.7.1