[ Index ]

PHP Cross Reference of phpBB-3.3.7-deutsch

title

Body

[close]

/phpbb/search/ -> fulltext_sphinx.php (source)

   1  <?php
   2  /**
   3  *
   4  * This file is part of the phpBB Forum Software package.
   5  *
   6  * @copyright (c) phpBB Limited <https://www.phpbb.com>
   7  * @license GNU General Public License, version 2 (GPL-2.0)
   8  *
   9  * For full copyright and license information, please see
  10  * the docs/CREDITS.txt file.
  11  *
  12  */
  13  
  14  namespace phpbb\search;
  15  
  16  define('SPHINX_MAX_MATCHES', 20000);
  17  define('SPHINX_CONNECT_RETRIES', 3);
  18  define('SPHINX_CONNECT_WAIT_TIME', 300);
  19  
  20  /**
  21  * Fulltext search based on the sphinx search daemon
  22  */
  23  class fulltext_sphinx
  24  {
  25      /**
  26       * Associative array holding index stats
  27       * @var array
  28       */
  29      protected $stats = array();
  30  
  31      /**
  32       * Holds the words entered by user, obtained by splitting the entered query on whitespace
  33       * @var array
  34       */
  35      protected $split_words = array();
  36  
  37      /**
  38       * Holds unique sphinx id
  39       * @var string
  40       */
  41      protected $id;
  42  
  43      /**
  44       * Stores the names of both main and delta sphinx indexes
  45       * separated by a semicolon
  46       * @var string
  47       */
  48      protected $indexes;
  49  
  50      /**
  51       * Sphinx searchd client object
  52       * @var SphinxClient
  53       */
  54      protected $sphinx;
  55  
  56      /**
  57       * Relative path to board root
  58       * @var string
  59       */
  60      protected $phpbb_root_path;
  61  
  62      /**
  63       * PHP Extension
  64       * @var string
  65       */
  66      protected $php_ext;
  67  
  68      /**
  69       * Auth object
  70       * @var \phpbb\auth\auth
  71       */
  72      protected $auth;
  73  
  74      /**
  75       * Config object
  76       * @var \phpbb\config\config
  77       */
  78      protected $config;
  79  
  80      /**
  81       * Database connection
  82       * @var \phpbb\db\driver\driver_interface
  83       */
  84      protected $db;
  85  
  86      /**
  87       * Database Tools object
  88       * @var \phpbb\db\tools\tools_interface
  89       */
  90      protected $db_tools;
  91  
  92      /**
  93       * Stores the database type if supported by sphinx
  94       * @var string
  95       */
  96      protected $dbtype;
  97  
  98      /**
  99       * phpBB event dispatcher object
 100       * @var \phpbb\event\dispatcher_interface
 101       */
 102      protected $phpbb_dispatcher;
 103  
 104      /**
 105       * User object
 106       * @var \phpbb\user
 107       */
 108      protected $user;
 109  
 110      /**
 111       * Stores the generated content of the sphinx config file
 112       * @var string
 113       */
 114      protected $config_file_data = '';
 115  
 116      /**
 117       * Contains tidied search query.
 118       * Operators are prefixed in search query and common words excluded
 119       * @var string
 120       */
 121      protected $search_query;
 122  
 123      /**
 124       * Constructor
 125       * Creates a new \phpbb\search\fulltext_postgres, which is used as a search backend
 126       *
 127       * @param string|bool $error Any error that occurs is passed on through this reference variable otherwise false
 128       * @param string $phpbb_root_path Relative path to phpBB root
 129       * @param string $phpEx PHP file extension
 130       * @param \phpbb\auth\auth $auth Auth object
 131       * @param \phpbb\config\config $config Config object
 132       * @param \phpbb\db\driver\driver_interface $db Database object
 133       * @param \phpbb\user $user User object
 134       * @param \phpbb\event\dispatcher_interface    $phpbb_dispatcher    Event dispatcher object
 135       */
 136  	public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher)
 137      {
 138          $this->phpbb_root_path = $phpbb_root_path;
 139          $this->php_ext = $phpEx;
 140          $this->config = $config;
 141          $this->phpbb_dispatcher = $phpbb_dispatcher;
 142          $this->user = $user;
 143          $this->db = $db;
 144          $this->auth = $auth;
 145  
 146          // Initialize \phpbb\db\tools\tools object
 147          global $phpbb_container; // TODO inject into object
 148          $this->db_tools = $phpbb_container->get('dbal.tools');
 149  
 150          if (!$this->config['fulltext_sphinx_id'])
 151          {
 152              $this->config->set('fulltext_sphinx_id', unique_id());
 153          }
 154          $this->id = $this->config['fulltext_sphinx_id'];
 155          $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main';
 156  
 157          if (!class_exists('SphinxClient'))
 158          {
 159              require($this->phpbb_root_path . 'includes/sphinxapi.' . $this->php_ext);
 160          }
 161  
 162          // Initialize sphinx client
 163          $this->sphinx = new \SphinxClient();
 164  
 165          $this->sphinx->SetServer(($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost'), ($this->config['fulltext_sphinx_port'] ? (int) $this->config['fulltext_sphinx_port'] : 9312));
 166  
 167          $error = false;
 168      }
 169  
 170      /**
 171      * Returns the name of this search backend to be displayed to administrators
 172      *
 173      * @return string Name
 174      */
 175  	public function get_name()
 176      {
 177          return 'Sphinx Fulltext';
 178      }
 179  
 180      /**
 181       * Returns the search_query
 182       *
 183       * @return string search query
 184       */
 185  	public function get_search_query()
 186      {
 187          return $this->search_query;
 188      }
 189  
 190      /**
 191       * Returns false as there is no word_len array
 192       *
 193       * @return false
 194       */
 195  	public function get_word_length()
 196      {
 197          return false;
 198      }
 199  
 200      /**
 201       * Returns an empty array as there are no common_words
 202       *
 203       * @return array common words that are ignored by search backend
 204       */
 205  	public function get_common_words()
 206      {
 207          return array();
 208      }
 209  
 210      /**
 211      * Checks permissions and paths, if everything is correct it generates the config file
 212      *
 213      * @return string|bool Language key of the error/incompatibility encountered, or false if successful
 214      */
 215  	public function init()
 216      {
 217          if ($this->db->get_sql_layer() != 'mysqli' && $this->db->get_sql_layer() != 'postgres')
 218          {
 219              return $this->user->lang['FULLTEXT_SPHINX_WRONG_DATABASE'];
 220          }
 221  
 222          // Move delta to main index each hour
 223          $this->config->set('search_gc', 3600);
 224  
 225          return false;
 226      }
 227  
 228      /**
 229       * Generates content of sphinx.conf
 230       *
 231       * @return bool True if sphinx.conf content is correctly generated, false otherwise
 232       */
 233  	protected function config_generate()
 234      {
 235          // Check if Database is supported by Sphinx
 236          if ($this->db->get_sql_layer() == 'mysqli')
 237          {
 238              $this->dbtype = 'mysql';
 239          }
 240          else if ($this->db->get_sql_layer() == 'postgres')
 241          {
 242              $this->dbtype = 'pgsql';
 243          }
 244          else
 245          {
 246              $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_WRONG_DATABASE');
 247              return false;
 248          }
 249  
 250          // Check if directory paths have been filled
 251          if (!$this->config['fulltext_sphinx_data_path'])
 252          {
 253              $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA');
 254              return false;
 255          }
 256  
 257          include($this->phpbb_root_path . 'config.' . $this->php_ext);
 258  
 259          /* Now that we're sure everything was entered correctly,
 260          generate a config for the index. We use a config value
 261          fulltext_sphinx_id for this, as it should be unique. */
 262          $config_object = new \phpbb\search\sphinx\config($this->config_file_data);
 263          $config_data = array(
 264              'source source_phpbb_' . $this->id . '_main' => array(
 265                  array('type',                        $this->dbtype . ' # mysql or pgsql'),
 266                  // This config value sql_host needs to be changed incase sphinx and sql are on different servers
 267                  array('sql_host',                    $dbhost . ' # SQL server host sphinx connects to'),
 268                  array('sql_user',                    '[dbuser]'),
 269                  array('sql_pass',                    '[dbpassword]'),
 270                  array('sql_db',                        $dbname),
 271                  array('sql_port',                    $dbport . ' # optional, default is 3306 for mysql and 5432 for pgsql'),
 272                  array('sql_query_pre',                'SET NAMES \'utf8\''),
 273                  array('sql_query_pre',                'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'),
 274                  array('sql_query_range',            'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
 275                  array('sql_range_step',                '5000'),
 276                  array('sql_query',                    'SELECT
 277                          p.post_id AS id,
 278                          p.forum_id,
 279                          p.topic_id,
 280                          p.poster_id,
 281                          p.post_visibility,
 282                          CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
 283                          p.post_time,
 284                          p.post_subject,
 285                          p.post_subject as title,
 286                          p.post_text as data,
 287                          t.topic_last_post_time,
 288                          0 as deleted
 289                      FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
 290                      WHERE
 291                          p.topic_id = t.topic_id
 292                          AND p.post_id >= $start AND p.post_id <= $end'),
 293                  array('sql_query_post',                ''),
 294                  array('sql_query_post_index',        'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'),
 295                  array('sql_attr_uint',                'forum_id'),
 296                  array('sql_attr_uint',                'topic_id'),
 297                  array('sql_attr_uint',                'poster_id'),
 298                  array('sql_attr_uint',                'post_visibility'),
 299                  array('sql_attr_bool',                'topic_first_post'),
 300                  array('sql_attr_bool',                'deleted'),
 301                  array('sql_attr_timestamp',            'post_time'),
 302                  array('sql_attr_timestamp',            'topic_last_post_time'),
 303                  array('sql_attr_string',            'post_subject'),
 304              ),
 305              'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
 306                  array('sql_query_pre',                'SET NAMES \'utf8\''),
 307                  array('sql_query_range',            ''),
 308                  array('sql_range_step',                ''),
 309                  array('sql_query',                    'SELECT
 310                          p.post_id AS id,
 311                          p.forum_id,
 312                          p.topic_id,
 313                          p.poster_id,
 314                          p.post_visibility,
 315                          CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
 316                          p.post_time,
 317                          p.post_subject,
 318                          p.post_subject as title,
 319                          p.post_text as data,
 320                          t.topic_last_post_time,
 321                          0 as deleted
 322                      FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
 323                      WHERE
 324                          p.topic_id = t.topic_id
 325                          AND p.post_id >=  ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
 326                  array('sql_query_post_index',        ''),
 327              ),
 328              'index index_phpbb_' . $this->id . '_main' => array(
 329                  array('path',                        $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
 330                  array('source',                        'source_phpbb_' . $this->id . '_main'),
 331                  array('docinfo',                    'extern'),
 332                  array('morphology',                    'none'),
 333                  array('stopwords',                    ''),
 334                  array('wordforms',                    '  # optional, specify path to wordforms file. See ./docs/sphinx_wordforms.txt for example'),
 335                  array('exceptions',                    '  # optional, specify path to exceptions file. See ./docs/sphinx_exceptions.txt for example'),
 336                  array('min_word_len',                '2'),
 337                  array('charset_table',                'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
 338                  array('ignore_chars',                 'U+0027, U+002C'),
 339                  array('min_prefix_len',                '3 # Minimum number of characters for wildcard searches by prefix (min 1). Default is 3. If specified, set min_infix_len to 0'),
 340                  array('min_infix_len',                '0 # Minimum number of characters for wildcard searches by infix (min 2). If specified, set min_prefix_len to 0'),
 341                  array('html_strip',                    '1'),
 342                  array('index_exact_words',            '0 # Set to 1 to enable exact search operator. Requires wordforms or morphology'),
 343                  array('blend_chars',                 'U+23, U+24, U+25, U+26, U+40'),
 344              ),
 345              'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
 346                  array('path',                        $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
 347                  array('source',                        'source_phpbb_' . $this->id . '_delta'),
 348              ),
 349              'indexer' => array(
 350                  array('mem_limit',                    $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
 351              ),
 352              'searchd' => array(
 353                  array('listen'    ,                    ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')),
 354                  array('log',                        $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
 355                  array('query_log',                    $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
 356                  array('read_timeout',                '5'),
 357                  array('max_children',                '30'),
 358                  array('pid_file',                    $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
 359                  array('binlog_path',                $this->config['fulltext_sphinx_data_path']),
 360              ),
 361          );
 362  
 363          $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
 364          $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
 365  
 366          /**
 367          * Allow adding/changing the Sphinx configuration data
 368          *
 369          * @event core.search_sphinx_modify_config_data
 370          * @var    array    config_data    Array with the Sphinx configuration data
 371          * @var    array    non_unique    Array with the Sphinx non-unique variables to delete
 372          * @var    array    delete        Array with the Sphinx variables to delete
 373          * @since 3.1.7-RC1
 374          */
 375          $vars = array(
 376              'config_data',
 377              'non_unique',
 378              'delete',
 379          );
 380          extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_modify_config_data', compact($vars)));
 381  
 382          foreach ($config_data as $section_name => $section_data)
 383          {
 384              $section = $config_object->get_section_by_name($section_name);
 385              if (!$section)
 386              {
 387                  $section = $config_object->add_section($section_name);
 388              }
 389  
 390              foreach ($delete as $key => $void)
 391              {
 392                  $section->delete_variables_by_name($key);
 393              }
 394  
 395              foreach ($non_unique as $key => $void)
 396              {
 397                  $section->delete_variables_by_name($key);
 398              }
 399  
 400              foreach ($section_data as $entry)
 401              {
 402                  $key = $entry[0];
 403                  $value = $entry[1];
 404  
 405                  if (!isset($non_unique[$key]))
 406                  {
 407                      $variable = $section->get_variable_by_name($key);
 408                      if (!$variable)
 409                      {
 410                          $section->create_variable($key, $value);
 411                      }
 412                      else
 413                      {
 414                          $variable->set_value($value);
 415                      }
 416                  }
 417                  else
 418                  {
 419                      $section->create_variable($key, $value);
 420                  }
 421              }
 422          }
 423          $this->config_file_data = $config_object->get_data();
 424  
 425          return true;
 426      }
 427  
 428      /**
 429      * Splits keywords entered by a user into an array of words stored in $this->split_words
 430      * Stores the tidied search query in $this->search_query
 431      *
 432      * @param string $keywords Contains the keyword as entered by the user
 433      * @param string $terms is either 'all' or 'any'
 434      * @return false if no valid keywords were found and otherwise true
 435      */
 436  	public function split_keywords(&$keywords, $terms)
 437      {
 438          // Keep quotes and new lines
 439          $keywords = str_replace(['&quot;', "\n"], ['"', ' '], trim($keywords));
 440  
 441          if ($terms == 'all')
 442          {
 443              // Replaces verbal operators OR and NOT with special characters | and -, unless appearing within quotation marks
 444              $match        = ['#\sor\s(?=([^"]*"[^"]*")*[^"]*$)#i', '#\snot\s(?=([^"]*"[^"]*")*[^"]*$)#i'];
 445              $replace    = [' | ', ' -'];
 446  
 447              $keywords = preg_replace($match, $replace, $keywords);
 448              $this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED);
 449          }
 450          else
 451          {
 452              $match = ['\\', '(',')', '|', '!', '@', '~', '/', '^', '$', '=', '&amp;', '&lt;', '&gt;'];
 453  
 454              $keywords = str_replace($match, ' ', $keywords);
 455              $this->sphinx->SetMatchMode(SPH_MATCH_ANY);
 456          }
 457  
 458          if (strlen($keywords) > 0)
 459          {
 460              $this->search_query = str_replace('"', '&quot;', $keywords);
 461              return true;
 462          }
 463  
 464          return false;
 465      }
 466  
 467      /**
 468      * Cleans search query passed into Sphinx search engine, as follows:
 469      * 1. Hyphenated words are replaced with keyword search for either the exact phrase with spaces
 470      *    or as a single word without spaces eg search for "know-it-all" becomes ("know it all"|"knowitall*")
 471      * 2. Words with apostrophes are contracted eg "it's" becomes "its"
 472      * 3. <, >, " and & are decoded from HTML entities.
 473      * 4. Following special characters used as search operators in Sphinx are preserved when used with correct syntax:
 474      *    (a) quorum matching: "the world is a wonderful place"/3
 475      *        Finds 3 of the words within the phrase. Number must be between 1 and 9.
 476      *    (b) proximity search: "hello world"~10
 477      *        Finds hello and world within 10 words of each other. Number can be between 1 and 99.
 478      *    (c) strict word order: aaa << bbb << ccc
 479      *        Finds "aaa" only where it appears before "bbb" and only where "bbb" appears before "ccc".
 480      *    (d) exact match operator: if lemmatizer or stemming enabled,
 481      *        search will find exact match only and ignore other grammatical forms of the same word stem.
 482      *        eg. raining =cats and =dogs
 483      *            will not return "raining cat and dog"
 484      *        eg. ="search this exact phrase"
 485      *            will not return "searched this exact phrase", "searching these exact phrases".
 486      * 5. Special characters /, ~, << and = not complying with the correct syntax
 487      *    and other reserved operators are escaped and searched literally.
 488      *    Special characters not explicitly listed in charset_table or blend_chars in sphinx.conf
 489      *    will not be indexed and keywords containing them will be ignored by Sphinx.
 490      *    By default, only $, %, & and @ characters are indexed and searchable.
 491      *    String transformation is in backend only and not visible to the end user
 492      *    nor reflected in the results page URL or keyword highlighting.
 493      *
 494      * @param string    $search_string
 495      * @return string
 496      */
 497  	public function sphinx_clean_search_string($search_string)
 498      {
 499          $from = ['@', '^', '$', '!', '&lt;', '&gt;', '&quot;', '&amp;', '\''];
 500          $to = ['\@', '\^', '\$', '\!', '<', '>', '"', '&', ''];
 501  
 502          $search_string = str_replace($from, $to, $search_string);
 503  
 504          $search_string = strrev($search_string);
 505          $search_string = preg_replace(['#\/(?!"[^"]+")#', '#~(?!"[^"]+")#'], ['/\\', '~\\'], $search_string);
 506          $search_string = strrev($search_string);
 507  
 508          $match = ['#(/|\\\\/)(?![1-9](\s|$))#', '#(~|\\\\~)(?!\d{1,2}(\s|$))#', '#((?:\p{L}|\p{N})+)-((?:\p{L}|\p{N})+)(?:-((?:\p{L}|\p{N})+))?(?:-((?:\p{L}|\p{N})+))?#i', '#<<\s*$#', '#(\S\K=|=(?=\s)|=$)#'];
 509          $replace = ['\/', '\~', '("$1 $2 $3 $4"|$1$2$3$4*)', '\<\<', '\='];
 510  
 511          $search_string = preg_replace($match, $replace, $search_string);
 512          $search_string = preg_replace('#\s+"\|#', '"|', $search_string);
 513  
 514          /**
 515          * OPTIONAL: Thousands separator stripped from numbers, eg search for '90,000' is queried as '90000'.
 516          * By default commas are stripped from search index so that '90,000' is indexed as '90000'
 517          */
 518          // $search_string = preg_replace('#[0-9]{1,3}\K,(?=[0-9]{3})#', '', $search_string);
 519  
 520          return $search_string;
 521      }
 522  
 523      /**
 524      * Performs a search on keywords depending on display specific params. You have to run split_keywords() first
 525      *
 526      * @param    string        $type                contains either posts or topics depending on what should be searched for
 527      * @param    string        $fields                contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
 528      * @param    string        $terms                is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
 529      * @param    array        $sort_by_sql        contains SQL code for the ORDER BY part of a query
 530      * @param    string        $sort_key            is the key of $sort_by_sql for the selected sorting
 531      * @param    string        $sort_dir            is either a or d representing ASC and DESC
 532      * @param    string        $sort_days            specifies the maximum amount of days a post may be old
 533      * @param    array        $ex_fid_ary            specifies an array of forum ids which should not be searched
 534      * @param    string        $post_visibility    specifies which types of posts the user can view in which forums
 535      * @param    int            $topic_id            is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
 536      * @param    array        $author_ary            an array of author ids if the author should be ignored during the search the array is empty
 537      * @param    string        $author_name        specifies the author match, when ANONYMOUS is also a search-match
 538      * @param    array        &$id_ary            passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
 539      * @param    int            $start                indicates the first index of the page
 540      * @param    int            $per_page            number of ids each page is supposed to contain
 541      * @return    boolean|int                        total number of results
 542      */
 543  	public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
 544      {
 545          global $user, $phpbb_log;
 546  
 547          // No keywords? No posts.
 548          if (!strlen($this->search_query) && !count($author_ary))
 549          {
 550              return false;
 551          }
 552  
 553          $id_ary = array();
 554  
 555          // Sorting
 556  
 557          if ($type == 'topics')
 558          {
 559              switch ($sort_key)
 560              {
 561                  case 'a':
 562                      $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
 563                  break;
 564  
 565                  case 'f':
 566                      $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
 567                  break;
 568  
 569                  case 'i':
 570  
 571                  case 's':
 572                      $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
 573                  break;
 574  
 575                  case 't':
 576  
 577                  default:
 578                      $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
 579                  break;
 580              }
 581          }
 582          else
 583          {
 584              switch ($sort_key)
 585              {
 586                  case 'a':
 587                      $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id');
 588                  break;
 589  
 590                  case 'f':
 591                      $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id');
 592                  break;
 593  
 594                  case 'i':
 595  
 596                  case 's':
 597                      $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject');
 598                  break;
 599  
 600                  case 't':
 601  
 602                  default:
 603                      $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time');
 604                  break;
 605              }
 606          }
 607  
 608          // Most narrow filters first
 609          if ($topic_id)
 610          {
 611              $this->sphinx->SetFilter('topic_id', array($topic_id));
 612          }
 613  
 614          /**
 615          * Allow modifying the Sphinx search options
 616          *
 617          * @event core.search_sphinx_keywords_modify_options
 618          * @var    string    type                Searching type ('posts', 'topics')
 619          * @var    string    fields                Searching fields ('titleonly', 'msgonly', 'firstpost', 'all')
 620          * @var    string    terms                Searching terms ('all', 'any')
 621          * @var    int        sort_days            Time, in days, of the oldest possible post to list
 622          * @var    string    sort_key            The sort type used from the possible sort types
 623          * @var    int        topic_id            Limit the search to this topic_id only
 624          * @var    array    ex_fid_ary            Which forums not to search on
 625          * @var    string    post_visibility        Post visibility data
 626          * @var    array    author_ary            Array of user_id containing the users to filter the results to
 627          * @var    string    author_name            The username to search on
 628          * @var    object    sphinx                The Sphinx searchd client object
 629          * @since 3.1.7-RC1
 630          */
 631          $sphinx = $this->sphinx;
 632          $vars = array(
 633              'type',
 634              'fields',
 635              'terms',
 636              'sort_days',
 637              'sort_key',
 638              'topic_id',
 639              'ex_fid_ary',
 640              'post_visibility',
 641              'author_ary',
 642              'author_name',
 643              'sphinx',
 644          );
 645          extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_keywords_modify_options', compact($vars)));
 646          $this->sphinx = $sphinx;
 647          unset($sphinx);
 648  
 649          $search_query_prefix = '';
 650  
 651          switch ($fields)
 652          {
 653              case 'titleonly':
 654                  // Only search the title
 655                  if ($terms == 'all')
 656                  {
 657                      $search_query_prefix = '@title ';
 658                  }
 659                  // Weight for the title
 660                  $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
 661                  // 1 is first_post, 0 is not first post
 662                  $this->sphinx->SetFilter('topic_first_post', array(1));
 663              break;
 664  
 665              case 'msgonly':
 666                  // Only search the body
 667                  if ($terms == 'all')
 668                  {
 669                      $search_query_prefix = '@data ';
 670                  }
 671                  // Weight for the body
 672                  $this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5));
 673              break;
 674  
 675              case 'firstpost':
 676                  // More relative weight for the title, also search the body
 677                  $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
 678                  // 1 is first_post, 0 is not first post
 679                  $this->sphinx->SetFilter('topic_first_post', array(1));
 680              break;
 681  
 682              default:
 683                  // More relative weight for the title, also search the body
 684                  $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
 685              break;
 686          }
 687  
 688          if (count($author_ary))
 689          {
 690              $this->sphinx->SetFilter('poster_id', $author_ary);
 691          }
 692  
 693          // As this is not simply possible at the moment, we limit the result to approved posts.
 694          // This will make it impossible for moderators to search unapproved and softdeleted posts,
 695          // but at least it will also cause the same for normal users.
 696          $this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED));
 697  
 698          if (count($ex_fid_ary))
 699          {
 700              // All forums that a user is allowed to access
 701              $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true))));
 702              // All forums that the user wants to and can search in
 703              $search_forums = array_diff($fid_ary, $ex_fid_ary);
 704  
 705              if (count($search_forums))
 706              {
 707                  $this->sphinx->SetFilter('forum_id', $search_forums);
 708              }
 709          }
 710  
 711          $this->sphinx->SetFilter('deleted', array(0));
 712  
 713          $this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
 714          $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
 715  
 716          // Could be connection to localhost:9312 failed (errno=111,
 717          // msg=Connection refused) during rotate, retry if so
 718          $retries = SPHINX_CONNECT_RETRIES;
 719          while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
 720          {
 721              usleep(SPHINX_CONNECT_WAIT_TIME);
 722              $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
 723          }
 724  
 725          if ($this->sphinx->GetLastError())
 726          {
 727              $phpbb_log->add('critical', $user->data['user_id'], $user->ip, 'LOG_SPHINX_ERROR', false, array($this->sphinx->GetLastError()));
 728              if ($this->auth->acl_get('a_'))
 729              {
 730                  trigger_error($this->user->lang('SPHINX_SEARCH_FAILED', $this->sphinx->GetLastError()));
 731              }
 732              else
 733              {
 734                  trigger_error($this->user->lang('SPHINX_SEARCH_FAILED_LOG'));
 735              }
 736          }
 737  
 738          $result_count = $result['total_found'];
 739  
 740          if ($result_count && $start >= $result_count)
 741          {
 742              $start = floor(($result_count - 1) / $per_page) * $per_page;
 743  
 744              $this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
 745              $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
 746  
 747              // Could be connection to localhost:9312 failed (errno=111,
 748              // msg=Connection refused) during rotate, retry if so
 749              $retries = SPHINX_CONNECT_RETRIES;
 750              while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
 751              {
 752                  usleep(SPHINX_CONNECT_WAIT_TIME);
 753                  $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
 754              }
 755          }
 756  
 757          $id_ary = array();
 758          if (isset($result['matches']))
 759          {
 760              if ($type == 'posts')
 761              {
 762                  $id_ary = array_keys($result['matches']);
 763              }
 764              else
 765              {
 766                  foreach ($result['matches'] as $key => $value)
 767                  {
 768                      $id_ary[] = $value['attrs']['topic_id'];
 769                  }
 770              }
 771          }
 772          else
 773          {
 774              return false;
 775          }
 776  
 777          $id_ary = array_slice($id_ary, 0, (int) $per_page);
 778  
 779          return $result_count;
 780      }
 781  
 782      /**
 783      * Performs a search on an author's posts without caring about message contents. Depends on display specific params
 784      *
 785      * @param    string        $type                contains either posts or topics depending on what should be searched for
 786      * @param    boolean        $firstpost_only        if true, only topic starting posts will be considered
 787      * @param    array        $sort_by_sql        contains SQL code for the ORDER BY part of a query
 788      * @param    string        $sort_key            is the key of $sort_by_sql for the selected sorting
 789      * @param    string        $sort_dir            is either a or d representing ASC and DESC
 790      * @param    string        $sort_days            specifies the maximum amount of days a post may be old
 791      * @param    array        $ex_fid_ary            specifies an array of forum ids which should not be searched
 792      * @param    string        $post_visibility    specifies which types of posts the user can view in which forums
 793      * @param    int            $topic_id            is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
 794      * @param    array        $author_ary            an array of author ids
 795      * @param    string        $author_name        specifies the author match, when ANONYMOUS is also a search-match
 796      * @param    array        &$id_ary            passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
 797      * @param    int            $start                indicates the first index of the page
 798      * @param    int            $per_page            number of ids each page is supposed to contain
 799      * @return    boolean|int                        total number of results
 800      */
 801  	public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page)
 802      {
 803          $this->search_query = '';
 804  
 805          $this->sphinx->SetMatchMode(SPH_MATCH_FULLSCAN);
 806          $fields = ($firstpost_only) ? 'firstpost' : 'all';
 807          $terms = 'all';
 808          return $this->keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, $id_ary, $start, $per_page);
 809      }
 810  
 811      /**
 812       * Updates wordlist and wordmatch tables when a message is posted or changed
 813       *
 814       * @param    string    $mode    Contains the post mode: edit, post, reply, quote
 815       * @param    int    $post_id    The id of the post which is modified/created
 816       * @param    string    &$message    New or updated post content
 817       * @param    string    &$subject    New or updated post subject
 818       * @param    int    $poster_id    Post author's user id
 819       * @param    int    $forum_id    The id of the forum in which the post is located
 820       */
 821  	public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
 822      {
 823          /**
 824          * Event to modify method arguments before the Sphinx search index is updated
 825          *
 826          * @event core.search_sphinx_index_before
 827          * @var string    mode                Contains the post mode: edit, post, reply, quote
 828          * @var int        post_id                The id of the post which is modified/created
 829          * @var string    message                New or updated post content
 830          * @var string    subject                New or updated post subject
 831          * @var int        poster_id            Post author's user id
 832          * @var int        forum_id            The id of the forum in which the post is located
 833          * @since 3.2.3-RC1
 834          */
 835          $vars = array(
 836              'mode',
 837              'post_id',
 838              'message',
 839              'subject',
 840              'poster_id',
 841              'forum_id',
 842          );
 843          extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_index_before', compact($vars)));
 844  
 845          if ($mode == 'edit')
 846          {
 847              $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int) $post_id => array((int) $forum_id, (int) $poster_id)));
 848          }
 849          else if ($mode != 'post' && $post_id)
 850          {
 851              // Update topic_last_post_time for full topic
 852              $sql_array = array(
 853                  'SELECT'    => 'p1.post_id',
 854                  'FROM'        => array(
 855                      POSTS_TABLE    => 'p1',
 856                  ),
 857                  'LEFT_JOIN'    => array(array(
 858                      'FROM'    => array(
 859                          POSTS_TABLE    => 'p2'
 860                      ),
 861                      'ON'    => 'p1.topic_id = p2.topic_id',
 862                  )),
 863                  'WHERE' => 'p2.post_id = ' . ((int) $post_id),
 864              );
 865  
 866              $sql = $this->db->sql_build_query('SELECT', $sql_array);
 867              $result = $this->db->sql_query($sql);
 868  
 869              $post_updates = array();
 870              $post_time = time();
 871              while ($row = $this->db->sql_fetchrow($result))
 872              {
 873                  $post_updates[(int) $row['post_id']] = array($post_time);
 874              }
 875              $this->db->sql_freeresult($result);
 876  
 877              if (count($post_updates))
 878              {
 879                  $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates);
 880              }
 881          }
 882      }
 883  
 884      /**
 885      * Delete a post from the index after it was deleted
 886      */
 887  	public function index_remove($post_ids, $author_ids, $forum_ids)
 888      {
 889          $values = array();
 890          foreach ($post_ids as $post_id)
 891          {
 892              $values[$post_id] = array(1);
 893          }
 894  
 895          $this->sphinx->UpdateAttributes($this->indexes, array('deleted'), $values);
 896      }
 897  
 898      /**
 899      * Nothing needs to be destroyed
 900      */
 901  	public function tidy($create = false)
 902      {
 903          $this->config->set('search_last_gc', time(), false);
 904      }
 905  
 906      /**
 907      * Create sphinx table
 908      *
 909      * @return string|bool error string is returned incase of errors otherwise false
 910      */
 911  	public function create_index($acp_module, $u_action)
 912      {
 913          if (!$this->index_created())
 914          {
 915              $table_data = array(
 916                  'COLUMNS'    => array(
 917                      'counter_id'    => array('UINT', 0),
 918                      'max_doc_id'    => array('UINT', 0),
 919                  ),
 920                  'PRIMARY_KEY'    => 'counter_id',
 921              );
 922              $this->db_tools->sql_create_table(SPHINX_TABLE, $table_data);
 923  
 924              $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE;
 925              $this->db->sql_query($sql);
 926  
 927              $data = array(
 928                  'counter_id'    => '1',
 929                  'max_doc_id'    => '0',
 930              );
 931              $sql = 'INSERT INTO ' . SPHINX_TABLE . ' ' . $this->db->sql_build_array('INSERT', $data);
 932              $this->db->sql_query($sql);
 933          }
 934  
 935          return false;
 936      }
 937  
 938      /**
 939      * Drop sphinx table
 940      *
 941      * @return string|bool error string is returned incase of errors otherwise false
 942      */
 943  	public function delete_index($acp_module, $u_action)
 944      {
 945          if (!$this->index_created())
 946          {
 947              return false;
 948          }
 949  
 950          $this->db_tools->sql_table_drop(SPHINX_TABLE);
 951  
 952          return false;
 953      }
 954  
 955      /**
 956      * Returns true if the sphinx table was created
 957      *
 958      * @return bool true if sphinx table was created
 959      */
 960  	public function index_created($allow_new_files = true)
 961      {
 962          $created = false;
 963  
 964          if ($this->db_tools->sql_table_exists(SPHINX_TABLE))
 965          {
 966              $created = true;
 967          }
 968  
 969          return $created;
 970      }
 971  
 972      /**
 973      * Returns an associative array containing information about the indexes
 974      *
 975      * @return string|bool Language string of error false otherwise
 976      */
 977  	public function index_stats()
 978      {
 979          if (empty($this->stats))
 980          {
 981              $this->get_stats();
 982          }
 983  
 984          return array(
 985              $this->user->lang['FULLTEXT_SPHINX_MAIN_POSTS']            => ($this->index_created()) ? $this->stats['main_posts'] : 0,
 986              $this->user->lang['FULLTEXT_SPHINX_DELTA_POSTS']            => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0,
 987              $this->user->lang['FULLTEXT_MYSQL_TOTAL_POSTS']            => ($this->index_created()) ? $this->stats['total_posts'] : 0,
 988          );
 989      }
 990  
 991      /**
 992      * Collects stats that can be displayed on the index maintenance page
 993      */
 994  	protected function get_stats()
 995      {
 996          if ($this->index_created())
 997          {
 998              $sql = 'SELECT COUNT(post_id) as total_posts
 999                  FROM ' . POSTS_TABLE;
1000              $result = $this->db->sql_query($sql);
1001              $this->stats['total_posts'] = (int) $this->db->sql_fetchfield('total_posts');
1002              $this->db->sql_freeresult($result);
1003  
1004              $sql = 'SELECT COUNT(p.post_id) as main_posts
1005                  FROM ' . POSTS_TABLE . ' p, ' . SPHINX_TABLE . ' m
1006                  WHERE p.post_id <= m.max_doc_id
1007                      AND m.counter_id = 1';
1008              $result = $this->db->sql_query($sql);
1009              $this->stats['main_posts'] = (int) $this->db->sql_fetchfield('main_posts');
1010              $this->db->sql_freeresult($result);
1011          }
1012      }
1013  
1014      /**
1015      * Returns a list of options for the ACP to display
1016      *
1017      * @return associative array containing template and config variables
1018      */
1019  	public function acp()
1020      {
1021          $config_vars = array(
1022              'fulltext_sphinx_data_path' => 'string',
1023              'fulltext_sphinx_host' => 'string',
1024              'fulltext_sphinx_port' => 'string',
1025              'fulltext_sphinx_indexer_mem_limit' => 'int',
1026          );
1027  
1028          $tpl = '
1029          <span class="error">' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE']. '</span>
1030          <dl>
1031              <dt><label for="fulltext_sphinx_data_path">' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '</span></dt>
1032              <dd><input id="fulltext_sphinx_data_path" type="text" size="40" maxlength="255" name="config[fulltext_sphinx_data_path]" value="' . $this->config['fulltext_sphinx_data_path'] . '" /></dd>
1033          </dl>
1034          <dl>
1035              <dt><label for="fulltext_sphinx_host">' . $this->user->lang['FULLTEXT_SPHINX_HOST'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_HOST_EXPLAIN'] . '</span></dt>
1036              <dd><input id="fulltext_sphinx_host" type="text" size="40" maxlength="255" name="config[fulltext_sphinx_host]" value="' . $this->config['fulltext_sphinx_host'] . '" /></dd>
1037          </dl>
1038          <dl>
1039              <dt><label for="fulltext_sphinx_port">' . $this->user->lang['FULLTEXT_SPHINX_PORT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '</span></dt>
1040              <dd><input id="fulltext_sphinx_port" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_port]" value="' . $this->config['fulltext_sphinx_port'] . '" /></dd>
1041          </dl>
1042          <dl>
1043              <dt><label for="fulltext_sphinx_indexer_mem_limit">' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '</span></dt>
1044              <dd><input id="fulltext_sphinx_indexer_mem_limit" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_indexer_mem_limit]" value="' . $this->config['fulltext_sphinx_indexer_mem_limit'] . '" /> ' . $this->user->lang['MIB'] . '</dd>
1045          </dl>
1046          <dl>
1047              <dt><label for="fulltext_sphinx_config_file">' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '</span></dt>
1048              <dd>' . (($this->config_generate()) ? '<textarea readonly="readonly" rows="6" id="sphinx_config_data">' . htmlspecialchars($this->config_file_data, ENT_COMPAT) . '</textarea>' : $this->config_file_data) . '</dd>
1049          <dl>
1050          ';
1051  
1052          // These are fields required in the config table
1053          return array(
1054              'tpl'        => $tpl,
1055              'config'    => $config_vars
1056          );
1057      }
1058  }


Generated: Thu Mar 24 21:31:15 2022 Cross-referenced by PHPXref 0.7.1