[ Index ]

PHP Cross Reference of DokuWiki

title

Body

[close]

/inc/ -> indexer.php (source)

   1  <?php
   2  /**
   3   * Functions to create the fulltext search index
   4   *
   5   * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
   6   * @author     Andreas Gohr <andi@splitbrain.org>
   7   * @author     Tom N Harris <tnharris@whoopdedo.org>
   8   */
   9  
  10  if(!defined('DOKU_INC')) die('meh.');
  11  
  12  // Version tag used to force rebuild on upgrade
  13  define('INDEXER_VERSION', 8);
  14  
  15  // set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
  16  if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
  17  
  18  // Asian characters are handled as words. The following regexp defines the
  19  // Unicode-Ranges for Asian characters
  20  // Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
  21  // I'm no language expert. If you think some ranges are wrongly chosen or
  22  // a range is missing, please contact me
  23  define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
  24  define('IDX_ASIAN2','['.
  25                     '\x{2E80}-\x{3040}'.  // CJK -> Hangul
  26                     '\x{309D}-\x{30A0}'.
  27                     '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
  28                     '\x{F900}-\x{FAFF}'.  // CJK Compatibility Ideographs
  29                     '\x{FE30}-\x{FE4F}'.  // CJK Compatibility Forms
  30                     "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B
  31                     "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C
  32                     "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D
  33                     "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement
  34                     ']');
  35  define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two characters)
  36                     '\x{3042}\x{3044}\x{3046}\x{3048}'.
  37                     '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
  38                     '\x{3084}\x{3086}\x{3088}-\x{308D}'.
  39                     '\x{308F}-\x{3094}'.
  40                     '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
  41                     '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
  42                     '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
  43                     '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
  44                     ']['.
  45                     '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
  46                     '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
  47                     '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
  48                     '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
  49                     '\x{31F0}-\x{31FF}'.
  50                     ']?');
  51  define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
  52  
  53  /**
  54   * Version of the indexer taking into consideration the external tokenizer.
  55   * The indexer is only compatible with data written by the same version.
  56   *
  57   * @triggers INDEXER_VERSION_GET
  58   * Plugins that modify what gets indexed should hook this event and
  59   * add their version info to the event data like so:
  60   *     $data[$plugin_name] = $plugin_version;
  61   *
  62   * @author Tom N Harris <tnharris@whoopdedo.org>
  63   * @author Michael Hamann <michael@content-space.de>
  64   *
  65   * @return int|string
  66   */
  67  function idx_get_version(){
  68      static $indexer_version = null;
  69      if ($indexer_version == null) {
  70          $version = INDEXER_VERSION;
  71  
  72          // DokuWiki version is included for the convenience of plugins
  73          $data = array('dokuwiki'=>$version);
  74          trigger_event('INDEXER_VERSION_GET', $data, null, false);
  75          unset($data['dokuwiki']); // this needs to be first
  76          ksort($data);
  77          foreach ($data as $plugin=>$vers)
  78              $version .= '+'.$plugin.'='.$vers;
  79          $indexer_version = $version;
  80      }
  81      return $indexer_version;
  82  }
  83  
  84  /**
  85   * Measure the length of a string.
  86   * Differs from strlen in handling of asian characters.
  87   *
  88   * @author Tom N Harris <tnharris@whoopdedo.org>
  89   *
  90   * @param string $w
  91   * @return int
  92   */
  93  function wordlen($w){
  94      $l = strlen($w);
  95      // If left alone, all chinese "words" will get put into w3.idx
  96      // So the "length" of a "word" is faked
  97      if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) {
  98          foreach($leadbytes[0] as $b)
  99              $l += ord($b) - 0xE1;
 100      }
 101      return $l;
 102  }
 103  
 104  /**
 105   * Class that encapsulates operations on the indexer database.
 106   *
 107   * @author Tom N Harris <tnharris@whoopdedo.org>
 108   */
 109  class Doku_Indexer {
 110      /**
 111       * @var array $pidCache Cache for getPID()
 112       */
 113      protected $pidCache = array();
 114  
 115      /**
 116       * Adds the contents of a page to the fulltext index
 117       *
 118       * The added text replaces previous words for the same page.
 119       * An empty value erases the page.
 120       *
 121       * @param string    $page   a page name
 122       * @param string    $text   the body of the page
 123       * @return string|boolean  the function completed successfully
 124       *
 125       * @author Tom N Harris <tnharris@whoopdedo.org>
 126       * @author Andreas Gohr <andi@splitbrain.org>
 127       */
 128      public function addPageWords($page, $text) {
 129          if (!$this->lock())
 130              return "locked";
 131  
 132          // load known documents
 133          $pid = $this->getPIDNoLock($page);
 134          if ($pid === false) {
 135              $this->unlock();
 136              return false;
 137          }
 138  
 139          $pagewords = array();
 140          // get word usage in page
 141          $words = $this->getPageWords($text);
 142          if ($words === false) {
 143              $this->unlock();
 144              return false;
 145          }
 146  
 147          if (!empty($words)) {
 148              foreach (array_keys($words) as $wlen) {
 149                  $index = $this->getIndex('i', $wlen);
 150                  foreach ($words[$wlen] as $wid => $freq) {
 151                      $idx = ($wid<count($index)) ? $index[$wid] : '';
 152                      $index[$wid] = $this->updateTuple($idx, $pid, $freq);
 153                      $pagewords[] = "$wlen*$wid";
 154                  }
 155                  if (!$this->saveIndex('i', $wlen, $index)) {
 156                      $this->unlock();
 157                      return false;
 158                  }
 159              }
 160          }
 161  
 162          // Remove obsolete index entries
 163          $pageword_idx = $this->getIndexKey('pageword', '', $pid);
 164          if ($pageword_idx !== '') {
 165              $oldwords = explode(':',$pageword_idx);
 166              $delwords = array_diff($oldwords, $pagewords);
 167              $upwords = array();
 168              foreach ($delwords as $word) {
 169                  if ($word != '') {
 170                      list($wlen,$wid) = explode('*', $word);
 171                      $wid = (int)$wid;
 172                      $upwords[$wlen][] = $wid;
 173                  }
 174              }
 175              foreach ($upwords as $wlen => $widx) {
 176                  $index = $this->getIndex('i', $wlen);
 177                  foreach ($widx as $wid) {
 178                      $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
 179                  }
 180                  $this->saveIndex('i', $wlen, $index);
 181              }
 182          }
 183          // Save the reverse index
 184          $pageword_idx = join(':', $pagewords);
 185          if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) {
 186              $this->unlock();
 187              return false;
 188          }
 189  
 190          $this->unlock();
 191          return true;
 192      }
 193  
 194      /**
 195       * Split the words in a page and add them to the index.
 196       *
 197       * @param string    $text   content of the page
 198       * @return array            list of word IDs and number of times used
 199       *
 200       * @author Andreas Gohr <andi@splitbrain.org>
 201       * @author Christopher Smith <chris@jalakai.co.uk>
 202       * @author Tom N Harris <tnharris@whoopdedo.org>
 203       */
 204      protected function getPageWords($text) {
 205  
 206          $tokens = $this->tokenizer($text);
 207          $tokens = array_count_values($tokens);  // count the frequency of each token
 208  
 209          $words = array();
 210          foreach ($tokens as $w=>$c) {
 211              $l = wordlen($w);
 212              if (isset($words[$l])){
 213                  $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
 214              }else{
 215                  $words[$l] = array($w => $c);
 216              }
 217          }
 218  
 219          // arrive here with $words = array(wordlen => array(word => frequency))
 220          $word_idx_modified = false;
 221          $index = array();   //resulting index
 222          foreach (array_keys($words) as $wlen) {
 223              $word_idx = $this->getIndex('w', $wlen);
 224              foreach ($words[$wlen] as $word => $freq) {
 225                  $word = (string)$word;
 226                  $wid = array_search($word, $word_idx, true);
 227                  if ($wid === false) {
 228                      $wid = count($word_idx);
 229                      $word_idx[] = $word;
 230                      $word_idx_modified = true;
 231                  }
 232                  if (!isset($index[$wlen]))
 233                      $index[$wlen] = array();
 234                  $index[$wlen][$wid] = $freq;
 235              }
 236              // save back the word index
 237              if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx))
 238                  return false;
 239          }
 240  
 241          return $index;
 242      }
 243  
 244      /**
 245       * Add/update keys to/of the metadata index.
 246       *
 247       * Adding new keys does not remove other keys for the page.
 248       * An empty value will erase the key.
 249       * The $key parameter can be an array to add multiple keys. $value will
 250       * not be used if $key is an array.
 251       *
 252       * @param string    $page   a page name
 253       * @param mixed     $key    a key string or array of key=>value pairs
 254       * @param mixed     $value  the value or list of values
 255       * @return boolean|string     the function completed successfully
 256       *
 257       * @author Tom N Harris <tnharris@whoopdedo.org>
 258       * @author Michael Hamann <michael@content-space.de>
 259       */
 260      public function addMetaKeys($page, $key, $value=null) {
 261          if (!is_array($key)) {
 262              $key = array($key => $value);
 263          } elseif (!is_null($value)) {
 264              // $key is array, but $value is not null
 265              trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING);
 266          }
 267  
 268          if (!$this->lock())
 269              return "locked";
 270  
 271          // load known documents
 272          $pid = $this->getPIDNoLock($page);
 273          if ($pid === false) {
 274              $this->unlock();
 275              return false;
 276          }
 277  
 278          // Special handling for titles so the index file is simpler
 279          if (array_key_exists('title', $key)) {
 280              $value = $key['title'];
 281              if (is_array($value)) {
 282                  $value = $value[0];
 283              }
 284              $this->saveIndexKey('title', '', $pid, $value);
 285              unset($key['title']);
 286          }
 287  
 288          foreach ($key as $name => $values) {
 289              $metaname = idx_cleanName($name);
 290              $this->addIndexKey('metadata', '', $metaname);
 291              $metaidx = $this->getIndex($metaname.'_i', '');
 292              $metawords = $this->getIndex($metaname.'_w', '');
 293              $addwords = false;
 294  
 295              if (!is_array($values)) $values = array($values);
 296  
 297              $val_idx = $this->getIndexKey($metaname.'_p', '', $pid);
 298              if ($val_idx !== '') {
 299                  $val_idx = explode(':', $val_idx);
 300                  // -1 means remove, 0 keep, 1 add
 301                  $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1));
 302              } else {
 303                  $val_idx = array();
 304              }
 305  
 306              foreach ($values as $val) {
 307                  $val = (string)$val;
 308                  if ($val !== "") {
 309                      $id = array_search($val, $metawords, true);
 310                      if ($id === false) {
 311                          // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx
 312                          $id = count($metawords);
 313                          $metawords[$id] = $val;
 314                          $metaidx[$id] = '';
 315                          $addwords = true;
 316                      }
 317                      // test if value is already in the index
 318                      if (isset($val_idx[$id]) && $val_idx[$id] <= 0){
 319                          $val_idx[$id] = 0;
 320                      } else { // else add it
 321                          $val_idx[$id] = 1;
 322                      }
 323                  }
 324              }
 325  
 326              if ($addwords) {
 327                  $this->saveIndex($metaname.'_w', '', $metawords);
 328              }
 329              $vals_changed = false;
 330              foreach ($val_idx as $id => $action) {
 331                  if ($action == -1) {
 332                      $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0);
 333                      $vals_changed = true;
 334                      unset($val_idx[$id]);
 335                  } elseif ($action == 1) {
 336                      $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1);
 337                      $vals_changed = true;
 338                  }
 339              }
 340  
 341              if ($vals_changed) {
 342                  $this->saveIndex($metaname.'_i', '', $metaidx);
 343                  $val_idx = implode(':', array_keys($val_idx));
 344                  $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx);
 345              }
 346  
 347              unset($metaidx);
 348              unset($metawords);
 349          }
 350  
 351          $this->unlock();
 352          return true;
 353      }
 354  
 355      /**
 356       * Rename a page in the search index without changing the indexed content. This function doesn't check if the
 357       * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the
 358       * indexer and it deletes all previously indexed content of the new page.
 359       *
 360       * @param string $oldpage The old page name
 361       * @param string $newpage The new page name
 362       * @return string|bool If the page was successfully renamed, can be a message in the case of an error
 363       */
 364      public function renamePage($oldpage, $newpage) {
 365          if (!$this->lock()) return 'locked';
 366  
 367          $pages = $this->getPages();
 368  
 369          $id = array_search($oldpage, $pages, true);
 370          if ($id === false) {
 371              $this->unlock();
 372              return 'page is not in index';
 373          }
 374  
 375          $new_id = array_search($newpage, $pages, true);
 376          if ($new_id !== false) {
 377              // make sure the page is not in the index anymore
 378              if ($this->deletePageNoLock($newpage) !== true) {
 379                  return false;
 380              }
 381  
 382              $pages[$new_id] = 'deleted:'.time().rand(0, 9999);
 383          }
 384  
 385          $pages[$id] = $newpage;
 386  
 387          // update index
 388          if (!$this->saveIndex('page', '', $pages)) {
 389              $this->unlock();
 390              return false;
 391          }
 392  
 393          // reset the pid cache
 394          $this->pidCache = array();
 395  
 396          $this->unlock();
 397          return true;
 398      }
 399  
 400      /**
 401       * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages
 402       * will be updated.
 403       *
 404       * @param string $key       The metadata key of which a value shall be changed
 405       * @param string $oldvalue  The old value that shall be renamed
 406       * @param string $newvalue  The new value to which the old value shall be renamed, can exist (then values will be merged)
 407       * @return bool|string      If renaming the value has been successful, false or error message on error.
 408       */
 409      public function renameMetaValue($key, $oldvalue, $newvalue) {
 410          if (!$this->lock()) return 'locked';
 411  
 412          // change the relation references index
 413          $metavalues = $this->getIndex($key, '_w');
 414          $oldid = array_search($oldvalue, $metavalues, true);
 415          if ($oldid !== false) {
 416              $newid = array_search($newvalue, $metavalues, true);
 417              if ($newid !== false) {
 418                  // free memory
 419                  unset ($metavalues);
 420  
 421                  // okay, now we have two entries for the same value. we need to merge them.
 422                  $indexline = $this->getIndexKey($key.'_i', '', $oldid);
 423                  if ($indexline != '') {
 424                      $newindexline = $this->getIndexKey($key.'_i', '', $newid);
 425                      $pagekeys     = $this->getIndex($key.'_p', '');
 426                      $parts = explode(':', $indexline);
 427                      foreach ($parts as $part) {
 428                          list($id, $count) = explode('*', $part);
 429                          $newindexline =  $this->updateTuple($newindexline, $id, $count);
 430  
 431                          $keyline = explode(':', $pagekeys[$id]);
 432                          // remove old meta value
 433                          $keyline = array_diff($keyline, array($oldid));
 434                          // add new meta value when not already present
 435                          if (!in_array($newid, $keyline)) {
 436                              array_push($keyline, $newid);
 437                          }
 438                          $pagekeys[$id] = implode(':', $keyline);
 439                      }
 440                      $this->saveIndex($key.'_p', '', $pagekeys);
 441                      unset($pagekeys);
 442                      $this->saveIndexKey($key.'_i', '', $oldid, '');
 443                      $this->saveIndexKey($key.'_i', '', $newid, $newindexline);
 444                  }
 445              } else {
 446                  $metavalues[$oldid] = $newvalue;
 447                  if (!$this->saveIndex($key.'_w', '', $metavalues)) {
 448                      $this->unlock();
 449                      return false;
 450                  }
 451              }
 452          }
 453  
 454          $this->unlock();
 455          return true;
 456      }
 457  
 458      /**
 459       * Remove a page from the index
 460       *
 461       * Erases entries in all known indexes.
 462       *
 463       * @param string    $page   a page name
 464       * @return string|boolean  the function completed successfully
 465       *
 466       * @author Tom N Harris <tnharris@whoopdedo.org>
 467       */
 468      public function deletePage($page) {
 469          if (!$this->lock())
 470              return "locked";
 471  
 472          $result = $this->deletePageNoLock($page);
 473  
 474          $this->unlock();
 475  
 476          return $result;
 477      }
 478  
 479      /**
 480       * Remove a page from the index without locking the index, only use this function if the index is already locked
 481       *
 482       * Erases entries in all known indexes.
 483       *
 484       * @param string    $page   a page name
 485       * @return boolean          the function completed successfully
 486       *
 487       * @author Tom N Harris <tnharris@whoopdedo.org>
 488       */
 489      protected function deletePageNoLock($page) {
 490          // load known documents
 491          $pid = $this->getPIDNoLock($page);
 492          if ($pid === false) {
 493              return false;
 494          }
 495  
 496          // Remove obsolete index entries
 497          $pageword_idx = $this->getIndexKey('pageword', '', $pid);
 498          if ($pageword_idx !== '') {
 499              $delwords = explode(':',$pageword_idx);
 500              $upwords = array();
 501              foreach ($delwords as $word) {
 502                  if ($word != '') {
 503                      list($wlen,$wid) = explode('*', $word);
 504                      $wid = (int)$wid;
 505                      $upwords[$wlen][] = $wid;
 506                  }
 507              }
 508              foreach ($upwords as $wlen => $widx) {
 509                  $index = $this->getIndex('i', $wlen);
 510                  foreach ($widx as $wid) {
 511                      $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
 512                  }
 513                  $this->saveIndex('i', $wlen, $index);
 514              }
 515          }
 516          // Save the reverse index
 517          if (!$this->saveIndexKey('pageword', '', $pid, "")) {
 518              return false;
 519          }
 520  
 521          $this->saveIndexKey('title', '', $pid, "");
 522          $keyidx = $this->getIndex('metadata', '');
 523          foreach ($keyidx as $metaname) {
 524              $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid));
 525              $meta_idx = $this->getIndex($metaname.'_i', '');
 526              foreach ($val_idx as $id) {
 527                  if ($id === '') continue;
 528                  $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0);
 529              }
 530              $this->saveIndex($metaname.'_i', '', $meta_idx);
 531              $this->saveIndexKey($metaname.'_p', '', $pid, '');
 532          }
 533  
 534          return true;
 535      }
 536  
 537      /**
 538       * Clear the whole index
 539       *
 540       * @return bool If the index has been cleared successfully
 541       */
 542      public function clear() {
 543          global $conf;
 544  
 545          if (!$this->lock()) return false;
 546  
 547          @unlink($conf['indexdir'].'/page.idx');
 548          @unlink($conf['indexdir'].'/title.idx');
 549          @unlink($conf['indexdir'].'/pageword.idx');
 550          @unlink($conf['indexdir'].'/metadata.idx');
 551          $dir = @opendir($conf['indexdir']);
 552          if($dir!==false){
 553              while(($f = readdir($dir)) !== false){
 554                  if(substr($f,-4)=='.idx' &&
 555                      (substr($f,0,1)=='i' || substr($f,0,1)=='w'
 556                          || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx'))
 557                      @unlink($conf['indexdir']."/$f");
 558              }
 559          }
 560          @unlink($conf['indexdir'].'/lengths.idx');
 561  
 562          // clear the pid cache
 563          $this->pidCache = array();
 564  
 565          $this->unlock();
 566          return true;
 567      }
 568  
 569      /**
 570       * Split the text into words for fulltext search
 571       *
 572       * TODO: does this also need &$stopwords ?
 573       *
 574       * @triggers INDEXER_TEXT_PREPARE
 575       * This event allows plugins to modify the text before it gets tokenized.
 576       * Plugins intercepting this event should also intercept INDEX_VERSION_GET
 577       *
 578       * @param string    $text   plain text
 579       * @param boolean   $wc     are wildcards allowed?
 580       * @return array            list of words in the text
 581       *
 582       * @author Tom N Harris <tnharris@whoopdedo.org>
 583       * @author Andreas Gohr <andi@splitbrain.org>
 584       */
 585      public function tokenizer($text, $wc=false) {
 586          $wc = ($wc) ? '' : '\*';
 587          $stopwords =& idx_get_stopwords();
 588  
 589          // prepare the text to be tokenized
 590          $evt = new Doku_Event('INDEXER_TEXT_PREPARE', $text);
 591          if ($evt->advise_before(true)) {
 592              if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
 593                  // handle asian chars as single words (may fail on older PHP version)
 594                  $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text);
 595                  if (!is_null($asia)) $text = $asia; // recover from regexp falure
 596              }
 597          }
 598          $evt->advise_after();
 599          unset($evt);
 600  
 601          $text = strtr($text,
 602                         array(
 603                             "\r" => ' ',
 604                             "\n" => ' ',
 605                             "\t" => ' ',
 606                             "\xC2\xAD" => '', //soft-hyphen
 607                         )
 608                       );
 609          if (preg_match('/[^0-9A-Za-z ]/u', $text))
 610              $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc);
 611  
 612          $wordlist = explode(' ', $text);
 613          foreach ($wordlist as $i => $word) {
 614              $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
 615                  utf8_strtolower($word) : strtolower($word);
 616          }
 617  
 618          foreach ($wordlist as $i => $word) {
 619              if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH)
 620                || array_search($word, $stopwords, true) !== false)
 621                  unset($wordlist[$i]);
 622          }
 623          return array_values($wordlist);
 624      }
 625  
 626      /**
 627       * Get the numeric PID of a page
 628       *
 629       * @param string $page The page to get the PID for
 630       * @return bool|int The page id on success, false on error
 631       */
 632      public function getPID($page) {
 633          // return PID without locking when it is in the cache
 634          if (isset($this->pidCache[$page])) return $this->pidCache[$page];
 635  
 636          if (!$this->lock())
 637              return false;
 638  
 639          // load known documents
 640          $pid = $this->getPIDNoLock($page);
 641          if ($pid === false) {
 642              $this->unlock();
 643              return false;
 644          }
 645  
 646          $this->unlock();
 647          return $pid;
 648      }
 649  
 650      /**
 651       * Get the numeric PID of a page without locking the index.
 652       * Only use this function when the index is already locked.
 653       *
 654       * @param string $page The page to get the PID for
 655       * @return bool|int The page id on success, false on error
 656       */
 657      protected function getPIDNoLock($page) {
 658          // avoid expensive addIndexKey operation for the most recently requested pages by using a cache
 659          if (isset($this->pidCache[$page])) return $this->pidCache[$page];
 660          $pid = $this->addIndexKey('page', '', $page);
 661          // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently
 662          // added item will be requested again
 663          if (count($this->pidCache) > 10) array_shift($this->pidCache);
 664          $this->pidCache[$page] = $pid;
 665          return $pid;
 666      }
 667  
 668      /**
 669       * Get the page id of a numeric PID
 670       *
 671       * @param int $pid The PID to get the page id for
 672       * @return string The page id
 673       */
 674      public function getPageFromPID($pid) {
 675          return $this->getIndexKey('page', '', $pid);
 676      }
 677  
 678      /**
 679       * Find pages in the fulltext index containing the words,
 680       *
 681       * The search words must be pre-tokenized, meaning only letters and
 682       * numbers with an optional wildcard
 683       *
 684       * The returned array will have the original tokens as key. The values
 685       * in the returned list is an array with the page names as keys and the
 686       * number of times that token appears on the page as value.
 687       *
 688       * @param array  $tokens list of words to search for
 689       * @return array         list of page names with usage counts
 690       *
 691       * @author Tom N Harris <tnharris@whoopdedo.org>
 692       * @author Andreas Gohr <andi@splitbrain.org>
 693       */
 694      public function lookup(&$tokens) {
 695          $result = array();
 696          $wids = $this->getIndexWords($tokens, $result);
 697          if (empty($wids)) return array();
 698          // load known words and documents
 699          $page_idx = $this->getIndex('page', '');
 700          $docs = array();
 701          foreach (array_keys($wids) as $wlen) {
 702              $wids[$wlen] = array_unique($wids[$wlen]);
 703              $index = $this->getIndex('i', $wlen);
 704              foreach($wids[$wlen] as $ixid) {
 705                  if ($ixid < count($index))
 706                      $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]);
 707              }
 708          }
 709          // merge found pages into final result array
 710          $final = array();
 711          foreach ($result as $word => $res) {
 712              $final[$word] = array();
 713              foreach ($res as $wid) {
 714                  // handle the case when ($ixid < count($index)) has been false
 715                  // and thus $docs[$wid] hasn't been set.
 716                  if (!isset($docs[$wid])) continue;
 717                  $hits = &$docs[$wid];
 718                  foreach ($hits as $hitkey => $hitcnt) {
 719                      // make sure the document still exists
 720                      if (!page_exists($hitkey, '', false)) continue;
 721                      if (!isset($final[$word][$hitkey]))
 722                          $final[$word][$hitkey] = $hitcnt;
 723                      else
 724                          $final[$word][$hitkey] += $hitcnt;
 725                  }
 726              }
 727          }
 728          return $final;
 729      }
 730  
 731      /**
 732       * Find pages containing a metadata key.
 733       *
 734       * The metadata values are compared as case-sensitive strings. Pass a
 735       * callback function that returns true or false to use a different
 736       * comparison function. The function will be called with the $value being
 737       * searched for as the first argument, and the word in the index as the
 738       * second argument. The function preg_match can be used directly if the
 739       * values are regexes.
 740       *
 741       * @param string    $key    name of the metadata key to look for
 742       * @param string    $value  search term to look for, must be a string or array of strings
 743       * @param callback  $func   comparison function
 744       * @return array            lists with page names, keys are query values if $value is array
 745       *
 746       * @author Tom N Harris <tnharris@whoopdedo.org>
 747       * @author Michael Hamann <michael@content-space.de>
 748       */
 749      public function lookupKey($key, &$value, $func=null) {
 750          if (!is_array($value))
 751              $value_array = array($value);
 752          else
 753              $value_array =& $value;
 754  
 755          // the matching ids for the provided value(s)
 756          $value_ids = array();
 757  
 758          $metaname = idx_cleanName($key);
 759  
 760          // get all words in order to search the matching ids
 761          if ($key == 'title') {
 762              $words = $this->getIndex('title', '');
 763          } else {
 764              $words = $this->getIndex($metaname.'_w', '');
 765          }
 766  
 767          if (!is_null($func)) {
 768              foreach ($value_array as $val) {
 769                  foreach ($words as $i => $word) {
 770                      if (call_user_func_array($func, array($val, $word)))
 771                          $value_ids[$i][] = $val;
 772                  }
 773              }
 774          } else {
 775              foreach ($value_array as $val) {
 776                  $xval = $val;
 777                  $caret = '^';
 778                  $dollar = '$';
 779                  // check for wildcards
 780                  if (substr($xval, 0, 1) == '*') {
 781                      $xval = substr($xval, 1);
 782                      $caret = '';
 783                  }
 784                  if (substr($xval, -1, 1) == '*') {
 785                      $xval = substr($xval, 0, -1);
 786                      $dollar = '';
 787                  }
 788                  if (!$caret || !$dollar) {
 789                      $re = $caret.preg_quote($xval, '/').$dollar;
 790                      foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i)
 791                          $value_ids[$i][] = $val;
 792                  } else {
 793                      if (($i = array_search($val, $words, true)) !== false)
 794                          $value_ids[$i][] = $val;
 795                  }
 796              }
 797          }
 798  
 799          unset($words); // free the used memory
 800  
 801          // initialize the result so it won't be null
 802          $result = array();
 803          foreach ($value_array as $val) {
 804              $result[$val] = array();
 805          }
 806  
 807          $page_idx = $this->getIndex('page', '');
 808  
 809          // Special handling for titles
 810          if ($key == 'title') {
 811              foreach ($value_ids as $pid => $val_list) {
 812                  $page = $page_idx[$pid];
 813                  foreach ($val_list as $val) {
 814                      $result[$val][] = $page;
 815                  }
 816              }
 817          } else {
 818              // load all lines and pages so the used lines can be taken and matched with the pages
 819              $lines = $this->getIndex($metaname.'_i', '');
 820  
 821              foreach ($value_ids as $value_id => $val_list) {
 822                  // parse the tuples of the form page_id*1:page2_id*1 and so on, return value
 823                  // is an array with page_id => 1, page2_id => 1 etc. so take the keys only
 824                  $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id]));
 825                  foreach ($val_list as $val) {
 826                      $result[$val] = array_merge($result[$val], $pages);
 827                  }
 828              }
 829          }
 830          if (!is_array($value)) $result = $result[$value];
 831          return $result;
 832      }
 833  
 834      /**
 835       * Find the index ID of each search term.
 836       *
 837       * The query terms should only contain valid characters, with a '*' at
 838       * either the beginning or end of the word (or both).
 839       * The $result parameter can be used to merge the index locations with
 840       * the appropriate query term.
 841       *
 842       * @param array  $words  The query terms.
 843       * @param array  $result Set to word => array("length*id" ...)
 844       * @return array         Set to length => array(id ...)
 845       *
 846       * @author Tom N Harris <tnharris@whoopdedo.org>
 847       */
 848      protected function getIndexWords(&$words, &$result) {
 849          $tokens = array();
 850          $tokenlength = array();
 851          $tokenwild = array();
 852          foreach ($words as $word) {
 853              $result[$word] = array();
 854              $caret = '^';
 855              $dollar = '$';
 856              $xword = $word;
 857              $wlen = wordlen($word);
 858  
 859              // check for wildcards
 860              if (substr($xword, 0, 1) == '*') {
 861                  $xword = substr($xword, 1);
 862                  $caret = '';
 863                  $wlen -= 1;
 864              }
 865              if (substr($xword, -1, 1) == '*') {
 866                  $xword = substr($xword, 0, -1);
 867                  $dollar = '';
 868                  $wlen -= 1;
 869              }
 870              if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword))
 871                  continue;
 872              if (!isset($tokens[$xword]))
 873                  $tokenlength[$wlen][] = $xword;
 874              if (!$caret || !$dollar) {
 875                  $re = $caret.preg_quote($xword, '/').$dollar;
 876                  $tokens[$xword][] = array($word, '/'.$re.'/');
 877                  if (!isset($tokenwild[$xword]))
 878                      $tokenwild[$xword] = $wlen;
 879              } else {
 880                  $tokens[$xword][] = array($word, null);
 881              }
 882          }
 883          asort($tokenwild);
 884          // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
 885          // $tokenlength = array( base word length => base word ... )
 886          // $tokenwild = array( base word => base word length ... )
 887          $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
 888          $indexes_known = $this->indexLengths($length_filter);
 889          if (!empty($tokenwild)) sort($indexes_known);
 890          // get word IDs
 891          $wids = array();
 892          foreach ($indexes_known as $ixlen) {
 893              $word_idx = $this->getIndex('w', $ixlen);
 894              // handle exact search
 895              if (isset($tokenlength[$ixlen])) {
 896                  foreach ($tokenlength[$ixlen] as $xword) {
 897                      $wid = array_search($xword, $word_idx, true);
 898                      if ($wid !== false) {
 899                          $wids[$ixlen][] = $wid;
 900                          foreach ($tokens[$xword] as $w)
 901                              $result[$w[0]][] = "$ixlen*$wid";
 902                      }
 903                  }
 904              }
 905              // handle wildcard search
 906              foreach ($tokenwild as $xword => $wlen) {
 907                  if ($wlen >= $ixlen) break;
 908                  foreach ($tokens[$xword] as $w) {
 909                      if (is_null($w[1])) continue;
 910                      foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) {
 911                          $wids[$ixlen][] = $wid;
 912                          $result[$w[0]][] = "$ixlen*$wid";
 913                      }
 914                  }
 915              }
 916          }
 917          return $wids;
 918      }
 919  
 920      /**
 921       * Return a list of all pages
 922       * Warning: pages may not exist!
 923       *
 924       * @param string    $key    list only pages containing the metadata key (optional)
 925       * @return array            list of page names
 926       *
 927       * @author Tom N Harris <tnharris@whoopdedo.org>
 928       */
 929      public function getPages($key=null) {
 930          $page_idx = $this->getIndex('page', '');
 931          if (is_null($key)) return $page_idx;
 932  
 933          $metaname = idx_cleanName($key);
 934  
 935          // Special handling for titles
 936          if ($key == 'title') {
 937              $title_idx = $this->getIndex('title', '');
 938              array_splice($page_idx, count($title_idx));
 939              foreach ($title_idx as $i => $title)
 940                  if ($title === "") unset($page_idx[$i]);
 941              return array_values($page_idx);
 942          }
 943  
 944          $pages = array();
 945          $lines = $this->getIndex($metaname.'_i', '');
 946          foreach ($lines as $line) {
 947              $pages = array_merge($pages, $this->parseTuples($page_idx, $line));
 948          }
 949          return array_keys($pages);
 950      }
 951  
 952      /**
 953       * Return a list of words sorted by number of times used
 954       *
 955       * @param int       $min    bottom frequency threshold
 956       * @param int       $max    upper frequency limit. No limit if $max<$min
 957       * @param int       $minlen minimum length of words to count
 958       * @param string    $key    metadata key to list. Uses the fulltext index if not given
 959       * @return array            list of words as the keys and frequency as values
 960       *
 961       * @author Tom N Harris <tnharris@whoopdedo.org>
 962       */
 963      public function histogram($min=1, $max=0, $minlen=3, $key=null) {
 964          if ($min < 1)
 965              $min = 1;
 966          if ($max < $min)
 967              $max = 0;
 968  
 969          $result = array();
 970  
 971          if ($key == 'title') {
 972              $index = $this->getIndex('title', '');
 973              $index = array_count_values($index);
 974              foreach ($index as $val => $cnt) {
 975                  if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen)
 976                      $result[$val] = $cnt;
 977              }
 978          }
 979          elseif (!is_null($key)) {
 980              $metaname = idx_cleanName($key);
 981              $index = $this->getIndex($metaname.'_i', '');
 982              $val_idx = array();
 983              foreach ($index as $wid => $line) {
 984                  $freq = $this->countTuples($line);
 985                  if ($freq >= $min && (!$max || $freq <= $max))
 986                      $val_idx[$wid] = $freq;
 987              }
 988              if (!empty($val_idx)) {
 989                  $words = $this->getIndex($metaname.'_w', '');
 990                  foreach ($val_idx as $wid => $freq) {
 991                      if (strlen($words[$wid]) >= $minlen)
 992                          $result[$words[$wid]] = $freq;
 993                  }
 994              }
 995          }
 996          else {
 997              $lengths = idx_listIndexLengths();
 998              foreach ($lengths as $length) {
 999                  if ($length < $minlen) continue;
1000                  $index = $this->getIndex('i', $length);
1001                  $words = null;
1002                  foreach ($index as $wid => $line) {
1003                      $freq = $this->countTuples($line);
1004                      if ($freq >= $min && (!$max || $freq <= $max)) {
1005                          if ($words === null)
1006                              $words = $this->getIndex('w', $length);
1007                          $result[$words[$wid]] = $freq;
1008                      }
1009                  }
1010              }
1011          }
1012  
1013          arsort($result);
1014          return $result;
1015      }
1016  
1017      /**
1018       * Lock the indexer.
1019       *
1020       * @author Tom N Harris <tnharris@whoopdedo.org>
1021       *
1022       * @return bool|string
1023       */
1024      protected function lock() {
1025          global $conf;
1026          $status = true;
1027          $run = 0;
1028          $lock = $conf['lockdir'].'/_indexer.lock';
1029          while (!@mkdir($lock, $conf['dmode'])) {
1030              usleep(50);
1031              if(is_dir($lock) && time()-@filemtime($lock) > 60*5){
1032                  // looks like a stale lock - remove it
1033                  if (!@rmdir($lock)) {
1034                      $status = "removing the stale lock failed";
1035                      return false;
1036                  } else {
1037                      $status = "stale lock removed";
1038                  }
1039              }elseif($run++ == 1000){
1040                  // we waited 5 seconds for that lock
1041                  return false;
1042              }
1043          }
1044          if (!empty($conf['dperm'])) {
1045              chmod($lock, $conf['dperm']);
1046          }
1047          return $status;
1048      }
1049  
1050      /**
1051       * Release the indexer lock.
1052       *
1053       * @author Tom N Harris <tnharris@whoopdedo.org>
1054       *
1055       * @return bool
1056       */
1057      protected function unlock() {
1058          global $conf;
1059          @rmdir($conf['lockdir'].'/_indexer.lock');
1060          return true;
1061      }
1062  
1063      /**
1064       * Retrieve the entire index.
1065       *
1066       * The $suffix argument is for an index that is split into
1067       * multiple parts. Different index files should use different
1068       * base names.
1069       *
1070       * @param string    $idx    name of the index
1071       * @param string    $suffix subpart identifier
1072       * @return array            list of lines without CR or LF
1073       *
1074       * @author Tom N Harris <tnharris@whoopdedo.org>
1075       */
1076      protected function getIndex($idx, $suffix) {
1077          global $conf;
1078          $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
1079          if (!file_exists($fn)) return array();
1080          return file($fn, FILE_IGNORE_NEW_LINES);
1081      }
1082  
1083      /**
1084       * Replace the contents of the index with an array.
1085       *
1086       * @param string    $idx    name of the index
1087       * @param string    $suffix subpart identifier
1088       * @param array     $lines  list of lines without LF
1089       * @return bool             If saving succeeded
1090       *
1091       * @author Tom N Harris <tnharris@whoopdedo.org>
1092       */
1093      protected function saveIndex($idx, $suffix, &$lines) {
1094          global $conf;
1095          $fn = $conf['indexdir'].'/'.$idx.$suffix;
1096          $fh = @fopen($fn.'.tmp', 'w');
1097          if (!$fh) return false;
1098          fwrite($fh, join("\n", $lines));
1099          if (!empty($lines))
1100              fwrite($fh, "\n");
1101          fclose($fh);
1102          if (isset($conf['fperm']))
1103              chmod($fn.'.tmp', $conf['fperm']);
1104          io_rename($fn.'.tmp', $fn.'.idx');
1105          return true;
1106      }
1107  
1108      /**
1109       * Retrieve a line from the index.
1110       *
1111       * @param string    $idx    name of the index
1112       * @param string    $suffix subpart identifier
1113       * @param int       $id     the line number
1114       * @return string           a line with trailing whitespace removed
1115       *
1116       * @author Tom N Harris <tnharris@whoopdedo.org>
1117       */
1118      protected function getIndexKey($idx, $suffix, $id) {
1119          global $conf;
1120          $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
1121          if (!file_exists($fn)) return '';
1122          $fh = @fopen($fn, 'r');
1123          if (!$fh) return '';
1124          $ln = -1;
1125          while (($line = fgets($fh)) !== false) {
1126              if (++$ln == $id) break;
1127          }
1128          fclose($fh);
1129          return rtrim((string)$line);
1130      }
1131  
1132      /**
1133       * Write a line into the index.
1134       *
1135       * @param string    $idx    name of the index
1136       * @param string    $suffix subpart identifier
1137       * @param int       $id     the line number
1138       * @param string    $line   line to write
1139       * @return bool             If saving succeeded
1140       *
1141       * @author Tom N Harris <tnharris@whoopdedo.org>
1142       */
1143      protected function saveIndexKey($idx, $suffix, $id, $line) {
1144          global $conf;
1145          if (substr($line, -1) != "\n")
1146              $line .= "\n";
1147          $fn = $conf['indexdir'].'/'.$idx.$suffix;
1148          $fh = @fopen($fn.'.tmp', 'w');
1149          if (!$fh) return false;
1150          $ih = @fopen($fn.'.idx', 'r');
1151          if ($ih) {
1152              $ln = -1;
1153              while (($curline = fgets($ih)) !== false) {
1154                  fwrite($fh, (++$ln == $id) ? $line : $curline);
1155              }
1156              if ($id > $ln) {
1157                  while ($id > ++$ln)
1158                      fwrite($fh, "\n");
1159                  fwrite($fh, $line);
1160              }
1161              fclose($ih);
1162          } else {
1163              $ln = -1;
1164              while ($id > ++$ln)
1165                  fwrite($fh, "\n");
1166              fwrite($fh, $line);
1167          }
1168          fclose($fh);
1169          if (isset($conf['fperm']))
1170              chmod($fn.'.tmp', $conf['fperm']);
1171          io_rename($fn.'.tmp', $fn.'.idx');
1172          return true;
1173      }
1174  
1175      /**
1176       * Retrieve or insert a value in the index.
1177       *
1178       * @param string    $idx    name of the index
1179       * @param string    $suffix subpart identifier
1180       * @param string    $value  line to find in the index
1181       * @return int|bool          line number of the value in the index or false if writing the index failed
1182       *
1183       * @author Tom N Harris <tnharris@whoopdedo.org>
1184       */
1185      protected function addIndexKey($idx, $suffix, $value) {
1186          $index = $this->getIndex($idx, $suffix);
1187          $id = array_search($value, $index, true);
1188          if ($id === false) {
1189              $id = count($index);
1190              $index[$id] = $value;
1191              if (!$this->saveIndex($idx, $suffix, $index)) {
1192                  trigger_error("Failed to write $idx index", E_USER_ERROR);
1193                  return false;
1194              }
1195          }
1196          return $id;
1197      }
1198  
1199      /**
1200       * Get the list of lengths indexed in the wiki.
1201       *
1202       * Read the index directory or a cache file and returns
1203       * a sorted array of lengths of the words used in the wiki.
1204       *
1205       * @author YoBoY <yoboy.leguesh@gmail.com>
1206       *
1207       * @return array
1208       */
1209      protected function listIndexLengths() {
1210          return idx_listIndexLengths();
1211      }
1212  
1213      /**
1214       * Get the word lengths that have been indexed.
1215       *
1216       * Reads the index directory and returns an array of lengths
1217       * that there are indices for.
1218       *
1219       * @author YoBoY <yoboy.leguesh@gmail.com>
1220       *
1221       * @param array|int $filter
1222       * @return array
1223       */
1224      protected function indexLengths($filter) {
1225          global $conf;
1226          $idx = array();
1227          if (is_array($filter)) {
1228              // testing if index files exist only
1229              $path = $conf['indexdir']."/i";
1230              foreach ($filter as $key => $value) {
1231                  if (file_exists($path.$key.'.idx'))
1232                      $idx[] = $key;
1233              }
1234          } else {
1235              $lengths = idx_listIndexLengths();
1236              foreach ($lengths as $key => $length) {
1237                  // keep all the values equal or superior
1238                  if ((int)$length >= (int)$filter)
1239                      $idx[] = $length;
1240              }
1241          }
1242          return $idx;
1243      }
1244  
1245      /**
1246       * Insert or replace a tuple in a line.
1247       *
1248       * @author Tom N Harris <tnharris@whoopdedo.org>
1249       *
1250       * @param string $line
1251       * @param string|int $id
1252       * @param int    $count
1253       * @return string
1254       */
1255      protected function updateTuple($line, $id, $count) {
1256          if ($line != ''){
1257              $line = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $line);
1258          }
1259          $line = trim($line, ':');
1260          if ($count) {
1261              if ($line) {
1262                  return "$id*$count:".$line;
1263              } else {
1264                  return "$id*$count";
1265              }
1266          }
1267          return $line;
1268      }
1269  
1270      /**
1271       * Split a line into an array of tuples.
1272       *
1273       * @author Tom N Harris <tnharris@whoopdedo.org>
1274       * @author Andreas Gohr <andi@splitbrain.org>
1275       *
1276       * @param array $keys
1277       * @param string $line
1278       * @return array
1279       */
1280      protected function parseTuples(&$keys, $line) {
1281          $result = array();
1282          if ($line == '') return $result;
1283          $parts = explode(':', $line);
1284          foreach ($parts as $tuple) {
1285              if ($tuple === '') continue;
1286              list($key, $cnt) = explode('*', $tuple);
1287              if (!$cnt) continue;
1288              $key = $keys[$key];
1289              if ($key === false || is_null($key)) continue;
1290              $result[$key] = $cnt;
1291          }
1292          return $result;
1293      }
1294  
1295      /**
1296       * Sum the counts in a list of tuples.
1297       *
1298       * @author Tom N Harris <tnharris@whoopdedo.org>
1299       *
1300       * @param string $line
1301       * @return int
1302       */
1303      protected function countTuples($line) {
1304          $freq = 0;
1305          $parts = explode(':', $line);
1306          foreach ($parts as $tuple) {
1307              if ($tuple === '') continue;
1308              list(/* $pid */, $cnt) = explode('*', $tuple);
1309              $freq += (int)$cnt;
1310          }
1311          return $freq;
1312      }
1313  }
1314  
1315  /**
1316   * Create an instance of the indexer.
1317   *
1318   * @return Doku_Indexer    a Doku_Indexer
1319   *
1320   * @author Tom N Harris <tnharris@whoopdedo.org>
1321   */
1322  function idx_get_indexer() {
1323      static $Indexer;
1324      if (!isset($Indexer)) {
1325          $Indexer = new Doku_Indexer();
1326      }
1327      return $Indexer;
1328  }
1329  
1330  /**
1331   * Returns words that will be ignored.
1332   *
1333   * @return array                list of stop words
1334   *
1335   * @author Tom N Harris <tnharris@whoopdedo.org>
1336   */
1337  function & idx_get_stopwords() {
1338      static $stopwords = null;
1339      if (is_null($stopwords)) {
1340          global $conf;
1341          $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
1342          if(file_exists($swfile)){
1343              $stopwords = file($swfile, FILE_IGNORE_NEW_LINES);
1344          }else{
1345              $stopwords = array();
1346          }
1347      }
1348      return $stopwords;
1349  }
1350  
1351  /**
1352   * Adds/updates the search index for the given page
1353   *
1354   * Locking is handled internally.
1355   *
1356   * @param string        $page   name of the page to index
1357   * @param boolean       $verbose    print status messages
1358   * @param boolean       $force  force reindexing even when the index is up to date
1359   * @return string|boolean  the function completed successfully
1360   *
1361   * @author Tom N Harris <tnharris@whoopdedo.org>
1362   */
1363  function idx_addPage($page, $verbose=false, $force=false) {
1364      $idxtag = metaFN($page,'.indexed');
1365      // check if page was deleted but is still in the index
1366      if (!page_exists($page)) {
1367          if (!file_exists($idxtag)) {
1368              if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF);
1369              return false;
1370          }
1371          $Indexer = idx_get_indexer();
1372          $result = $Indexer->deletePage($page);
1373          if ($result === "locked") {
1374              if ($verbose) print("Indexer: locked".DOKU_LF);
1375              return false;
1376          }
1377          @unlink($idxtag);
1378          return $result;
1379      }
1380  
1381      // check if indexing needed
1382      if(!$force && file_exists($idxtag)){
1383          if(trim(io_readFile($idxtag)) == idx_get_version()){
1384              $last = @filemtime($idxtag);
1385              if($last > @filemtime(wikiFN($page))){
1386                  if ($verbose) print("Indexer: index for $page up to date".DOKU_LF);
1387                  return false;
1388              }
1389          }
1390      }
1391  
1392      $indexenabled = p_get_metadata($page, 'internal index', METADATA_RENDER_UNLIMITED);
1393      if ($indexenabled === false) {
1394          $result = false;
1395          if (file_exists($idxtag)) {
1396              $Indexer = idx_get_indexer();
1397              $result = $Indexer->deletePage($page);
1398              if ($result === "locked") {
1399                  if ($verbose) print("Indexer: locked".DOKU_LF);
1400                  return false;
1401              }
1402              @unlink($idxtag);
1403          }
1404          if ($verbose) print("Indexer: index disabled for $page".DOKU_LF);
1405          return $result;
1406      }
1407  
1408      $Indexer = idx_get_indexer();
1409      $pid = $Indexer->getPID($page);
1410      if ($pid === false) {
1411          if ($verbose) print("Indexer: getting the PID failed for $page".DOKU_LF);
1412          return false;
1413      }
1414      $body = '';
1415      $metadata = array();
1416      $metadata['title'] = p_get_metadata($page, 'title', METADATA_RENDER_UNLIMITED);
1417      if (($references = p_get_metadata($page, 'relation references', METADATA_RENDER_UNLIMITED)) !== null)
1418          $metadata['relation_references'] = array_keys($references);
1419      else
1420          $metadata['relation_references'] = array();
1421  
1422      if (($media = p_get_metadata($page, 'relation media', METADATA_RENDER_UNLIMITED)) !== null)
1423          $metadata['relation_media'] = array_keys($media);
1424      else
1425          $metadata['relation_media'] = array();
1426  
1427      $data = compact('page', 'body', 'metadata', 'pid');
1428      $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
1429      if ($evt->advise_before()) $data['body'] = $data['body'] . " " . rawWiki($page);
1430      $evt->advise_after();
1431      unset($evt);
1432      extract($data);
1433  
1434      $result = $Indexer->addPageWords($page, $body);
1435      if ($result === "locked") {
1436          if ($verbose) print("Indexer: locked".DOKU_LF);
1437          return false;
1438      }
1439  
1440      if ($result) {
1441          $result = $Indexer->addMetaKeys($page, $metadata);
1442          if ($result === "locked") {
1443              if ($verbose) print("Indexer: locked".DOKU_LF);
1444              return false;
1445          }
1446      }
1447  
1448      if ($result)
1449          io_saveFile(metaFN($page,'.indexed'), idx_get_version());
1450      if ($verbose) {
1451          print("Indexer: finished".DOKU_LF);
1452          return true;
1453      }
1454      return $result;
1455  }
1456  
1457  /**
1458   * Find tokens in the fulltext index
1459   *
1460   * Takes an array of words and will return a list of matching
1461   * pages for each one.
1462   *
1463   * Important: No ACL checking is done here! All results are
1464   *            returned, regardless of permissions
1465   *
1466   * @param array      $words  list of words to search for
1467   * @return array             list of pages found, associated with the search terms
1468   */
1469  function idx_lookup(&$words) {
1470      $Indexer = idx_get_indexer();
1471      return $Indexer->lookup($words);
1472  }
1473  
1474  /**
1475   * Split a string into tokens
1476   *
1477   * @param string $string
1478   * @param bool $wc
1479   *
1480   * @return array
1481   */
1482  function idx_tokenizer($string, $wc=false) {
1483      $Indexer = idx_get_indexer();
1484      return $Indexer->tokenizer($string, $wc);
1485  }
1486  
1487  /* For compatibility */
1488  
1489  /**
1490   * Read the list of words in an index (if it exists).
1491   *
1492   * @author Tom N Harris <tnharris@whoopdedo.org>
1493   *
1494   * @param string $idx
1495   * @param string $suffix
1496   * @return array
1497   */
1498  function idx_getIndex($idx, $suffix) {
1499      global $conf;
1500      $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
1501      if (!file_exists($fn)) return array();
1502      return file($fn);
1503  }
1504  
1505  /**
1506   * Get the list of lengths indexed in the wiki.
1507   *
1508   * Read the index directory or a cache file and returns
1509   * a sorted array of lengths of the words used in the wiki.
1510   *
1511   * @author YoBoY <yoboy.leguesh@gmail.com>
1512   *
1513   * @return array
1514   */
1515  function idx_listIndexLengths() {
1516      global $conf;
1517      // testing what we have to do, create a cache file or not.
1518      if ($conf['readdircache'] == 0) {
1519          $docache = false;
1520      } else {
1521          clearstatcache();
1522          if (file_exists($conf['indexdir'].'/lengths.idx')
1523          && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) {
1524              if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) {
1525                  $idx = array();
1526                  foreach ($lengths as $length) {
1527                      $idx[] = (int)$length;
1528                  }
1529                  return $idx;
1530              }
1531          }
1532          $docache = true;
1533      }
1534  
1535      if ($conf['readdircache'] == 0 || $docache) {
1536          $dir = @opendir($conf['indexdir']);
1537          if ($dir === false)
1538              return array();
1539          $idx = array();
1540          while (($f = readdir($dir)) !== false) {
1541              if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') {
1542                  $i = substr($f, 1, -4);
1543                  if (is_numeric($i))
1544                      $idx[] = (int)$i;
1545              }
1546          }
1547          closedir($dir);
1548          sort($idx);
1549          // save this in a file
1550          if ($docache) {
1551              $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w');
1552              @fwrite($handle, implode("\n", $idx));
1553              @fclose($handle);
1554          }
1555          return $idx;
1556      }
1557  
1558      return array();
1559  }
1560  
1561  /**
1562   * Get the word lengths that have been indexed.
1563   *
1564   * Reads the index directory and returns an array of lengths
1565   * that there are indices for.
1566   *
1567   * @author YoBoY <yoboy.leguesh@gmail.com>
1568   *
1569   * @param array|int $filter
1570   * @return array
1571   */
1572  function idx_indexLengths($filter) {
1573      global $conf;
1574      $idx = array();
1575      if (is_array($filter)) {
1576          // testing if index files exist only
1577          $path = $conf['indexdir']."/i";
1578          foreach ($filter as $key => $value) {
1579              if (file_exists($path.$key.'.idx'))
1580                  $idx[] = $key;
1581          }
1582      } else {
1583          $lengths = idx_listIndexLengths();
1584          foreach ($lengths as $key => $length) {
1585              // keep all the values equal or superior
1586              if ((int)$length >= (int)$filter)
1587                  $idx[] = $length;
1588          }
1589      }
1590      return $idx;
1591  }
1592  
1593  /**
1594   * Clean a name of a key for use as a file name.
1595   *
1596   * Romanizes non-latin characters, then strips away anything that's
1597   * not a letter, number, or underscore.
1598   *
1599   * @author Tom N Harris <tnharris@whoopdedo.org>
1600   *
1601   * @param string $name
1602   * @return string
1603   */
1604  function idx_cleanName($name) {
1605      $name = utf8_romanize(trim((string)$name));
1606      $name = preg_replace('#[ \./\\:-]+#', '_', $name);
1607      $name = preg_replace('/[^A-Za-z0-9_]/', '', $name);
1608      return strtolower($name);
1609  }
1610  
1611  //Setup VIM: ex: et ts=4 :