Changeset 58

Timestamp:

12/21/05 11:03:12 (3 years ago)

Author:

fabien

Message:

day 21 modifications

Files:

• trunk/batch/load_data.php (modified) (1 diff)
• trunk/config/schema.xml (modified) (1 diff)
• trunk/data/sql/schema.sql (modified) (2 diffs)
• trunk/frontend/config/app.yml (modified) (1 diff)
• trunk/frontend/config/routing.yml (modified) (1 diff)
• trunk/frontend/modules/question/actions/actions.class.php (modified) (1 diff)
• trunk/frontend/modules/question/templates/_question_block.php (added)
• trunk/frontend/modules/question/templates/_question_list.php (modified) (1 diff)
• trunk/frontend/modules/question/templates/_search.php (added)
• trunk/frontend/modules/question/templates/searchSuccess.php (added)
• trunk/frontend/modules/sidebar/templates/defaultSuccess.php (modified) (1 diff)
• trunk/frontend/modules/sidebar/templates/questionSuccess.php (modified) (1 diff)
• trunk/lib/PorterStemmer.class.php (added)
• trunk/lib/model/Question.php (modified) (1 diff)
• trunk/lib/model/QuestionPeer.php (modified) (1 diff)
• trunk/lib/model/QuestionTag.php (modified) (1 diff)
• trunk/lib/model/SearchIndex.php (added)
• trunk/lib/model/SearchIndexPeer.php (added)
• trunk/lib/model/map/SearchIndexMapBuilder.php (added)
• trunk/lib/model/om/BaseQuestion.php (modified) (5 diffs)
• trunk/lib/model/om/BaseSearchIndex.php (added)
• trunk/lib/model/om/BaseSearchIndexPeer.php (added)
• trunk/lib/myTools.class.php (modified) (1 diff)

trunk/batch/load_data.php

@@ -8 +8 @@
-require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
-
+SearchIndexPeer::doDeleteAll();
+
-$data = new sfPropelData();
-$data->loadData(SF_DATA_DIR.DIRECTORY_SEPARATOR.'fixtures');

trunk/config/schema.xml

@@ -93 +93 @@
-   </table>
-
+   <table name="ask_search_index" phpName="SearchIndex">
+     <column name="question_id" type="integer" />
+     <foreign-key foreignTable="ask_question" onDelete="cascade">
+       <reference local="question_id" foreign="id" />
+     </foreign-key>
+     <column name="word" type="varchar" size="255" />
+     <column name="weight" type="integer" />
+   </table>
+   
-   <table name="ask_report_question" phpName="ReportQuestion">
-     <column name="question_id" type="integer" primaryKey="true" />

trunk/data/sql/schema.sql

@@ -139 +139 @@
-Type=InnoDB;
-# -----------------------------------------------------------------------
+# ask_search_index 
+# -----------------------------------------------------------------------
+DROP TABLE IF EXISTS `ask_search_index`;
+
+CREATE TABLE `ask_search_index`(
+    `question_id` INTEGER , 
+    `word` VARCHAR(255) , 
+    `weight` INTEGER ,  
+    INDEX `ask_search_index_FI_1` (`question_id`), 
+    CONSTRAINT `ask_search_index_FK_1` 
+      FOREIGN KEY (`question_id`)
+      REFERENCES `ask_question` (`id`)
+      ON DELETE CASCADE)
+Type=InnoDB;
+# -----------------------------------------------------------------------
-# ask_report_question 
-# -----------------------------------------------------------------------
@@ -186 +201 @@
-  
-  
+  
-# This restores the fkey checks, after having unset them
-# in database-start.tpl

trunk/frontend/config/app.yml

@@ -9 +9 @@
-    max_questions:       10
-
+  search:
+    body_weight:         1
+    title_weight:        2
+    tag_weight:          3
+    results_max:         10
+
-  .global:
-    universe:            on

trunk/frontend/config/routing.yml

@@ -15 +15 @@
-  url:   /add_question
-  param: { module: question, action: add }
+
+search_question:
+  url:   /search/*
+  param: { module: question, action: search }
-
-# answers

trunk/frontend/modules/question/actions/actions.class.php

@@ -54 +54 @@
-  }
-
+  public function executeSearch ()
+  {
+    if ($this->getRequestParameter('search'))
+    {
+      $this->questions = QuestionPeer::search($this->getRequestParameter('search'), $this->getRequestParameter('search_all', false), ($this->getRequestParameter('page', 1) - 1) * APP_SEARCH_RESULTS_MAX, APP_SEARCH_RESULTS_MAX);
+    }
+    else
+    {
+      $this->redirect('@homepage');
+    }
+  }
+
-  public function handleErrorAdd()
-  {

trunk/frontend/modules/question/templates/_question_list.php

@@ -1 @@
-Text', 'Global', 'Question', 'Date
+Global
-
-<?php foreach($question_pager->getResults() as $question): ?>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-<?php endforeach ?>
-

trunk/frontend/modules/sidebar/templates/defaultSuccess.php

@@ -9 +9 @@
-<div class="right" style="padding-top: 5px"><?php echo link_to('more popular tags &raquo;', '@popular_tags') ?></div>
-
+<h2>find it</h2>
+<?php echo include_partial('question/search') ?>
+
-<h2>browse askeet</h2>
-<?php echo include_partial('rss_links') ?>

trunk/frontend/modules/sidebar/templates/questionSuccess.php

@@ -4 +4 @@
-  <?php echo link_to_login('ask a new question', '@add_question') ?>
-</div>
+
+<h2>find it</h2>
+<?php echo include_partial('question/search') ?>
-
-<h2>browse askeet</h2>

trunk/lib/model/Question.php

@@ -150 +150 @@
-  }
-
+  public function getWords()
+  {
+    // body
+    $raw_text =  str_repeat(' '.strip_tags($this->getHtmlBody()), APP_SEARCH_BODY_WEIGHT);
+
+    // title
+    $raw_text .= str_repeat(' '.$this->getTitle(), APP_SEARCH_TITLE_WEIGHT);
+
+    // title and body stemming
+    $stemmed_words = myTools::stemPhrase($raw_text);
+
+    // unique words with weight
+    $words = array_count_values($stemmed_words);
+
+    // add tags
+    $max = 0;
+    foreach ($this->getPopularTags(20) as $tag => $count)
+    {
+      if (!$max)
+      {
+        $max = $count;
+      }
+
+      $stemmed_tag = PorterStemmer::stem($tag);
+
+      if (!isset($words[$stemmed_tag]))
+      {
+        $words[$stemmed_tag] = 0;
+      }
+      $words[$stemmed_tag] += ceil(($count / $max) * APP_SEARCH_TAG_WEIGHT);
+    }
+
+    return $words;
+  }
+
+  public function save($con = null)
+  {
+    $con = Propel::getConnection();
+    try
+    {
+      $con->begin();
+
+      $ret = parent::save();
+
+      $this->updateSearchIndex();
+
+      $con->commit();
+
+      return $ret;
+    }
+    catch (Exception $e)
+    {
+      $con->rollback();
+      throw $e;
+    }
+  }
+
+  public function updateSearchIndex()
+  {
+    // update search index
+    $c = new Criteria();
+    $c->add(SearchIndexPeer::QUESTION_ID, $this->getId());
+    SearchIndexPeer::doDelete($c);
+
+    foreach ($this->getWords() as $word => $weight)
+    {
+      $index = new SearchIndex();
+      $index->setQuestionId($this->getId());
+      $index->setWord($word);
+      $index->setWeight($weight);
+      $index->save();
+    }
+  }
+
+  public function hasTag($tag)
+  {
+    $c = new Criteria();
+    $c->add(QuestionTagPeer::QUESTION_ID, $this->getId());
+    $c->add(QuestionTagPeer::NORMALIZED_TAG, Tag::normalize($tag));
+
+    return QuestionTagPeer::doSelectOne($c) ? true : false;
+  }
+
-  public function deleteReports()
-  {

trunk/lib/model/QuestionPeer.php

@@ -103 +103 @@
-  }
-
+  public static function search($phrase, $exact = false, $offset = 0, $max = 10)
+  {
+    $words    = array_values(myTools::stemPhrase($phrase));
+    $nb_words = count($words);
+
+    $con = Propel::getConnection();
+    $query = '
+      SELECT DISTINCT %s, COUNT(*) AS nb, SUM(%s) AS total_weight
+      FROM %s
+    ';
+
+    if (defined('APP_PERMANENT_TAG'))
+    {
+      $query .= sprintf('
+        LEFT JOIN %s ON %s = %s
+        WHERE %s = ? AND ',
+        QuestionTagPeer::TABLE_NAME,
+        QuestionTagPeer::QUESTION_ID,
+        SearchIndexPeer::QUESTION_ID,
+        QuestionTagPeer::NORMALIZED_TAG
+      );
+    }
+    else
+    {
+      $query .= 'WHERE';
+    }
+
+    $query .= '
+      ('.implode(' OR ', array_fill(0, $nb_words, SearchIndexPeer::WORD.' = ?')).')
+      GROUP BY %s
+    ';
+
+    // AND query?
+    if ($exact)
+    {
+      $query .= ' HAVING nb = '.$nb_words;
+    }
+
+    $query .= ' ORDER BY nb DESC, total_weight DESC';
+
+    $query = sprintf($query,
+      SearchIndexPeer::QUESTION_ID,
+      SearchIndexPeer::WEIGHT,
+      SearchIndexPeer::TABLE_NAME,
+      SearchIndexPeer::QUESTION_ID
+    );
+
+    $stmt = $con->prepareStatement($query);
+    $stmt->setOffset($offset);
+    $stmt->setLimit($max);
+    $placeholder_offset = 1;
+    if (defined('APP_PERMANENT_TAG'))
+    {
+      $stmt->setString(1, APP_PERMANENT_TAG);
+      $placeholder_offset = 2;
+    }
+    for ($i = 0; $i < $nb_words; $i++)
+    {
+      $stmt->setString($i + $placeholder_offset, $words[$i]);
+    }
+    $rs = $stmt->executeQuery();
+    $questions = array();
+    while ($rs->next())
+    {
+      $questions[] = self::retrieveByPK($rs->getInt('question_id'));
+    }
+
+    return $questions;
+  }
+
-  public static function getReportedSpamPager($page)
-  {

trunk/lib/model/QuestionTag.php

@@ -23 +23 @@
-    $this->setNormalizedTag(Tag::normalize($v));
-  }
+
+  public function save($con = null)
+  {
+    $ret = parent::save($con);
+
+    $this->getQuestion()->updateSearchIndex();
+
+    return $ret;
+  }
-}
-

trunk/lib/model/om/BaseQuestion.php

@@ -138 +138 @@
-     */
-    private $lastQuestionTagCriteria = null;
+
+    /**
+     * Collection to store aggregation of collSearchIndexs.
+     * @var array
+     */
+    protected $collSearchIndexs;
+    
+    /**
+     * The criteria used to select the current contents of collSearchIndexs.
+     * @var Criteria
+     */
+    private $lastSearchIndexCriteria = null;
-
-    /**
@@ -683 +695 @@
-            }
-
+            if ($this->collSearchIndexs !== null) {
+                foreach($this->collSearchIndexs as $referrerFK) {
+                    if (!$referrerFK->isDeleted()) {
+                        $affectedRows += $referrerFK->save($con);
+                    }
+                }
+            }
+
-            if ($this->collReportQuestions !== null) {
-                foreach($this->collReportQuestions as $referrerFK) {
@@ -769 +789 @@
-                if ($this->collQuestionTags !== null) {
-                    foreach($this->collQuestionTags as $referrerFK) {
+                        if (($retval = $referrerFK->validate()) !== true) {
+                            $failureMap = array_merge($failureMap, $retval);
+                        }
+                    }
+                }
+
+                if ($this->collSearchIndexs !== null) {
+                    foreach($this->collSearchIndexs as $referrerFK) {
-                        if (($retval = $referrerFK->validate()) !== true) {
-                            $failureMap = array_merge($failureMap, $retval);
@@ -1082 +1110 @@
-            }
-
+            foreach($this->getSearchIndexs() as $relObj) {
+                $copyObj->addSearchIndex($relObj->copy($deepCopy));
+            }
+
-            foreach($this->getReportQuestions() as $relObj) {
-                $copyObj->addReportQuestion($relObj->copy($deepCopy));
@@ -1650 +1682 @@
-
-    /**
+     * Temporary storage of collSearchIndexs to save a possible db hit in
+     * the event objects are add to the collection, but the
+     * complete collection is never requested.
+     * @return void
+     */
+    public function initSearchIndexs()
+    {
+        if ($this->collSearchIndexs === null) {
+            $this->collSearchIndexs = array();
+        }
+    }
+
+    /**
+     * If this collection has already been initialized with
+     * an identical criteria, it returns the collection.
+     * Otherwise if this Question has previously
+     * been saved, it will retrieve related SearchIndexs from storage.
+     * If this Question is new, it will return
+     * an empty collection or the current collection, the criteria
+     * is ignored on a new object.
+     *
+     * @param Connection $con
+     * @param Criteria $criteria
+     * @throws PropelException
+     */
+    public function getSearchIndexs($criteria = null, $con = null)
+    {
+        // include the Peer class
+        include_once 'model/om/BaseSearchIndexPeer.php';
+        if ($criteria === null) {
+            $criteria = new Criteria();
+        }
+        elseif ($criteria instanceof Criteria)
+        {
+            $criteria = clone $criteria;
+        }
+
+        if ($this->collSearchIndexs === null) {
+            if ($this->isNew()) {
+               $this->collSearchIndexs = array();
+            } else {
+
+                $criteria->add(SearchIndexPeer::QUESTION_ID, $this->getId());
+
+                SearchIndexPeer::addSelectColumns($criteria);
+                $this->collSearchIndexs = SearchIndexPeer::doSelect($criteria, $con);
+            }
+        } else {
+            // criteria has no effect for a new object
+            if (!$this->isNew()) {
+                // the following code is to determine if a new query is
+                // called for.  If the criteria is the same as the last
+                // one, just return the collection.
+
+
+                $criteria->add(SearchIndexPeer::QUESTION_ID, $this->getId());
+
+                SearchIndexPeer::addSelectColumns($criteria);
+                if (!isset($this->lastSearchIndexCriteria) || !$this->lastSearchIndexCriteria->equals($criteria)) {
+                    $this->collSearchIndexs = SearchIndexPeer::doSelect($criteria, $con);
+                }
+            }
+        }
+        $this->lastSearchIndexCriteria = $criteria;
+        return $this->collSearchIndexs;
+    }
+
+    /**
+     * Returns the number of related SearchIndexs.
+     *
+     * @param Criteria $criteria
+     * @param Connection $con
+     * @throws PropelException
+     */
+    public function countSearchIndexs($criteria = null, $con = null)
+    {
+        // include the Peer class
+        include_once 'model/om/BaseSearchIndexPeer.php';
+        if ($criteria === null) {
+            $criteria = new Criteria();
+        }
+        elseif ($criteria instanceof Criteria)
+        {
+            $criteria = clone $criteria;
+        }
+
+        $criteria->add(SearchIndexPeer::QUESTION_ID, $this->getId());
+
+        return SearchIndexPeer::doCount($criteria, $con);
+    }
+
+    /**
+     * Method called to associate a SearchIndex object to this object
+     * through the SearchIndex foreign key attribute
+     *
+     * @param SearchIndex $l SearchIndex
+     * @return void
+     * @throws PropelException
+     */
+    public function addSearchIndex(SearchIndex $l)
+    {
+        $this->collSearchIndexs[] = $l;
+        $l->setQuestion($this);
+    }
+
+    /**
-     * Temporary storage of collReportQuestions to save a possible db hit in
-     * the event objects are add to the collection, but the

trunk/lib/myTools.class.php

@@ -19 +19 @@
-    return $text;
-  }
+
+  public static function stemPhrase($phrase)
+  {
+    // split into words
+    $words = str_word_count(strtolower($phrase), 1);
+
+    // ignore stop words
+    $words = myTools::removeStopWordsFromArray($words);
+
+    // stem words
+    $stemmed_words = array();
+    foreach ($words as $word)
+    {
+      // ignore 1 and 2 letter words
+      if (strlen($word) <= 2)
+      {
+        continue;
+      }
+
+      $stemmed_words[] = PorterStemmer::stem($word, true);
+    }
+
+    return $stemmed_words;
+  }
+
+  public static function removeStopWordsFromArray($words)
+  {
+    $stop_words = array(
+      'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 
+      'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 
+      'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 
+      'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 
+      'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 
+      'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 
+      'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 
+      'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 
+      'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 
+      'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 
+      'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 
+      'than', 'too', 'very',
+    );
+
+    return array_diff($words, $stop_words);
+  }
-}
-