makary/phpbb
1
0
Fork 0
mirror of https://github.com/phpbb/phpbb.git synced 2025-06-28 22:28:51 +00:00
Code Issues Projects Releases Packages Wiki Activity

Added title support and updated for changes in includes/search ... title slows it down, guess it needs some work if possible ...

Browse source 
git-svn-id: file:///svn/phpbb/trunk@1939 89ea8834-ac86-4346-8a33-228a782c2dd0
This commit is contained in:
Paul S. Owen 2002-01-25 01:31:42 +00:00
parent c8c850f104
commit fdc8973252
1 changed files with 86 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
phpBB/develop/search_fill.php
View file

@ -1,7 +1,7 @@
<?php
set_time_limit(0);
$common_percent = 40; // Percentage of posts in which a word has to appear to be marked as common
$common_percent = 0.4; // Percentage of posts in which a word has to appear to be marked as common
$phpbb_root_path = "../";
@ -21,28 +21,21 @@ print "<html>\n<body>\n";
//$synonym_array = file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_synonyms.txt");
// This needs fixing! Shouldn't be hardcoded to English files!
$stopword_array = file($phpbb_root_path . "language/lang_english/search_stopwords.txt");
//$stopword_array = file($phpbb_root_path . "language/lang_english/search_stopwords.txt");
$synonym_array = file($phpbb_root_path . "language/lang_english/search_synonyms.txt");
/*
for ($j = 0; $j < count($stopword_array); $j++)
{
$filter_word = trim(strtolower($stopword_array[$j]));
$search[] = "/\b" . phpbb_preg_quote($filter_word, "/") . "\b/is";
$replace[] = '';
}
for ($j = 0; $j < count($synonym_list); $j++)
{
list($replace_synonym, $match_synonym) = split(" ", trim(strtolower($synonym_list[$j])));
$search[] = "/\b" . phpbb_preg_quote(trim($match_synonym), "/") . "\b/is";
$replace[] = " " . trim($replace_synonym) . " ";
}
*/
//
// Fetch a batch of posts_text entries
//
$sql = "
SELECT count(*) as total, max(post_id) as max_post_id
$sql = "SELECT count(*) as total, max(post_id) as max_post_id
FROM ". POSTS_TEXT_TABLE;
if(!$result = $db->sql_query($sql))
{
@ -92,37 +85,47 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
$matches = array();
$post_id = $rowset[$post_nr]['post_id'];
$data = $rowset[$post_nr]['post_text']; // Raw data
$text = clean_words("post", $rowset[$post_nr]['post_text'], $synonym_array); // Cleaned up post
$text_title = clean_words("post", $rowset[$post_nr]['post_subject'], $synonym_array);
$matches = array();
$matches['text'] = split_words($text);
$matches['title'] = split_words($text_title);
while( list($match_type, $match_ary) = @each($matches) )
{
$title_match = ( $match_type == 'title' ) ? 1 : 0;
$num_matches = count($match_ary);
$text = clean_words($data, $search, $replace); // Cleaned up post
$matches = split_words($text);
$num_matches = count($matches);
if ( $num_matches < 1 )
{
// Skip this post if no words where found
continue;
}
$word = array();
$word_count = array();
$sql_in = "";
$phrase_string = $text;
// For all words in the posting
$sql_in = "";
$sql_insert = '';
$sql_select = '';
$word = array();
$word_count = array();
for($j = 0; $j < $num_matches; $j++)
{
$this_word = strtolower(trim($matches[$j]));
$this_word = strtolower(trim($match_ary[$j]));
if ( $this_word != '' )
{
$word_count[$this_word]++;
$word_count[$this_word] = ( isset($word_count[$this_word]) ) ? $word_count[$this_word] + 1 : 0;
$comma = ($sql_insert != '')? ', ': '';
$sql_insert .= "$comma('" . $this_word . "')";
$sql_select .= "$comma'" . $this_word . "'";
}
}
if ( $sql_insert == '' )
{
die("no words found");
@ -148,28 +151,18 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
$error = $db->sql_error();
die("Couldn't select words :: " . $sql . " :: " . $error['message']);
}
if( $word_check_count = $db->sql_numrows($result) )
{
$selected_words = $db->sql_fetchrowset($result);
}
else
{
print "Couldn't do sql_numrows<br>\n";
}
$db->sql_freeresult($result);
$sql_insert = '';
while(list($junk, $row) = each($selected_words))
$sql_insert = array();
while( $row = $db->sql_fetchrow($result) )
{
$comma = ($sql_insert != '')? ', ': '';
$sql_insert .= "$comma($post_id, ".$row['word_id'].", 0)";
$sql_insert[] = "($post_id, " . $row['word_id'] . ", $title_match)";
}
$db->sql_freeresult($result);
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . "
(post_id, word_id, title_match)
VALUES
$sql_insert
";
VALUES " . implode(", ", $sql_insert);
$result = $db->sql_query($sql);
if ( !$result )
{
@ -177,20 +170,8 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
die("Couldn't insert new word match :: " . $sql . " :: " . $error['message']);
}
/*
//$phrase_string = preg_replace("/\b" . phpbb_preg_quote($word[$j], "/") . "\b/is", $word_id, $phrase_string);
$phrase_string = trim(preg_replace("/ {2,}/s", " ", str_replace(array("*", "'"), " ", $phrase_string)));
$sql = "INSERT INTO phpbb_search_phrasematch (post_id, phrase_list)
VALUES ($post_id, '$phrase_string')";
$result = $db->sql_query($sql);
if( !$result )
{
$error = $db->sql_error();
die("Couldn't insert new phrase match :: " . $sql . " :: " . $error['message']);
}
*/
} // All posts
}
// $sql = "UNLOCK TABLES";
// $result = $db->sql_query($sql);
@ -208,9 +189,8 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
{
print "<br>Removing common words (words that appear in more than $common_percent of the posts)<br>\n";
flush();
print "Removed ". remove_common_global($common_percent, 1) ." words that where too common.<br>";
print "Removed ". remove_common("global", $common_percent) ." words that where too common.<br>";
}
}
echo "<br>Done";