You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
tdepim/kmail/index.cpp

595 lines
17 KiB

/* This file is part of KMail
* Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
*
* KMail is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation.
*
* KMail is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of this program with any edition of
* the TQt library by Trolltech AS, Norway (or with modified versions
* of TQt that use the same license as TQt), and distribute linked
* combinations including the two. You must obey the GNU General
* Public License in all respects for all of the code used other than
* TQt. If you modify this file, you may extend this exception to
* your version of the file, but you are not obligated to do so. If
* you do not wish to do so, delete this exception statement from
* your version.
*/
#include "index.h"
#include "kmkernel.h"
#include "kmfoldermgr.h"
#include "kmmsgdict.h"
#include "kmfolder.h"
#include "kmsearchpattern.h"
#include "kmfoldersearch.h"
#include <kdebug.h>
#include <kapplication.h>
#include <tqfile.h>
#include <tqtimer.h>
#include <tqvaluestack.h>
#include <tqptrlist.h>
#include <tqfileinfo.h>
#ifdef HAVE_INDEXLIB
#include <indexlib/create.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <iostream>
#include <algorithm>
#include <cstdlib>
namespace {
const unsigned int MaintenanceLimit = 1000;
const char* const folderIndexDisabledKey = "fulltextIndexDisabled";
}
#ifdef HAVE_INDEXLIB
static
TQValueList<int> vectorToTQValueList( const std::vector<TQ_UINT32>& input ) {
TQValueList<int> res;
std::copy( input.begin(), input.end(), std::back_inserter( res ) );
return res;
}
static
std::vector<TQ_UINT32> TQValueListToVector( const TQValueList<int>& input ) {
std::vector<TQ_UINT32> res;
// res.assign( input.begin(), input.end() ) doesn't work for some reason
for ( TQValueList<int>::const_iterator first = input.begin(), past = input.end(); first != past; ++first ) {
res.push_back( *first );
}
return res;
}
#endif
KMMsgIndex::KMMsgIndex( TQObject* tqparent ):
TQObject( tqparent, "index" ),
mState( s_idle ),
#ifdef HAVE_INDEXLIB
mLockFile( std::string( static_cast<const char*>( TQFile::encodeName( defaultPath() ) + "/lock" ) ) ),
mIndex( 0 ),
#endif
mIndexPath( TQFile::encodeName( defaultPath() ) ),
mTimer( new TQTimer( this, "mTimer" ) ),
//mSyncState( ss_none ),
//mSyncTimer( new TQTimer( this ) ),
mSlowDown( false ) {
kdDebug( 5006 ) << "KMMsgIndex::KMMsgIndex()" << endl;
connect( kmkernel->folderMgr(), TQT_SIGNAL( msgRemoved( KMFolder*, TQ_UINT32 ) ), TQT_SLOT( slotRemoveMessage( TQ_UINT32 ) ) );
connect( kmkernel->folderMgr(), TQT_SIGNAL( msgAdded( KMFolder*, TQ_UINT32 ) ), TQT_SLOT( slotAddMessage( TQ_UINT32 ) ) );
connect( kmkernel->dimapFolderMgr(), TQT_SIGNAL( msgRemoved( KMFolder*, TQ_UINT32 ) ), TQT_SLOT( slotRemoveMessage( TQ_UINT32 ) ) );
connect( kmkernel->dimapFolderMgr(), TQT_SIGNAL( msgAdded( KMFolder*, TQ_UINT32 ) ), TQT_SLOT( slotAddMessage( TQ_UINT32 ) ) );
connect( mTimer, TQT_SIGNAL( timeout() ), TQT_SLOT( act() ) );
//connect( mSyncTimer, TQT_SIGNAL( timeout() ), TQT_SLOT( syncIndex() ) );
#ifdef HAVE_INDEXLIB
KConfigGroup cfg( KMKernel::config(), "text-index" );
if ( !cfg.readBoolEntry( "enabled", false ) ) {
indexlib::remove( mIndexPath );
mLockFile.force_unlock();
mState = s_disabled;
return;
}
if ( !mLockFile.trylock() ) {
indexlib::remove( mIndexPath );
mLockFile.force_unlock();
mLockFile.trylock();
} else {
mIndex = indexlib::open( mIndexPath, indexlib::open_flags::fail_if_nonexistant ).release();
}
if ( !mIndex ) {
TQTimer::singleShot( 8000, this, TQT_SLOT( create() ) );
mState = s_willcreate;
} else {
if ( cfg.readBoolEntry( "creating" ) ) {
TQTimer::singleShot( 8000, this, TQT_SLOT( continueCreation() ) );
mState = s_creating;
} else {
mPendingMsgs = TQValueListToVector( cfg.readIntListEntry( "pending" ) );
mRemovedMsgs = TQValueListToVector( cfg.readIntListEntry( "removed" ) );
}
}
mIndex = 0;
#else
mState = s_error;
#endif
//if ( mState == s_idle ) mSyncState = ss_synced;
}
KMMsgIndex::~KMMsgIndex() {
kdDebug( 5006 ) << "KMMsgIndex::~KMMsgIndex()" << endl;
#ifdef HAVE_INDEXLIB
KConfigGroup cfg( KMKernel::config(), "text-index" );
cfg.writeEntry( "creating", mState == s_creating );
TQValueList<int> pendingMsg;
if ( mState == s_processing ) {
Q_ASSERT( mAddedMsgs.empty() );
pendingMsg = vectorToTQValueList( mPendingMsgs );
}
cfg.writeEntry( "pending", pendingMsg );
cfg.writeEntry( "removed", vectorToTQValueList( mRemovedMsgs ) );
delete mIndex;
#endif
}
bool KMMsgIndex::isIndexable( KMFolder* folder ) const {
if ( !folder || !folder->tqparent() ) return false;
const KMFolderMgr* manager = folder->tqparent()->manager();
return manager == kmkernel->folderMgr() || manager == kmkernel->dimapFolderMgr();
}
bool KMMsgIndex::isIndexed( KMFolder* folder ) const {
if ( !isIndexable( folder ) ) return false;
KConfig* config = KMKernel::config();
KConfigGroupSaver saver( config, "Folder-" + folder->idString() );
return !config->readBoolEntry( folderIndexDisabledKey, false );
}
void KMMsgIndex::setEnabled( bool e ) {
kdDebug( 5006 ) << "KMMsgIndex::setEnabled( " << e << " )" << endl;
KConfig* config = KMKernel::config();
KConfigGroupSaver saver( config, "text-index" );
if ( config->readBoolEntry( "enabled", !e ) == e ) return;
config->writeEntry( "enabled", e );
if ( e ) {
switch ( mState ) {
case s_idle:
case s_willcreate:
case s_creating:
case s_processing:
// nothing to do
return;
case s_error:
// nothing can be done, probably
return;
case s_disabled:
TQTimer::singleShot( 8000, this, TQT_SLOT( create() ) );
mState = s_willcreate;
}
} else {
clear();
}
}
void KMMsgIndex::setIndexingEnabled( KMFolder* folder, bool e ) {
KConfig* config = KMKernel::config();
KConfigGroupSaver saver( config, "Folder-" + folder->idString() );
if ( config->readBoolEntry( folderIndexDisabledKey, e ) == e ) return; // nothing to do
config->writeEntry( folderIndexDisabledKey, e );
if ( e ) {
switch ( mState ) {
case s_idle:
case s_creating:
case s_processing:
mPendingFolders.push_back( folder );
scheduleAction();
break;
case s_willcreate:
// do nothing, create() will handle this
break;
case s_error:
case s_disabled:
// nothing can be done
break;
}
} else {
switch ( mState ) {
case s_willcreate:
// create() will notice that folder is disabled
break;
case s_creating:
if ( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) != mPendingFolders.end() ) {
// easy:
mPendingFolders.erase( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) );
break;
}
//else fall-through
case s_idle:
case s_processing:
case s_error:
case s_disabled:
// nothing can be done
break;
}
}
}
void KMMsgIndex::clear() {
kdDebug( 5006 ) << "KMMsgIndex::clear()" << endl;
#ifdef HAVE_INDEXLIB
delete mIndex;
mLockFile.force_unlock();
mIndex = 0;
indexlib::remove( mIndexPath );
mPendingMsgs.clear();
mPendingFolders.clear();
mMaintenanceCount = 0;
mAddedMsgs.clear();
mRemovedMsgs.clear();
mExisting.clear();
mState = s_disabled;
for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end(); first != past; ++first ) {
( *first )->close("msgindex");
}
mOpenedFolders.clear();
for ( std::vector<Search*>::const_iterator first = mSearches.begin(), past = mSearches.end(); first != past; ++first ) {
delete *first;
}
mSearches.clear();
mTimer->stop();
#endif
}
void KMMsgIndex::maintenance() {
#ifdef HAVE_INDEXLIB
if ( mState != s_idle || kapp->hasPendingEvents() ) {
TQTimer::singleShot( 8000, this, TQT_SLOT( maintenance() ) );
return;
}
mIndex->maintenance();
#endif
}
int KMMsgIndex::addMessage( TQ_UINT32 serNum ) {
kdDebug( 5006 ) << "KMMsgIndex::addMessage( " << serNum << " )" << endl;
if ( mState == s_error ) return 0;
#ifdef HAVE_INDEXLIB
assert( mIndex );
if ( !mExisting.empty() && std::binary_search( mExisting.begin(), mExisting.end(), serNum ) ) return 0;
int idx = -1;
KMFolder* folder = 0;
KMMsgDict::instance()->getLocation( serNum, &folder, &idx );
if ( !folder || idx == -1 ) return -1;
if ( !mOpenedFolders.count( folder ) ) {
mOpenedFolders.insert( folder );
folder->open("msgindex");
}
KMMessage* msg = folder->getMsg( idx );
/* I still don't know whether we should allow decryption or not.
* Setting to false which makes more sense.
* We keep signature to get the person's name
*/
TQString body = msg->asPlainText( false, false );
if ( !body.isEmpty() && static_cast<const char*>( body.latin1() ) ) {
mIndex->add( body.latin1(), TQString::number( serNum ).latin1() );
} else {
kdDebug( 5006 ) << "Funny, no body" << endl;
}
folder->unGetMsg( idx );
#endif
return 0;
}
void KMMsgIndex::act() {
kdDebug( 5006 ) << "KMMsgIndex::act()" << endl;
if ( kapp->hasPendingEvents() ) {
//nah, some other time..
mTimer->start( 500 );
mSlowDown = true;
return;
}
if ( mSlowDown ) {
mSlowDown = false;
mTimer->start( 0 );
}
if ( !mPendingMsgs.empty() ) {
addMessage( mPendingMsgs.back() );
mPendingMsgs.pop_back();
return;
}
if ( !mPendingFolders.empty() ) {
KMFolder *f = mPendingFolders.back();
mPendingFolders.pop_back();
if ( !mOpenedFolders.count( f ) ) {
mOpenedFolders.insert( f );
f->open("msgindex");
}
const KMMsgDict* dict = KMMsgDict::instance();
KConfig* config = KMKernel::config();
KConfigGroupSaver saver( config, "Folder-" + f->idString() );
if ( config->readBoolEntry( folderIndexDisabledKey, true ) ) {
for ( int i = 0; i < f->count(); ++i ) {
mPendingMsgs.push_back( dict->getMsgSerNum( f, i ) );
}
}
return;
}
if ( !mAddedMsgs.empty() ) {
std::swap( mAddedMsgs, mPendingMsgs );
mState = s_processing;
return;
}
for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end();
first != past;
++first ) {
( *first )->close("msgindex");
}
mOpenedFolders.clear();
mState = s_idle;
mTimer->stop();
}
void KMMsgIndex::continueCreation() {
kdDebug( 5006 ) << "KMMsgIndex::continueCreation()" << endl;
#ifdef HAVE_INDEXLIB
create();
unsigned count = mIndex->ndocs();
mExisting.clear();
mExisting.reserve( count );
for ( unsigned i = 0; i != count; ++i ) {
mExisting.push_back( std::atoi( mIndex->lookup_docname( i ).c_str() ) );
}
std::sort( mExisting.begin(), mExisting.end() );
#endif
}
void KMMsgIndex::create() {
kdDebug( 5006 ) << "KMMsgIndex::create()" << endl;
#ifdef HAVE_INDEXLIB
if ( !TQFileInfo( mIndexPath ).exists() ) {
::mkdir( mIndexPath, S_IRWXU );
}
mState = s_creating;
if ( !mIndex ) mIndex = indexlib::create( mIndexPath ).release();
if ( !mIndex ) {
kdDebug( 5006 ) << "Error creating index" << endl;
mState = s_error;
return;
}
TQValueStack<KMFolderDir*> folders;
folders.push(&(kmkernel->folderMgr()->dir()));
folders.push(&(kmkernel->dimapFolderMgr()->dir()));
while ( !folders.empty() ) {
KMFolderDir *dir = folders.pop();
for(KMFolderNode *child = dir->first(); child; child = dir->next()) {
if ( child->isDir() )
folders.push((KMFolderDir*)child);
else
mPendingFolders.push_back( (KMFolder*)child );
}
}
mTimer->start( 4000 ); // wait a couple of seconds before starting up...
mSlowDown = true;
#endif
}
bool KMMsgIndex::startQuery( KMSearch* s ) {
kdDebug( 5006 ) << "KMMsgIndex::startQuery( . )" << endl;
if ( mState != s_idle ) return false;
if ( !isIndexed( s->root() ) || !canHandleQuery( s->searchPattern() ) ) return false;
kdDebug( 5006 ) << "KMMsgIndex::startQuery( . ) starting query" << endl;
Search* search = new Search( s );
connect( search, TQT_SIGNAL( finished( bool ) ), s, TQT_SIGNAL( finished( bool ) ) );
connect( search, TQT_SIGNAL( finished( bool ) ), s, TQT_SLOT( indexFinished() ) );
connect( search, TQT_SIGNAL( destroyed( TQObject* ) ), TQT_SLOT( removeSearch( TQObject* ) ) );
connect( search, TQT_SIGNAL( found( TQ_UINT32 ) ), s, TQT_SIGNAL( found( TQ_UINT32 ) ) );
mSearches.push_back( search );
return true;
}
//void KMMsgIndex::startSync() {
// switch ( mSyncState ) {
// case ss_none:
// mIndex->start_sync();
// mSyncState = ss_started;
// mSyncTimer.start( 4000, true );
// break;
// case ss_started:
// mIndex->sync_now();
// mSyncState = ss_synced;
// mLockFile.unlock();
// break;
// }
//}
//
//void KMMsgIndex::finishSync() {
//
//}
void KMMsgIndex::removeSearch( TQObject* destroyed ) {
mSearches.erase( std::find( mSearches.begin(), mSearches.end(), destroyed ) );
}
bool KMMsgIndex::stopQuery( KMSearch* s ) {
kdDebug( 5006 ) << "KMMsgIndex::stopQuery( . )" << endl;
for ( std::vector<Search*>::iterator iter = mSearches.begin(), past = mSearches.end(); iter != past; ++iter ) {
if ( ( *iter )->search() == s ) {
delete *iter;
mSearches.erase( iter );
return true;
}
}
return false;
}
std::vector<TQ_UINT32> KMMsgIndex::simpleSearch( TQString s, bool* ok ) const {
kdDebug( 5006 ) << "KMMsgIndex::simpleSearch( -" << s.latin1() << "- )" << endl;
if ( mState == s_error || mState == s_disabled ) {
if ( ok ) *ok = false;
return std::vector<TQ_UINT32>();
}
std::vector<TQ_UINT32> res;
#ifdef HAVE_INDEXLIB
assert( mIndex );
std::vector<unsigned> residx = mIndex->search( s.latin1() )->list();
res.reserve( residx.size() );
for ( std::vector<unsigned>::const_iterator first = residx.begin(), past = residx.end();first != past; ++first ) {
res.push_back( std::atoi( mIndex->lookup_docname( *first ).c_str() ) );
}
if ( ok ) *ok = true;
#endif
return res;
}
bool KMMsgIndex::canHandleQuery( const KMSearchPattern* pat ) const {
kdDebug( 5006 ) << "KMMsgIndex::canHandleQuery( . )" << endl;
if ( !pat ) return false;
TQPtrListIterator<KMSearchRule> it( *pat );
KMSearchRule* rule;
while ( (rule = it.current()) != 0 ) {
++it;
if ( !rule->field().isEmpty() && !rule->contents().isEmpty() &&
rule->function() == KMSearchRule::FuncContains &&
rule->field() == "<body>" ) return true;
}
return false;
}
void KMMsgIndex::slotAddMessage( TQ_UINT32 serNum ) {
kdDebug( 5006 ) << "KMMsgIndex::slotAddMessage( . , " << serNum << " )" << endl;
if ( mState == s_error || mState == s_disabled ) return;
if ( mState == s_creating ) mAddedMsgs.push_back( serNum );
else mPendingMsgs.push_back( serNum );
if ( mState == s_idle ) mState = s_processing;
scheduleAction();
}
void KMMsgIndex::slotRemoveMessage( TQ_UINT32 serNum ) {
kdDebug( 5006 ) << "KMMsgIndex::slotRemoveMessage( . , " << serNum << " )" << endl;
if ( mState == s_error || mState == s_disabled ) return;
if ( mState == s_idle ) mState = s_processing;
mRemovedMsgs.push_back( serNum );
scheduleAction();
}
void KMMsgIndex::scheduleAction() {
#ifdef HAVE_INDEXLIB
if ( mState == s_willcreate || !mIndex ) return;
if ( !mSlowDown ) mTimer->start( 0 );
#endif
}
void KMMsgIndex::removeMessage( TQ_UINT32 serNum ) {
kdDebug( 5006 ) << "KMMsgIndex::removeMessage( " << serNum << " )" << endl;
if ( mState == s_error || mState == s_disabled ) return;
#ifdef HAVE_INDEXLIB
mIndex->remove_doc( TQString::number( serNum ).latin1() );
++mMaintenanceCount;
if ( mMaintenanceCount > MaintenanceLimit && mRemovedMsgs.empty() ) {
TQTimer::singleShot( 100, this, TQT_SLOT( maintenance() ) );
}
#endif
}
TQString KMMsgIndex::defaultPath() {
return KMKernel::localDataPath() + "text-index";
}
bool KMMsgIndex::creating() const {
return !mPendingMsgs.empty() || !mPendingFolders.empty();
}
KMMsgIndex::Search::Search( KMSearch* s ):
mSearch( s ),
mTimer( new TQTimer( this, "mTimer" ) ),
mResidual( new KMSearchPattern ),
mState( s_starting ) {
connect( mTimer, TQT_SIGNAL( timeout() ), TQT_SLOT( act() ) );
mTimer->start( 0 );
}
KMMsgIndex::Search::~Search() {
delete mTimer;
}
void KMMsgIndex::Search::act() {
switch ( mState ) {
case s_starting: {
KMSearchPattern* pat = mSearch->searchPattern();
TQString terms;
for ( KMSearchRule* rule = pat->first(); rule; rule = pat->next() ) {
Q_ASSERT( rule->function() == KMSearchRule::FuncContains );
terms += TQString::tqfromLatin1( " %1 " ).tqarg( rule->contents() );
}
mValues = kmkernel->msgIndex()->simpleSearch( terms, 0 );
break;
}
case s_emitstopped:
mTimer->start( 0 );
mState = s_emitting;
// fall throu
case s_emitting:
if ( kapp->hasPendingEvents() ) {
//nah, some other time..
mTimer->start( 250 );
mState = s_emitstopped;
return;
}
for ( int i = 0; i != 16 && !mValues.empty(); ++i ) {
KMFolder* folder;
int index;
KMMsgDict::instance()->getLocation( mValues.back(), &folder, &index );
if ( folder &&
mSearch->inScope( folder ) &&
( !mResidual || mResidual->matches( mValues.back() ) ) ) {
emit found( mValues.back() );
}
mValues.pop_back();
}
if ( mValues.empty() ) {
emit finished( true );
mState = s_done;
mTimer->stop();
delete this;
}
break;
default:
Q_ASSERT( 0 );
}
}
#include "index.moc"