tdevelop/parts/documentation/tools/htdig/htdigindex.cpp

/***************************************************************************
 *   Copyright (C) 1999-2001 by Matthias Hoelzer-Kluepfel                  *
 *   hoelzer@kde.org                                                       *
 *   Copyright (C) 2001 by Bernd Gehrmann                                  *
 *   bernd@kdevelop.org                                                    *
 *   Copyright (C) 2004 by Alexander Dymo                                  *
 *   cloudtemple@mksat.net                                                 *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
#include "htdigindex.h"

#include <iostream>

#include <tqapplication.h>
#include <tqdir.h>
#include <tqfile.h>
#include <tqlayout.h>
#include <tqtextstream.h>
#include <tqtimer.h>
#include <tqlabel.h>

#include <tdeaboutdata.h>
#include <tdeapplication.h>
#include <tdecmdlineargs.h>
#include <tdeconfig.h>
#include <kdebug.h>
#include <tdeglobal.h>
#include <tdelocale.h>
#include <tdemessagebox.h>
#include <kstandarddirs.h>
#include <kprocess.h>
#include <tdeversion.h>
#include <kprogress.h>

#define INDEXER

ProgressDialog::ProgressDialog(bool index, TQWidget *parent, const char *name)
    :KDialogBase(KDialogBase::Plain, i18n("Generating Search Index"), Cancel | Ok, Close,
    parent, name, false)
{
    proc = 0;

    indexdir = kapp->dirs()->saveLocation("data", "tdevdocumentation/search");
    TQDir d; d.mkdir(indexdir);

    TDEConfig config("tdevdocumentation", true);
    config.setGroup("htdig");
    databaseDir = config.readPathEntry("databaseDir", indexdir);

    if (!index)
        return;

    d.mkdir( databaseDir );

    showButtonOK( false );
    TQGridLayout *grid = new TQGridLayout(plainPage(), 5,3, spacingHint());

    TQLabel *l = new TQLabel(i18n("Scanning for files"), plainPage());
    grid->addMultiCellWidget(l, 0, 0, 1, 2);

    filesLabel = new TQLabel(plainPage());
    grid->addWidget(filesLabel, 1, 2);
    setFilesScanned(0);

    check1 = new TQLabel(plainPage());
    grid->addWidget(check1, 0, 0);

    l = new TQLabel(i18n("Extracting search terms"), plainPage());
    grid->addMultiCellWidget(l, 2,2, 1,2);

    bar = new KProgress(plainPage());
    grid->addWidget(bar, 3,2);

    check2 = new TQLabel(plainPage());
    grid->addWidget(check2, 2,0);

    l = new TQLabel(i18n("Generating index..."), plainPage());
    grid->addMultiCellWidget(l, 4,4, 1,2);

    check3 = new TQLabel(plainPage());
    grid->addWidget(check3, 4,0);

    setState(0);

    setMinimumWidth(300);
    connect(this, TQT_SIGNAL(cancelClicked()), this, TQT_SLOT(cancelClicked()));
    connect(this, TQT_SIGNAL(okClicked()), this, TQT_SLOT(okClicked()));
    TQTimer::singleShot(0, this, TQT_SLOT(slotDelayedStart()));
}

ProgressDialog::~ProgressDialog()
{
}

void ProgressDialog::slotDelayedStart()
{
    procdone = false;
    scanDirectories();
    if (!createConfig())
    {
      done(1);
      return;
    }
    generateIndex();
}

void ProgressDialog::done(int r)
{
    if (!r)
    {
        showButtonCancel( false );
        showButtonOK( true );
    }
    else
        KDialogBase::done(r);
}

void ProgressDialog::setFilesScanned(int n)
{
    filesLabel->setText(i18n("Files processed: %1").arg(n));
}

void ProgressDialog::setFilesToDig(int n)
{
    bar->setTotalSteps(n);
}

void ProgressDialog::setFilesDigged(int n)
{
    bar->setValue(n);
}

void ProgressDialog::setState(int n)
{
    TQPixmap unchecked = TQPixmap(locate("data", "tdevdocumentation/pics/unchecked.xpm"));
    TQPixmap checked = TQPixmap(locate("data", "tdevdocumentation/pics/checked.xpm"));

    check1->setPixmap( n > 0 ? checked : unchecked);
    check2->setPixmap( n > 1 ? checked : unchecked);
    check3->setPixmap( n > 2 ? checked : unchecked);
}


void ProgressDialog::addDir(const TQString &dir)
{
    kdDebug(9002) << "Add dir : " << dir << endl;
    TQDir d(dir, "*.html", TQDir::Name|TQDir::IgnoreCase, TQDir::Files | TQDir::Readable);
    TQStringList list = d.entryList();

    TQStringList::ConstIterator it;
    for ( it=list.begin(); it!=list.end(); ++it )
    {
        if( (*it).right( 12 ).lower( ) == "-source.html" )
            continue;

        files.append(dir + "/" + *it);
        setFilesScanned(++filesScanned);
    }

    TQDir d2(dir, TQString(), TQDir::Name|TQDir::IgnoreCase, TQDir::Dirs);
    TQStringList dlist = d2.entryList();

    for ( it=dlist.begin(); it != dlist.end(); ++it )
    {
        if (*it != "." && *it != "..")
        {
            addDir(dir + "/" + *it);
            kapp->processEvents();
        }
        if (procdone)
        {
          return;
        }
    }
    kapp->processEvents();
}

void ProgressDialog::scanDirectories()
{
    TQString ftsLocationsFile = locateLocal("data", "tdevdocumentation/search/locations.txt");

    TQFile f(ftsLocationsFile);
    if (!f.open(IO_ReadOnly))
        return;
    TQTextStream str(&f);

    filesScanned = 0;

    while (!str.eof())
    {
        TQString loc = str.readLine();
        if (loc.isEmpty())
            continue;
        TQFileInfo fi(loc);
        if (fi.isDir())
            addDir(loc);
        else if (fi.isFile())
        {
            files.append(loc);
            setFilesScanned(++filesScanned);
        }
    }
}

bool ProgressDialog::createConfig()
{
    // locate the common dir
    TQString language = TDEGlobal::locale()->language();
    if (language == "C")
        language = "en";

    TQString wrapper = locate("data", TQString("tdevdocumentation/%1/wrapper.html").arg(language));
    if (wrapper.isEmpty())
        wrapper = locate("data", TQString("tdevdocumentation/en/wrapper.html"));
    if (wrapper.isEmpty())
        return false;
    wrapper = wrapper.left(wrapper.length()-12);

    // locate the image dir
    TQString images = locate("data", "tdevdocumentation/pics/star.png");
    if (images.isEmpty())
        return false;
    images = images.left(images.length()-8);

    TQFile f(indexdir + "/htdig.conf");
    if (f.open(IO_WriteOnly))
    {
        TQTextStream ts(&f);

        ts << "database_dir:\t\t" << databaseDir << endl;
        ts << "start_url:\t\t`" << indexdir << "/files`" << endl;
        ts << "local_urls:\t\thttp://localhost/=/" << endl;
//        ts << "local_urls:\t\tfile://=" << endl;
        ts << "local_urls_only:\ttrue" << endl;
        ts << "limit_urls_to:\t\tfile:// http://localhost/" << endl;
        ts << "maximum_pages:\t\t1" << endl;
        ts << "image_url_prefix:\t" << images << endl;
        ts << "star_image:\t\t" << images << "star.png" << endl;
        ts << "star_blank:\t\t" << images << "star_blank.png" << endl;
        ts << "compression_level:\t6" << endl;
        ts << "max_hop_count:\t\t0" << endl;

        ts << "search_results_wrapper:\t" << wrapper << "wrapper.html" << endl;
        ts << "nothing_found_file:\t" << wrapper << "nomatch.html" << endl;
        ts << "syntax_error_file:\t" << wrapper << "syntax.html" << endl;
        ts << "bad_word_list:\t\t" << wrapper << "bad_words" << endl;

        f.close();
        return true;
    }

    return false;
}

#define CHUNK_SIZE 100

void ProgressDialog::startHtdigProcess(bool initial)
{
    kdDebug(9002) << "htdig started" << endl;
    delete proc;
    proc = new TDEProcess();
    *proc << exe << "-c" << (indexdir + "/htdig.conf");
    if (initial) {
        *proc << "-i";
    }
    connect(proc, TQT_SIGNAL(processExited(TDEProcess *)),
            this, TQT_SLOT(htdigExited(TDEProcess *)));

    htdigRunning = true;

    // write out file
    TQFile f(indexdir+"/files");
    if (!f.open(IO_WriteOnly)) {
        kdDebug(9002) << "Could not open `files` for writing" << endl;
        done(1);
        return;
    }
    TQTextStream ts(&f);
    for (int i=0; i<CHUNK_SIZE; ++i, ++count) {
        if (count >= filesToDig) {
            procdone = true;
            break;
        }
    //    ts << "file://localhost/" + files[count] << endl;
        ts << "http://localhost/" + files[count] << endl;
    }
    f.close();

    // execute htdig
    proc->start(TDEProcess::NotifyOnExit, TDEProcess::Stdout);
}

bool ProgressDialog::generateIndex()
{
    setState(1);
    procdone = false;
    // run htdig
    TDEConfig config("tdevdocumentation", true);
    config.setGroup("htdig");
    exe = config.readPathEntry("htdigbin", kapp->dirs()->findExe("htdig"));
    if (exe.isEmpty())
    {
        done(1);
        return true;
    }
    filesToDig = files.count();
    count = 0;
    setFilesToDig(filesToDig);
    filesDigged = 0;

    //    TQDir d; d.mkdir(indexdir);
    startHtdigProcess(true);
    return true;
}

void ProgressDialog::htdigStdout(TDEProcess *, char *buffer, int len)
{
    TQString line = TQString(buffer).left(len);

    int cnt=0, index=-1;
    while ( (index = line.find("http://", index+1)) > 0)
        cnt++;
    filesDigged += cnt;

    cnt=0, index=-1;
    while ( (index = line.find("not changed", index+1)) > 0)
        cnt++;
    filesDigged -= cnt;

    setFilesDigged(filesDigged);
}

void ProgressDialog::htdigExited(TDEProcess *proc)
{
    kdDebug(9002) << "htdig terminated" << endl;
    if (!proc->normalExit())
    {
        delete proc;
        proc = 0L;
        done(1);
        return;
    }
    if (proc && proc->exitStatus() != 0)
    {
        KMessageBox::sorry(0, i18n("Running htdig failed"));
        delete proc;
        proc = 0L;
        done(1);
        return;
    }
    htdigRunning = false;
    filesDigged += CHUNK_SIZE;
    setFilesDigged(filesDigged);
    if (!procdone)
    {
        startHtdigProcess(false);
    }
    else
    {
        setFilesDigged(filesToDig);
        setState(2);

        TDEConfig config("tdevdocumentation", true);
        config.setGroup("htdig");
        // run htmerge -----------------------------------------------------
        exe = config.readPathEntry("htmergebin", kapp->dirs()->findExe("htmerge"));
        if (exe.isEmpty())
        {
            done(1);
            return;
        }
        startHtmergeProcess();
    }
}

void ProgressDialog::startHtmergeProcess()
{
    kdDebug(9002) << "htmerge started" << endl;
    delete proc;
    proc = new TDEProcess();
    *proc << exe << "-c" << (indexdir + "/htdig.conf");

    kdDebug(9002) << "Running htmerge" << endl;

    connect(proc, TQT_SIGNAL(processExited(TDEProcess *)),
            this, TQT_SLOT(htmergeExited(TDEProcess *)));

    htmergeRunning = true;

    proc->start(TDEProcess::NotifyOnExit, TDEProcess::Stdout);
}

void ProgressDialog::htmergeExited(TDEProcess *proc)
{
    kdDebug(9002) << "htmerge terminated" << endl;
    htmergeRunning = false;
    if (!proc->normalExit())
    {
        delete proc;
        proc = 0L;
        done(1);
        return;
    }
    if (proc && proc->exitStatus() != 0)
    {
        KMessageBox::sorry(0, i18n("Running htmerge failed"));
        delete proc;
        proc = 0L;
        done(1);
        return;
    }
    setState(3);
    done(0);
}

void ProgressDialog::cancelClicked()
{
    if ((htdigRunning || htmergeRunning) && proc && proc->isRunning())
    {
        kdDebug(9002) << "Killing " << (htdigRunning ? "htdig" : "htmerge") << "daemon with Sig. 9" << endl;
        proc->kill(9);
        htdigRunning = htmergeRunning = false;
    }
    else
    {
        procdone = true;
        done(2);
    }
}

void ProgressDialog::okClicked()
{
    if (proc)
        proc->kill();

    KDialogBase::done(0);
}

int main(int argc, char *argv[])
{
    static const TDECmdLineOptions options[] =
    {
        { "c", I18N_NOOP( "Update user's htdig configuration file only" ), 0 },
        { "i",  I18N_NOOP( "-c and generate index" ), 0 },
        TDECmdLineLastOption
    };

    TDEAboutData aboutData("tdevelop-htdig", I18N_NOOP("TDevelop ht://Dig Indexer"),
                         "0.2", I18N_NOOP("TDE Index generator for documentation files."));

    TDECmdLineArgs::init(argc, argv, &aboutData);
    TDECmdLineArgs::addCmdLineOptions(options);

    TDEApplication app;

    TDEGlobal::locale()->setMainCatalogue("tdevelop");

    TDECmdLineArgs *args = TDECmdLineArgs::parsedArgs();

    if (args->isSet("c"))
    {
        ProgressDialog *search = new ProgressDialog( false, 0, "progress dialog");

        if (search->createConfig())
            KMessageBox::information(0, i18n("Configuration file updated."));
        else
            KMessageBox::error(0, i18n("Configuration file update failed."));
    }
    else
        if (args->isSet("i"))
        {
            ProgressDialog *search = new ProgressDialog(true, 0, "progress dialog");
            app.setMainWidget(search);
            search->show();
            app.exec();
        }
        else
        {
            std::cerr << "Internal error generating index - unknown argument\n" << std::endl;
            return 1;
        }

    return 0;
}

#include "htdigindex.moc"