digikam/digikam/digikam/scanlib.cpp

/* ============================================================
 *
 * This file is a part of digiKam project
 * http://www.digikam.org
 *
 * Date        : 2005-01-01
 * Description : scan pictures interface.
 * 
 * Copyright (C) 2005-2006 by Tom Albers <tomalbers@kde.nl>
 * Copyright (C) 2007-2008 by Gilles Caulier <caulier dot gilles at gmail dot com>
 *
 * This program is free software; you can redistribute it
 * and/or modify it under the terms of the GNU General
 * Public License as published by the Free Software Foundation;
 * either version 2, or (at your option)
 * any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * ============================================================ */

// C Ansi includes.

extern "C"
{
#include <sys/time.h>
}

// C++ includes.

#include <ctime>
#include <cstdlib>

// TQt includes.

#include <tqapplication.h>
#include <tqstring.h>
#include <tqstringlist.h>
#include <tqmap.h>
#include <tqdir.h>
#include <tqfileinfo.h>
#include <tqwhatsthis.h>
#include <tqpixmap.h>

// KDE includes.

#include <kprogress.h>
#include <kmessagebox.h>
#include <kapplication.h>
#include <klocale.h>
#include <kiconloader.h>

// Local includes.

#include "ddebug.h"
#include "dprogressdlg.h"
#include "dmetadata.h"
#include "albumdb.h"
#include "albummanager.h"
#include "splashscreen.h"
#include "scanlib.h"

/** @file scanlib.cpp*/

namespace Digikam
{

ScanLib::ScanLib(SplashScreen *splash)
{
    m_splash = splash;
    m_progressBar = new DProgressDlg(0);
    m_progressBar->setInitialSize(TQSize(500, 100), true);
    m_progressBar->setActionListVSBarVisible(false);
    TQWhatsThis::add( m_progressBar, i18n("This shows the progress of the "
        "scan. During the scan, all files on disk are registered in a "
        "database. This is required for sorting by exif-date, and also speeds up "
        "the overall performance of digiKam.") );

    // these two lines prevent the dialog to be shown in
    // findFoldersWhichDoNotExist() method.
    m_progressBar->progressBar()->setTotalSteps(1);
    m_progressBar->progressBar()->setProgress(1);   
}

ScanLib::~ScanLib()
{
    delete m_progressBar;
}

void ScanLib::startScan()
{
    struct timeval tv1, tv2;
    TQPixmap pix = KApplication::kApplication()->iconLoader()->loadIcon(
                  "run", KIcon::NoGroup, 32);

    TQString message = i18n("Finding non-existent Albums");
    if (m_splash) m_splash->message(message);
    else m_progressBar->addedAction(pix, message);
    gettimeofday(&tv1, 0);
    findFoldersWhichDoNotExist();
    gettimeofday(&tv2, 0);
    timing(message, tv1, tv2);

    message = i18n("Finding items not in database");
    if (m_splash) m_splash->message(message);
    else m_progressBar->addedAction(pix, message);
    gettimeofday(&tv1, 0);
    findMissingItems();
    gettimeofday(&tv2, 0);
    timing(message, tv1, tv2);

    message = i18n("Updating items without a date");
    if (m_splash) m_splash->message(message);
    else m_progressBar->addedAction(pix, message);
    gettimeofday(&tv1, 0);
    updateItemsWithoutDate();
    gettimeofday(&tv2, 0);
    timing(message, tv1, tv2);

    deleteStaleEntries();

    AlbumDB* db = AlbumManager::instance()->albumDB();
    db->setSetting("Scanned", TQDateTime::currentDateTime().toString(Qt::ISODate));
}

void ScanLib::findFoldersWhichDoNotExist()
{
    TQMap<TQString, int> toBeDeleted;
    TQString basePath(AlbumManager::instance()->getLibraryPath());

    AlbumDB* db = AlbumManager::instance()->albumDB();
    AlbumInfo::List aList = db->scanAlbums();
    
    for (AlbumInfo::List::iterator it = aList.begin(); it != aList.end(); ++it)
    {
        AlbumInfo info = *it;
        info.url = TQDir::cleanDirPath(info.url);
        TQFileInfo fi(basePath + info.url);
        if (!fi.exists() || !fi.isDir())
        {
            toBeDeleted[info.url] = info.id;
        }
    }

    kapp->processEvents();

    if (!toBeDeleted.isEmpty())
    {
        int rc = KMessageBox::warningYesNoList(0,
            i18n("<p>There is an album in the database which does not appear to "
                 "be on disk. This album should be removed from the database, "
                 "however you may lose information because all images "
                 "associated with this album will be removed from the database "
                 "as well.<p>"
                 "digiKam cannot continue without removing the items "
                 "from the database because all views depend on the information "
                 "in the database. Do you want them to be removed from the "
                 "database?",
                 "<p>There are %n albums in the database which do not appear to "
                 "be on disk. These albums should be removed from the database, "
                 "however you may lose information because all images "
                 "associated with these albums will be removed from the database "
                 "as well.<p>"
                 "digiKam cannot continue without removing the items "
                 "from the database because all views depend on the information "
                 "in the database. Do you want them to be removed from the "
                 "database?",
                 toBeDeleted.count()),
            toBeDeleted.keys(),
            i18n("Albums are Missing"));

        if (rc != KMessageBox::Yes)
            exit(0);

        TQMapIterator<TQString,int> it;
        for (it = toBeDeleted.begin() ; it != toBeDeleted.end(); ++it)
        {
            DDebug() << "Removing Album: " << it.key() << endl;
            db->deleteAlbum( it.data() );
        }
    }
}

void ScanLib::findMissingItems(const TQString &path)
{
    allFiles(path);
}

void ScanLib::findMissingItems()
{
    TQString albumPath = AlbumManager::instance()->getLibraryPath();
    albumPath = TQDir::cleanDirPath(albumPath);

    m_progressBar->setAllowCancel( false );
    m_progressBar->showCancelButton (false );
    m_progressBar->progressBar()->setProgress( 0 );
    m_progressBar->setLabel(i18n("Scanning items, please wait..."));
    m_progressBar->progressBar()->setTotalSteps( countItemsInFolder( albumPath ) );
    if (!m_splash) m_progressBar->show();
    kapp->processEvents();

    TQDir dir(albumPath);
    TQStringList fileList(dir.entryList(TQDir::Dirs));
    TQPixmap pix = KApplication::kApplication()->iconLoader()->loadIcon(
                  "folder_image", KIcon::NoGroup, 32);

    AlbumDB* db = AlbumManager::instance()->albumDB();
    db->beginTransaction(); 

    for (TQStringList::iterator it = fileList.begin(); it != fileList.end(); ++it)
    {
        if ((*it) == "." || (*it) == "..")
            continue;

        TQString path = albumPath + '/' + (*it);
        allFiles(path);
        m_progressBar->addedAction(pix, path);
    }
    db->commitTransaction();    

    m_progressBar->hide();
    kapp->processEvents();
}

void ScanLib::updateItemsWithoutDate()
{
    AlbumDB* db = AlbumManager::instance()->albumDB();
    TQStringList urls = db->getAllItemURLsWithoutDate();

    if (urls.isEmpty())
    {
        m_progressBar->progressBar()->setTotalSteps(1);
        m_progressBar->progressBar()->setProgress(1);
        m_progressBar->hide();
        return;
    }

    m_progressBar->setAllowCancel( false );
    m_progressBar->showCancelButton (false );
    m_progressBar->progressBar()->setProgress(0);
    m_progressBar->progressBar()->setTotalSteps(urls.count());
    m_progressBar->setLabel(i18n("Updating items, please wait..."));
    m_progressBar->show();
    kapp->processEvents();

    TQString basePath = AlbumManager::instance()->getLibraryPath();
    basePath = TQDir::cleanDirPath(basePath);

    db->beginTransaction();    

    int counter=0;
    for (TQStringList::iterator it = urls.begin(); it != urls.end(); ++it)
    {
        m_progressBar->progressBar()->advance(1);
        ++counter;
        if ( counter % 30 == 0 )
        {
            kapp->processEvents();
        }

        TQFileInfo fi(*it);
        TQString albumURL = fi.dirPath();
        albumURL = TQDir::cleanDirPath(albumURL.remove(basePath));
        
        int albumID = db->getOrCreateAlbumId(albumURL);

        if (albumID <= 0)
        {
            DWarning() << "Album ID == -1: " << albumURL << endl;
        }
        
        if (fi.exists())
        {
            updateItemDate(albumURL, fi.fileName(), albumID);
        }
        else
        {
            TQPair<TQString, int> fileID = tqMakePair(fi.fileName(),albumID);
            
            if (m_filesToBeDeleted.findIndex(fileID) == -1)
            {
                m_filesToBeDeleted.append(fileID);
            }
        }
    }

    db->commitTransaction();    
    
    m_progressBar->hide();
    kapp->processEvents();
}

int ScanLib::countItemsInFolder(const TQString& directory)
{
    int items = 0;

    TQDir dir( directory );
    if ( !dir.exists() or !dir.isReadable() )
        return 0;

    const TQFileInfoList *list = dir.entryInfoList();
    TQFileInfoListIterator it( *list );
    TQFileInfo *fi;

    items += list->count();

    while ( (fi = it.current()) != 0 )
    {
        if ( fi->isDir() &&
             fi->fileName() != "." &&
             fi->fileName() != "..")
        {
            items += countItemsInFolder( fi->filePath() );
        }
        
        ++it;
    }

    return items;
}

void ScanLib::allFiles(const TQString& directory)
{
    TQDir dir( directory );
    if ( !dir.exists() or !dir.isReadable() )
    {
        DWarning() << "Folder does not exist or is not readable: "
                    << directory << endl;
        return;
    }

    TQString basePath = AlbumManager::instance()->getLibraryPath();
    basePath = TQDir::cleanDirPath(basePath);
            
    TQString albumURL = directory;
    albumURL = TQDir::cleanDirPath(albumURL.remove(basePath));

    AlbumDB* db = AlbumManager::instance()->albumDB();

    int albumID = db->getOrCreateAlbumId(albumURL);

    if (albumID <= 0)
    {
        DWarning() << "Album ID == -1: " << albumURL << endl;
    }
    
    TQStringList filesInAlbum = db->getItemNamesInAlbum( albumID );
    TQMap<TQString, bool> filesFoundInDB;
    
    for (TQStringList::iterator it = filesInAlbum.begin();
         it != filesInAlbum.end(); ++it)
    {
        filesFoundInDB.insert(*it, true);
    }

    const TQFileInfoList *list = dir.entryInfoList();
    if (!list)
        return;

    TQFileInfoListIterator it( *list );
    TQFileInfo *fi;

    m_progressBar->progressBar()->advance(list->count());
    kapp->processEvents();

    while ( (fi = it.current()) != 0 )
    {
        if ( fi->isFile())
        {
            if (filesFoundInDB.contains(fi->fileName()) )
            {
                filesFoundInDB.erase(fi->fileName());
            }
            else
            {
                storeItemInDatabase(albumURL, fi->fileName(), albumID);
            }
        }
        else if ( fi->isDir() && fi->fileName() != "." && fi->fileName() != "..")
        {
            allFiles( fi->filePath() );
        }
        
        ++it;
    }

    // Removing items from the db which we did not see on disk.
    if (!filesFoundInDB.isEmpty())
    {
        TQMapIterator<TQString,bool> it;
        for (it = filesFoundInDB.begin(); it != filesFoundInDB.end(); ++it)
        {
            if (m_filesToBeDeleted.findIndex(tqMakePair(it.key(),albumID)) == -1)
            {
                m_filesToBeDeleted.append(tqMakePair(it.key(),albumID));
            }
        }
    }
}

void ScanLib::storeItemInDatabase(const TQString& albumURL,
                                  const TQString& filename,
                                  int albumID)
{
    // Do not store items found in the root of the albumdb
    if (albumURL.isEmpty())
        return;

    TQString     comment;
    TQStringList keywords;
    TQDateTime   datetime;
    int         rating;

    TQString filePath( AlbumManager::instance()->getLibraryPath());
    filePath += albumURL + '/' + filename;

    DMetadata metadata(filePath);

    // Try to get comments from image :
    // In first, from standard JPEG comments, or
    // In second, from EXIF comments tag, or
    // In third, from IPTC comments tag.

    comment = metadata.getImageComment();

    // Try to get date and time from image :
    // In first, from EXIF date & time tags, or
    // In second, from IPTC date & time tags.

    datetime = metadata.getImageDateTime();

    // Try to get image rating from IPTC Urgency tag 
    // else use file system time stamp.
    rating = metadata.getImageRating();

    if ( !datetime.isValid() )
    {
        TQFileInfo info( filePath );
        datetime = info.lastModified();
    }

    // Try to get image tags from IPTC keywords tags.

    keywords = metadata.getImageKeywords();

    AlbumDB* dbstore = AlbumManager::instance()->albumDB();
    dbstore->addItem(albumID, filename, datetime, comment, rating, keywords);
}

void ScanLib::updateItemDate(const TQString& albumURL,
                             const TQString& filename,
                             int albumID)
{
    TQDateTime datetime;

    TQString filePath( AlbumManager::instance()->getLibraryPath());
    filePath += albumURL + '/' + filename;

    DMetadata metadata(filePath);

    // Trying to get date and time from image :
    // In first, from EXIF date & time tags, or
    // In second, from IPTC date & time tags.

    datetime = metadata.getImageDateTime();

    if ( !datetime.isValid() )
    {
        TQFileInfo info( filePath );
        datetime = info.lastModified();
    }

    AlbumDB* dbstore = AlbumManager::instance()->albumDB();
    dbstore->setItemDate(albumID, filename, datetime);
}

void ScanLib::deleteStaleEntries()
{
    TQStringList listToBeDeleted;
    TQValueList< TQPair<TQString,int> >::iterator it;
    
    for (it = m_filesToBeDeleted.begin() ; it != m_filesToBeDeleted.end(); ++it)
    {
        AlbumDB* dbstore = AlbumManager::instance()->albumDB();
        TQString location = " (" + dbstore->getAlbumURL((*it).second) + ')';

        listToBeDeleted.append((*it).first + location);
    }

    if ( !m_filesToBeDeleted.isEmpty() )
    {
        int rc = KMessageBox::warningYesNoList(0,
          i18n("<p>There is an item in the database which does not "
               "appear to be on disk or is located in the root album of "
               "the path. This file should be removed from the "
               "database, however you may lose information.<p>"
               "digiKam cannot continue without removing the item from "
               "the database because all views depend on the information "
               "in the database. Do you want it to be removed from the "
               "database?",
               "<p>There are %n items in the database which do not "
               "appear to be on disk or are located in the root album of "
               "the path. These files should be removed from the "
               "database, however you may lose information.<p>"
               "digiKam cannot continue without removing these items from "
               "the database because all views depend on the information "
               "in the database. Do you want them to be removed from the "
               "database?",
               listToBeDeleted.count()),
          listToBeDeleted,
          i18n("Files are Missing"));

        if (rc != KMessageBox::Yes)
            exit(0);

        AlbumDB* db = AlbumManager::instance()->albumDB();
        db->beginTransaction();
        for (it = m_filesToBeDeleted.begin() ; it != m_filesToBeDeleted.end();
             ++it)
        {
            DDebug() << "Removing: " << (*it).first << " in "
                      << (*it).second << endl;
            db->deleteItem( (*it).second, (*it).first );
        }
        db->commitTransaction();
    }
}

void ScanLib::timing(const TQString& text, struct timeval tv1, struct timeval tv2)
{
    DDebug() << "ScanLib: "
              << text + ": "
              << (((tv2.tv_sec-tv1.tv_sec)*1000000 +
                   (tv2.tv_usec-tv1.tv_usec))/1000)
              << " ms" << endl;
}

}  // namespace Digikam