tellico/src/fetch/entrezfetcher.cpp

/***************************************************************************
    copyright            : (C) 2005-2006 by Robby Stephenson
    email                : robby@periapsis.org
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of version 2 of the GNU General Public License as  *
 *   published by the Free Software Foundation;                            *
 *                                                                         *
 ***************************************************************************/

#include "entrezfetcher.h"
#include "../tellico_kernel.h"
#include "../latin1literal.h"
#include "../collection.h"
#include "../entry.h"
#include "../filehandler.h"
#include "../translators/xslthandler.h"
#include "../translators/tellicoimporter.h"
#include "../tellico_debug.h"

#include <klocale.h>
#include <kconfig.h>
#include <kstandarddirs.h>
#include <kio/job.h>

#include <tqdom.h>
#include <tqlabel.h>
#include <tqlayout.h>
#include <tqfile.h>

//#define ENTREZ_TEST

namespace {
  static const int ENTREZ_MAX_RETURNS_TOTAL = 25;
  static const char* ENTREZ_BASE_URL = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/";
  static const char* ENTREZ_SEARCH_CGI = "esearch.fcgi";
  static const char* ENTREZ_SUMMARY_CGI = "esummary.fcgi";
  static const char* ENTREZ_FETCH_CGI = "efetch.fcgi";
  static const char* ENTREZ_LINK_CGI = "elink.fcgi";
  static const char* ENTREZ_DEFAULT_DATABASE = "pubmed";
}

using Tellico::Fetch::EntrezFetcher;

EntrezFetcher::EntrezFetcher(TQObject* parent_, const char* name_) : Fetcher(parent_, name_), m_xsltHandler(0),
    m_step(Begin), m_started(false) {
}

EntrezFetcher::~EntrezFetcher() {
}

TQString EntrezFetcher::defaultName() {
  return i18n("Entrez Database");
}

TQString EntrezFetcher::source() const {
  return m_name.isEmpty() ? defaultName() : m_name;
}

bool EntrezFetcher::canFetch(int type) const {
  return type == Data::Collection::Bibtex;
}

void EntrezFetcher::readConfigHook(const KConfigGroup& config_) {
  TQString s = config_.readEntry("Database", TQString::tqfromLatin1(ENTREZ_DEFAULT_DATABASE)); // default to pubmed
  if(!s.isEmpty()) {
    m_dbname = s;
  }
  m_fields = config_.readListEntry("Custom Fields");
}

void EntrezFetcher::search(FetchKey key_, const TQString& value_) {
  m_started = true;
  m_start = 1;
  m_total = -1;

// only search if current collection is a bibliography
  if(!canFetch(Kernel::self()->collectionType())) {
    myDebug() << "EntrezFetcher::search() - collection type mismatch, stopping" << endl;
    stop();
    return;
  }
  if(m_dbname.isEmpty()) {
    m_dbname = TQString::tqfromLatin1(ENTREZ_DEFAULT_DATABASE);
  }

#ifdef ENTREZ_TEST
  KURL u = KURL::fromPathOrURL(TQString::tqfromLatin1("/home/robby/esearch.xml"));
#else
  KURL u(TQString::tqfromLatin1(ENTREZ_BASE_URL));
  u.addPath(TQString::tqfromLatin1(ENTREZ_SEARCH_CGI));
  u.addQueryItem(TQString::tqfromLatin1("tool"),       TQString::tqfromLatin1("Tellico"));
  u.addQueryItem(TQString::tqfromLatin1("retmode"),    TQString::tqfromLatin1("xml"));
  u.addQueryItem(TQString::tqfromLatin1("usehistory"), TQString::tqfromLatin1("y"));
  u.addQueryItem(TQString::tqfromLatin1("retmax"),     TQString::tqfromLatin1("1")); // we're just getting the count
  u.addQueryItem(TQString::tqfromLatin1("db"),         m_dbname);
  u.addQueryItem(TQString::tqfromLatin1("term"),       value_);
  switch(key_) {
    case Title:
      u.addQueryItem(TQString::tqfromLatin1("field"), TQString::tqfromLatin1("titl"));
      break;

    case Person:
      u.addQueryItem(TQString::tqfromLatin1("field"), TQString::tqfromLatin1("auth"));
      break;

    case Keyword:
      // for Tellico Keyword searches basically mean search for any field matching
//      u.addQueryItem(TQString::tqfromLatin1("field"), TQString::tqfromLatin1("word"));
      break;

    case PubmedID:
      u.addQueryItem(TQString::tqfromLatin1("field"), TQString::tqfromLatin1("pmid"));
      break;

    case DOI:
    case Raw:
      u.setQuery(u.query() + '&' + value_);
      break;

    default:
      kdWarning() << "EntrezFetcher::search() - FetchKey not supported" << endl;
      stop();
      return;
  }
#endif

  m_step = Search;
//  myLog() << "EntrezFetcher::doSearch() - url: " << u.url() << endl;
  m_job = KIO::get(u, false, false);
  connect(m_job, TQT_SIGNAL(data(KIO::Job*, const TQByteArray&)),
          TQT_SLOT(slotData(KIO::Job*, const TQByteArray&)));
  connect(m_job, TQT_SIGNAL(result(KIO::Job*)),
          TQT_SLOT(slotComplete(KIO::Job*)));
}

void EntrezFetcher::continueSearch() {
  m_started = true;
  doSummary();
}

void EntrezFetcher::stop() {
  if(!m_started) {
    return;
  }
  if(m_job) {
    m_job->kill();
    m_job = 0;
  }
  m_data.truncate(0);
  m_started = false;
  m_step = Begin;
  emit signalDone(this);
}

void EntrezFetcher::slotData(KIO::Job*, const TQByteArray& data_) {
  TQDataStream stream(m_data, IO_WriteOnly | IO_Append);
  stream.writeRawBytes(data_.data(), data_.size());
}

void EntrezFetcher::slotComplete(KIO::Job* job_) {
  // since the fetch is done, don't worry about holding the job pointer
  m_job = 0;

  if(job_->error()) {
    job_->showErrorDialog(Kernel::self()->widget());
    stop();
    return;
  }

  if(m_data.isEmpty()) {
    myDebug() << "EntrezFetcher::slotComplete() - no data" << endl;
    stop();
    return;
  }

#if 0
  kdWarning() << "Remove debug from entrezfetcher.cpp: " << __LINE__ << endl;
  TQFile f(TQString::tqfromLatin1("/tmp/test.xml"));
  if(f.open(IO_WriteOnly)) {
    TQTextStream t(&f);
    t.setEncoding(TQTextStream::UnicodeUTF8);
    t << TQCString(m_data, m_data.size()+1);
  }
  f.close();
#endif

  switch(m_step) {
    case Search:
      searchResults();
      break;
    case Summary:
      summaryResults();
      break;
    case Begin:
    case Fetch:
    default:
      myLog() << "EntrezFetcher::slotComplete() - wrong step = " << m_step << endl;
      stop();
      break;
  }
}

void EntrezFetcher::searchResults() {
  TQDomDocument dom;
  if(!dom.setContent(m_data, false)) {
    kdWarning() << "EntrezFetcher::searchResults() - server did not return valid XML." << endl;
    stop();
    return;
  }
  // find Count, QueryKey, and WebEnv elements
  int count = 0;
  for(TQDomNode n = dom.documentElement().firstChild(); !n.isNull(); n = n.nextSibling()) {
    TQDomElement e = n.toElement();
    if(e.isNull()) {
      continue;
    }
    if(e.tagName() == Latin1Literal("Count")) {
      m_total = e.text().toInt();
      ++count;
    } else if(e.tagName() == Latin1Literal("QueryKey")) {
      m_queryKey = e.text();
      ++count;
    } else if(e.tagName() == Latin1Literal("WebEnv")) {
      m_webEnv = e.text();
      ++count;
    }
    if(count >= 3) {
      break; // found them all
    }
  }

  m_data.truncate(0);
  doSummary();
}

void EntrezFetcher::doSummary() {
#ifdef ENTREZ_TEST
  KURL u = KURL::fromPathOrURL(TQString::tqfromLatin1("/home/robby/esummary.xml"));
#else
  KURL u(TQString::tqfromLatin1(ENTREZ_BASE_URL));
  u.addPath(TQString::tqfromLatin1(ENTREZ_SUMMARY_CGI));
  u.addQueryItem(TQString::tqfromLatin1("tool"),       TQString::tqfromLatin1("Tellico"));
  u.addQueryItem(TQString::tqfromLatin1("retmode"),    TQString::tqfromLatin1("xml"));
  u.addQueryItem(TQString::tqfromLatin1("retstart"),   TQString::number(m_start));
  u.addQueryItem(TQString::tqfromLatin1("retmax"),     TQString::number(TQMIN(m_total-m_start-1, ENTREZ_MAX_RETURNS_TOTAL)));
  u.addQueryItem(TQString::tqfromLatin1("usehistory"), TQString::tqfromLatin1("y"));
  u.addQueryItem(TQString::tqfromLatin1("db"),         m_dbname);
  u.addQueryItem(TQString::tqfromLatin1("query_key"),  m_queryKey);
  u.addQueryItem(TQString::tqfromLatin1("WebEnv"),     m_webEnv);
#endif

  m_step = Summary;
//  myLog() << "EntrezFetcher::searchResults() - url: " << u.url() << endl;
  m_job = KIO::get(u, false, false);
  connect(m_job, TQT_SIGNAL(data(KIO::Job*, const TQByteArray&)),
          TQT_SLOT(slotData(KIO::Job*, const TQByteArray&)));
  connect(m_job, TQT_SIGNAL(result(KIO::Job*)),
          TQT_SLOT(slotComplete(KIO::Job*)));
}

void EntrezFetcher::summaryResults() {
  TQDomDocument dom;
  if(!dom.setContent(m_data, false)) {
    kdWarning() << "EntrezFetcher::summaryResults() - server did not return valid XML." << endl;
    stop();
    return;
  }
  // top child is eSummaryResult
  // all tqchildren are DocSum
  for(TQDomNode n = dom.documentElement().firstChild(); !n.isNull(); n = n.nextSibling()) {
    TQDomElement e = n.toElement();
    if(e.isNull() || e.tagName() != Latin1Literal("DocSum")) {
      continue;
    }
    TQDomNodeList nodes = e.elementsByTagName(TQString::tqfromLatin1("Id"));
    if(nodes.count() == 0) {
      myDebug() << "EntrezFetcher::summaryResults() - no Id elements" << endl;
      continue;
    }
    int id = nodes.item(0).toElement().text().toInt();
    TQString title, pubdate, authors;
    nodes = e.elementsByTagName(TQString::tqfromLatin1("Item"));
    for(uint j = 0; j < nodes.count(); ++j) {
      if(nodes.item(j).toElement().attribute(TQString::tqfromLatin1("Name")) == Latin1Literal("Title")) {
        title = nodes.item(j).toElement().text();
      } else if(nodes.item(j).toElement().attribute(TQString::tqfromLatin1("Name")) == Latin1Literal("PubDate")) {
        pubdate = nodes.item(j).toElement().text();
      } else if(nodes.item(j).toElement().attribute(TQString::tqfromLatin1("Name")) == Latin1Literal("AuthorList")) {
        TQStringList list;
        for(TQDomNode aNode = nodes.item(j).firstChild(); !aNode.isNull(); aNode = aNode.nextSibling()) {
          // lazy, assume all tqchildren Items are authors
          if(aNode.nodeName() == Latin1Literal("Item")) {
            list << aNode.toElement().text();
          }
        }
        authors = list.join(TQString::tqfromLatin1("; "));
      }
      if(!title.isEmpty() && !pubdate.isEmpty() && !authors.isEmpty()) {
        break; // done now
      }
    }
    SearchResult* r = new SearchResult(this, title, pubdate + '/' + authors, TQString());
    m_matches.insert(r->uid, id);
    emit signalResultFound(r);
  }
  m_start = m_matches.count() + 1;
  m_hasMoreResults = m_start <= m_total;
  stop(); // done searching
}

Tellico::Data::EntryPtr EntrezFetcher::fetchEntry(uint uid_) {
  // if we already grabbed this one, then just pull it out of the dict
  Data::EntryPtr entry = m_entries[uid_];
  if(entry) {
    return entry;
  }

  if(!m_matches.contains(uid_)) {
    return 0;
  }

  if(!m_xsltHandler) {
    initXSLTHandler();
    if(!m_xsltHandler) { // probably an error somewhere in the stylesheet loading
      stop();
      return 0;
    }
  }

  int id = m_matches[uid_];
#ifdef ENTREZ_TEST
  KURL u = KURL::fromPathOrURL(TQString::tqfromLatin1("/home/robby/pubmed.xml"));
#else
  KURL u(TQString::tqfromLatin1(ENTREZ_BASE_URL));
  u.addPath(TQString::tqfromLatin1(ENTREZ_FETCH_CGI));
  u.addQueryItem(TQString::tqfromLatin1("tool"),       TQString::tqfromLatin1("Tellico"));
  u.addQueryItem(TQString::tqfromLatin1("retmode"),    TQString::tqfromLatin1("xml"));
  u.addQueryItem(TQString::tqfromLatin1("rettype"),    TQString::tqfromLatin1("abstract"));
  u.addQueryItem(TQString::tqfromLatin1("db"),         m_dbname);
  u.addQueryItem(TQString::tqfromLatin1("id"),         TQString::number(id));
#endif
  // now it's sychronous, and we know that it's utf8
  TQString xmlOutput = FileHandler::readTextFile(u, false /*quiet*/, true /*utf8*/);
  if(xmlOutput.isEmpty()) {
    kdWarning() << "EntrezFetcher::fetchEntry() - unable to download " << u << endl;
    return 0;
  }
#if 0
  kdWarning() << "EntrezFetcher::fetchEntry() - turn me off!" << endl;
  TQFile f1(TQString::tqfromLatin1("/tmp/test-entry.xml"));
  if(f1.open(IO_WriteOnly)) {
    TQTextStream t(&f1);
    t.setEncoding(TQTextStream::UnicodeUTF8);
    t << xmlOutput;
  }
  f1.close();
#endif
  TQString str = m_xsltHandler->applyStylesheet(xmlOutput);
  Import::TellicoImporter imp(str);
  Data::CollPtr coll = imp.collection();
  if(!coll) {
    kdWarning() << "EntrezFetcher::fetchEntry() - invalid collection" << endl;
    return 0;
  }
  if(coll->entryCount() == 0) {
    myDebug() << "EntrezFetcher::fetchEntry() - no entries in collection" << endl;
    return 0;
  } else if(coll->entryCount() > 1) {
    myDebug() << "EntrezFetcher::fetchEntry() - collection has multiple entries, taking first one" << endl;
  }

  Data::EntryPtr e = coll->entries().front();

  // try to get a link, but only if necessary
  if(m_fields.contains(TQString::tqfromLatin1("url"))) {
    KURL link(TQString::tqfromLatin1(ENTREZ_BASE_URL));
    link.addPath(TQString::tqfromLatin1(ENTREZ_LINK_CGI));
    link.addQueryItem(TQString::tqfromLatin1("tool"),   TQString::tqfromLatin1("Tellico"));
    link.addQueryItem(TQString::tqfromLatin1("cmd"),    TQString::tqfromLatin1("llinks"));
    link.addQueryItem(TQString::tqfromLatin1("db"),     m_dbname);
    link.addQueryItem(TQString::tqfromLatin1("dbfrom"), m_dbname);
    link.addQueryItem(TQString::tqfromLatin1("id"),     TQString::number(id));

    TQDomDocument linkDom = FileHandler::readXMLFile(link, false /* namespace */, true /* quiet */);
    // need eLinkResult/LinkSet/IdUrlList/IdUrlSet/ObjUrl/Url
    TQDomNode linkNode = linkDom.namedItem(TQString::tqfromLatin1("eLinkResult"))
                               .namedItem(TQString::tqfromLatin1("LinkSet"))
                               .namedItem(TQString::tqfromLatin1("IdUrlList"))
                               .namedItem(TQString::tqfromLatin1("IdUrlSet"))
                               .namedItem(TQString::tqfromLatin1("ObjUrl"))
                               .namedItem(TQString::tqfromLatin1("Url"));
    if(!linkNode.isNull()) {
      TQString u = linkNode.toElement().text();
//      myDebug() << u << endl;
      if(!u.isEmpty()) {
        if(!coll->hasField(TQString::tqfromLatin1("url"))) {
          Data::FieldPtr field = new Data::Field(TQString::tqfromLatin1("url"), i18n("URL"), Data::Field::URL);
          field->setCategory(i18n("Miscellaneous"));
          coll->addField(field);
        }
        e->setField(TQString::tqfromLatin1("url"), u);
      }
    }
  }

  const StringMap customFields = EntrezFetcher::customFields();
  for(StringMap::ConstIterator it = customFields.begin(); it != customFields.end(); ++it) {
    if(!m_fields.contains(it.key())) {
      coll->removeField(it.key());
    }
  }

  m_entries.insert(uid_, e);
  return e;
}

void EntrezFetcher::initXSLTHandler() {
  TQString xsltfile = locate("appdata", TQString::tqfromLatin1("pubmed2tellico.xsl"));
  if(xsltfile.isEmpty()) {
    kdWarning() << "EntrezFetcher::initXSLTHandler() - can not locate pubmed2tellico.xsl." << endl;
    return;
  }

  KURL u;
  u.setPath(xsltfile);

  if(!m_xsltHandler) {
    m_xsltHandler = new XSLTHandler(u);
  }
  if(!m_xsltHandler->isValid()) {
    kdWarning() << "EntrezFetcher::initXSLTHandler() - error in pubmed2tellico.xsl." << endl;
    delete m_xsltHandler;
    m_xsltHandler = 0;
    return;
  }
}

void EntrezFetcher::updateEntry(Data::EntryPtr entry_) {
//  myDebug() << "EntrezFetcher::updateEntry()" << endl;
  TQString s = entry_->field(TQString::tqfromLatin1("pmid"));
  if(!s.isEmpty()) {
    search(PubmedID, s);
    return;
  }

  s = entry_->field(TQString::tqfromLatin1("doi"));
  if(!s.isEmpty()) {
    search(DOI, s);
    return;
  }

  s = entry_->field(TQString::tqfromLatin1("title"));
  if(!s.isEmpty()) {
    search(Title, s);
    return;
  }

  myDebug() << "EntrezFetcher::updateEntry() - insufficient info to search" << endl;
  emit signalDone(this); // always need to emit this if not continuing with the search
}

Tellico::Fetch::ConfigWidget* EntrezFetcher::configWidget(TQWidget* parent_) const {
  return new EntrezFetcher::ConfigWidget(parent_, this);
}

EntrezFetcher::ConfigWidget::ConfigWidget(TQWidget* parent_, const EntrezFetcher* fetcher_/*=0*/)
    : Fetch::ConfigWidget(parent_) {
  TQVBoxLayout* l = new TQVBoxLayout(optionsWidget());
  l->addWidget(new TQLabel(i18n("This source has no options."), optionsWidget()));
  l->addStretch();

  // now add additional fields widget
  addFieldsWidget(EntrezFetcher::customFields(), fetcher_ ? fetcher_->m_fields : TQStringList());
}

void EntrezFetcher::ConfigWidget::saveConfig(KConfigGroup& config_) {
  saveFieldsConfig(config_);
  slotSetModified(false);
}

TQString EntrezFetcher::ConfigWidget::preferredName() const {
  return EntrezFetcher::defaultName();
}

//static
Tellico::StringMap EntrezFetcher::customFields() {
  StringMap map;
  map[TQString::tqfromLatin1("institution")] = i18n("Institution");
  map[TQString::tqfromLatin1("abstract")]    = i18n("Abstract");
  map[TQString::tqfromLatin1("url")]         = i18n("URL");
  return map;
}

#include "entrezfetcher.moc"