TDE
/
tellico
mirror of https://mirror.git.trinitydesktop.org/gitea/TDE/tellico
0
0
Fork 0
Code Issues Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '2595a15ebe'
${ noResults }
tellico/src/fetch/amazonfetcher.cpp

938 lines
34 KiB
Raw Blame History

/***************************************************************************
copyright : (C) 2004-2006 by Robby Stephenson
email : robby@periapsis.org
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of version 2 of the GNU General Public License as *
* published by the Free Software Foundation; *
* *
***************************************************************************/
#include "amazonfetcher.h"
#include "messagehandler.h"
#include "../translators/xslthandler.h"
#include "../translators/tellicoimporter.h"
#include "../imagefactory.h"
#include "../tellico_kernel.h"
#include "../latin1literal.h"
#include "../collection.h"
#include "../document.h"
#include "../entry.h"
#include "../field.h"
#include "../tellico_utils.h"
#include "../tellico_debug.h"
#include "../isbnvalidator.h"
#include "../gui/combobox.h"
#include <klocale.h>
#include <kio/job.h>
#include <kstandarddirs.h>
#include <kconfig.h>
#include <klineedit.h>
#include <kseparator.h>
#include <kcombobox.h>
#include <kaccelmanager.h>
#include <tqdom.h>
#include <tqlayout.h>
#include <tqlabel.h>
#include <tqwhatsthis.h>
#include <tqcheckbox.h>
#include <tqfile.h>
#include <tqtextcodec.h>
namespace {
static const int AMAZON_RETURNS_PER_REQUEST = 10;
static const int AMAZON_MAX_RETURNS_TOTAL = 20;
static const char* AMAZON_ACCESS_KEY = "0834VQ4S71KYPVSYQD02";
static const char* AMAZON_ASSOC_TOKEN = "tellico-20";
// need to have these in the translation file
static const char* linkText = I18N_NOOP("Amazon Link");
}
using Tellico::Fetch::AmazonFetcher;
// static
const AmazonFetcher::SiteData& AmazonFetcher::siteData(int site_) {
static SiteData dataVector[6] = {
{
i18n("Amazon (US)"),
"http://webservices.amazon.com/onca/xml"
}, {
i18n("Amazon (UK)"),
"http://webservices.amazon.co.uk/onca/xml"
}, {
i18n("Amazon (Germany)"),
"http://webservices.amazon.de/onca/xml"
}, {
i18n("Amazon (Japan)"),
"http://webservices.amazon.co.jp/onca/xml"
}, {
i18n("Amazon (France)"),
"http://webservices.amazon.fr/onca/xml"
}, {
i18n("Amazon (Canada)"),
"http://webservices.amazon.ca/onca/xml"
}
};
return dataVector[site_];
}
AmazonFetcher::AmazonFetcher(Site site_, TQObject* tqparent_, const char* name_)
: Fetcher(tqparent_, name_), m_xsltHandler(0), m_site(site_), m_imageSize(MediumImage),
m_access(TQString::tqfromLatin1(AMAZON_ACCESS_KEY)),
m_assoc(TQString::tqfromLatin1(AMAZON_ASSOC_TOKEN)), m_addLinkField(true), m_limit(AMAZON_MAX_RETURNS_TOTAL),
m_countOffset(0), m_page(1), m_total(-1), m_numResults(0), m_job(0), m_started(false) {
m_name = siteData(site_).title;
}
AmazonFetcher::~AmazonFetcher() {
delete m_xsltHandler;
m_xsltHandler = 0;
}
TQString AmazonFetcher::defaultName() {
return i18n("Amazon.com Web Services");
}
TQString AmazonFetcher::source() const {
return m_name.isEmpty() ? defaultName() : m_name;
}
bool AmazonFetcher::canFetch(int type) const {
return type == Data::Collection::Book
|| type == Data::Collection::ComicBook
|| type == Data::Collection::Bibtex
|| type == Data::Collection::Album
|| type == Data::Collection::Video
|| type == Data::Collection::Game;
}
void AmazonFetcher::readConfigHook(const KConfigGroup& config_) {
TQString s = config_.readEntry("AccessKey");
if(!s.isEmpty()) {
m_access = s;
}
s = config_.readEntry("AssocToken");
if(!s.isEmpty()) {
m_assoc = s;
}
int imageSize = config_.readNumEntry("Image Size", -1);
if(imageSize > -1) {
m_imageSize = static_cast<ImageSize>(imageSize);
}
m_fields = config_.readListEntry("Custom Fields", TQString::tqfromLatin1("keyword"));
}
void AmazonFetcher::search(FetchKey key_, const TQString& value_) {
m_key = key_;
m_value = value_.stripWhiteSpace();
m_started = true;
m_page = 1;
m_total = -1;
m_countOffset = 0;
m_numResults = 0;
doSearch();
}
void AmazonFetcher::continueSearch() {
m_started = true;
m_limit += AMAZON_MAX_RETURNS_TOTAL;
doSearch();
}
void AmazonFetcher::doSearch() {
m_data.truncate(0);
// myDebug() << "AmazonFetcher::doSearch() - value = " << m_value << endl;
// myDebug() << "AmazonFetcher::doSearch() - getting page " << m_page << endl;
const SiteData& data = siteData(m_site);
KURL u = data.url;
u.addQueryItem(TQString::tqfromLatin1("Service"), TQString::tqfromLatin1("AWSECommerceService"));
u.addQueryItem(TQString::tqfromLatin1("AssociateTag"), m_assoc);
u.addQueryItem(TQString::tqfromLatin1("AWSAccessKeyId"), m_access);
u.addQueryItem(TQString::tqfromLatin1("Operation"), TQString::tqfromLatin1("ItemSearch"));
u.addQueryItem(TQString::tqfromLatin1("ResponseGroup"), TQString::tqfromLatin1("Large"));
u.addQueryItem(TQString::tqfromLatin1("ItemPage"), TQString::number(m_page));
u.addQueryItem(TQString::tqfromLatin1("Version"), TQString::tqfromLatin1("2007-10-29"));
const int type = Kernel::self()->collectionType();
switch(type) {
case Data::Collection::Book:
case Data::Collection::ComicBook:
case Data::Collection::Bibtex:
u.addQueryItem(TQString::tqfromLatin1("SearchIndex"), TQString::tqfromLatin1("Books"));
u.addQueryItem(TQString::tqfromLatin1("SortIndex"), TQString::tqfromLatin1("relevancerank"));
break;
case Data::Collection::Album:
u.addQueryItem(TQString::tqfromLatin1("SearchIndex"), TQString::tqfromLatin1("Music"));
break;
case Data::Collection::Video:
u.addQueryItem(TQString::tqfromLatin1("SearchIndex"), TQString::tqfromLatin1("Video"));
u.addQueryItem(TQString::tqfromLatin1("SortIndex"), TQString::tqfromLatin1("relevancerank"));
break;
case Data::Collection::Game:
u.addQueryItem(TQString::tqfromLatin1("SearchIndex"), TQString::tqfromLatin1("VideoGames"));
break;
case Data::Collection::Coin:
case Data::Collection::Stamp:
case Data::Collection::Wine:
case Data::Collection::Base:
case Data::Collection::Card:
default:
message(i18n("%1 does not allow searching for this collection type.").tqarg(source()), MessageHandler::Warning);
stop();
return;
}
// I have not been able to find any documentation about what character set to use
// when URL encoding the search term in the Amazon REST interface. But I do know
// that utf8 DOES NOT WORK. So I'm arbitrarily using iso-8859-1, except for JP.
// Why different for JP? Well, I've not received any bug reports from that direction yet
// TQString value = KURL::decode_string(value_, 106);
// TQString value = TQString::fromLocal8Bit(value_.utf8());
TQString value = m_value;
// a mibenum of 106 is utf-8, 4 is iso-8859-1, 0 means use user's locale,
int mib = m_site == AmazonFetcher::JP ? 106 : 4;
switch(m_key) {
case Title:
u.addQueryItem(TQString::tqfromLatin1("Title"), value, mib);
break;
case Person:
if(type == Data::Collection::Video) {
u.addQueryItem(TQString::tqfromLatin1("Actor"), value, mib);
u.addQueryItem(TQString::tqfromLatin1("Director"), value, mib);
} else if(type == Data::Collection::Album) {
u.addQueryItem(TQString::tqfromLatin1("Artist"), value, mib);
} else if(type == Data::Collection::Game) {
u.addQueryItem(TQString::tqfromLatin1("Manufacturer"), value, mib);
} else { // books and bibtex
TQString s = TQString::tqfromLatin1("author:%1 or publisher:%2").tqarg(value, value);
// u.addQueryItem(TQString::tqfromLatin1("Author"), value, mib);
// u.addQueryItem(TQString::tqfromLatin1("Publisher"), value, mib);
u.addQueryItem(TQString::tqfromLatin1("Power"), s, mib);
}
break;
case ISBN:
{
u.removeQueryItem(TQString::tqfromLatin1("Operation"));
u.addQueryItem(TQString::tqfromLatin1("Operation"), TQString::tqfromLatin1("ItemLookup"));
TQString s = m_value; // not encValue!!!
s.remove('-');
// ISBN only get digits or 'X', and multiple values are connected with "; "
TQStringList isbns = TQStringList::split(TQString::tqfromLatin1("; "), s);
// Amazon isbn13 search is still very flaky, so if possible, we're going to convert
// all of them to isbn10. If we run into a 979 isbn13, then we're forced to do an
// isbn13 search
bool isbn13 = false;
for(TQStringList::Iterator it = isbns.begin(); it != isbns.end(); ) {
if(m_value.startsWith(TQString::tqfromLatin1("979"))) {
if(m_site == JP) { // never works for JP
kdWarning() << "AmazonFetcher:doSearch() - ISBN-13 searching not implemented for Japan" << endl;
isbns.remove(it); // automatically skips to next
continue;
}
isbn13 = true;
break;
}
++it;
}
// if we want isbn10, then convert all
if(!isbn13) {
for(TQStringList::Iterator it = isbns.begin(); it != isbns.end(); ++it) {
if((*it).length() > 12) {
(*it) = ISBNValidator::isbn10(*it);
(*it).remove('-');
}
}
// the default search is by ASIN, which prohibits SearchIndex
u.removeQueryItem(TQString::tqfromLatin1("SearchIndex"));
}
// limit to first 10
while(isbns.size() > 10) {
isbns.pop_back();
}
u.addQueryItem(TQString::tqfromLatin1("ItemId"), isbns.join(TQString::tqfromLatin1(",")));
if(isbn13) {
u.addQueryItem(TQString::tqfromLatin1("IdType"), TQString::tqfromLatin1("EAN"));
}
}
break;
case UPC:
{
u.removeQueryItem(TQString::tqfromLatin1("Operation"));
u.addQueryItem(TQString::tqfromLatin1("Operation"), TQString::tqfromLatin1("ItemLookup"));
// US allows UPC, all others are EAN
if(m_site == US) {
u.addQueryItem(TQString::tqfromLatin1("IdType"), TQString::tqfromLatin1("UPC"));
} else {
u.addQueryItem(TQString::tqfromLatin1("IdType"), TQString::tqfromLatin1("EAN"));
}
TQString s = m_value; // not encValue!!!
s.remove('-');
// limit to first 10
s.tqreplace(TQString::tqfromLatin1("; "), TQString::tqfromLatin1(","));
s = s.section(',', 0, 9);
u.addQueryItem(TQString::tqfromLatin1("ItemId"), s);
}
break;
case Keyword:
u.addQueryItem(TQString::tqfromLatin1("Keywords"), m_value, mib);
break;
case Raw:
{
TQString key = value.section('=', 0, 0).stripWhiteSpace();
TQString str = value.section('=', 1).stripWhiteSpace();
u.addQueryItem(key, str, mib);
}
break;
default:
kdWarning() << "AmazonFetcher::search() - key not recognized: " << m_key << endl;
stop();
return;
}
// myDebug() << "AmazonFetcher::search() - url: " << u.url() << endl;
m_job = KIO::get(u, false, false);
connect(m_job, TQT_SIGNAL(data(KIO::Job*, const TQByteArray&)),
TQT_SLOT(slotData(KIO::Job*, const TQByteArray&)));
connect(m_job, TQT_SIGNAL(result(KIO::Job*)),
TQT_SLOT(slotComplete(KIO::Job*)));
}
void AmazonFetcher::stop() {
if(!m_started) {
return;
}
// myDebug() << "AmazonFetcher::stop()" << endl;
if(m_job) {
m_job->kill();
m_job = 0;
}
m_data.truncate(0);
m_started = false;
emit signalDone(this);
}
void AmazonFetcher::slotData(KIO::Job*, const TQByteArray& data_) {
TQDataStream stream(m_data, IO_WriteOnly | IO_Append);
stream.writeRawBytes(data_.data(), data_.size());
}
void AmazonFetcher::slotComplete(KIO::Job* job_) {
// myDebug() << "AmazonFetcher::slotComplete()" << endl;
// since the fetch is done, don't worry about holding the job pointer
m_job = 0;
if(job_->error()) {
job_->showErrorDialog(Kernel::self()->widget());
stop();
return;
}
if(m_data.isEmpty()) {
myDebug() << "AmazonFetcher::slotComplete() - no data" << endl;
stop();
return;
}
#if 0
kdWarning() << "Remove debug from amazonfetcher.cpp" << endl;
TQFile f(TQString::tqfromLatin1("/tmp/test%1.xml").tqarg(m_page));
if(f.open(IO_WriteOnly)) {
TQTextStream t(&f);
t.setEncoding(TQTextStream::UnicodeUTF8);
t << TQCString(m_data, m_data.size()+1);
}
f.close();
#endif
TQStringList errors;
if(m_total == -1) {
TQDomDocument dom;
if(!dom.setContent(m_data, false)) {
kdWarning() << "AmazonFetcher::slotComplete() - server did not return valid XML." << endl;
stop();
return;
}
// find TotalResults element
// it's in the first level under the root element
//ItemSearchResponse/Items/TotalResults
TQDomNode n = dom.documentElement().namedItem(TQString::tqfromLatin1("Items"))
.namedItem(TQString::tqfromLatin1("TotalResults"));
TQDomElement e = n.toElement();
if(!e.isNull()) {
m_total = e.text().toInt();
}
n = dom.documentElement().namedItem(TQString::tqfromLatin1("Items"))
.namedItem(TQString::tqfromLatin1("Request"))
.namedItem(TQString::tqfromLatin1("Errors"));
e = n.toElement();
if(!e.isNull()) {
TQDomNodeList nodes = e.elementsByTagName(TQString::tqfromLatin1("Error"));
for(uint i = 0; i < nodes.count(); ++i) {
e = nodes.item(i).toElement().namedItem(TQString::tqfromLatin1("Code")).toElement();
if(!e.isNull() && e.text() == Latin1Literal("AWS.ECommerceService.NoExactMatches")) {
// no exact match, not a real error, so skip
continue;
}
// for some reason, Amazon will return an error simply when a valid ISBN is not found
// I really want to ignore that, so check the IsValid element in the Request element
TQDomNode isValidNode = n.tqparentNode().namedItem(TQString::tqfromLatin1("IsValid"));
if(m_key == ISBN && isValidNode.toElement().text().lower() == Latin1Literal("true")) {
continue;
}
e = nodes.item(i).toElement().namedItem(TQString::tqfromLatin1("Message")).toElement();
if(!e.isNull()) {
errors << e.text();
}
}
}
}
if(!m_xsltHandler) {
initXSLTHandler();
if(!m_xsltHandler) { // probably an error somewhere in the stylesheet loading
stop();
return;
}
}
// TQRegExp stripHTML(TQString::tqfromLatin1("<.*>"), true);
// stripHTML.setMinimal(true);
// assume amazon is always utf-8
TQString str = m_xsltHandler->applyStylesheet(TQString::fromUtf8(m_data, m_data.size()));
Import::TellicoImporter imp(str);
Data::CollPtr coll = imp.collection();
if(!coll) {
myDebug() << "AmazonFetcher::slotComplete() - no collection pointer" << endl;
stop();
return;
}
if(!m_addLinkField) {
// remove amazon field if it's not to be added
coll->removeField(TQString::tqfromLatin1("amazon"));
}
Data::EntryVec entries = coll->entries();
if(entries.isEmpty() && !errors.isEmpty()) {
for(TQStringList::ConstIterator it = errors.constBegin(); it != errors.constEnd(); ++it) {
myDebug() << "AmazonFetcher::" << *it << endl;
}
message(errors[0], MessageHandler::Error);
stop();
return;
}
int count = 0;
for(Data::EntryVec::Iterator entry = entries.begin();
m_numResults < m_limit && entry != entries.end();
++entry, ++count) {
if(count < m_countOffset) {
continue;
}
if(!m_started) {
// might get aborted
break;
}
// special case book author
// amazon is really bad about not putting spaces after periods
if(coll->type() == Data::Collection::Book) {
TQRegExp rx(TQString::tqfromLatin1("\\.([^\\s])"));
TQStringList values = entry->fields(TQString::tqfromLatin1("author"), false);
for(TQStringList::Iterator it = values.begin(); it != values.end(); ++it) {
(*it).tqreplace(rx, TQString::tqfromLatin1(". \\1"));
}
entry->setField(TQString::tqfromLatin1("author"), values.join(TQString::tqfromLatin1("; ")));
}
// UK puts the year in the title for some reason
if(m_site == UK && coll->type() == Data::Collection::Video) {
TQRegExp rx(TQString::tqfromLatin1("\\[(\\d{4})\\]"));
TQString t = entry->title();
if(t.tqfind(rx) > -1) {
TQString y = rx.cap(1);
t.remove(rx).simplifyWhiteSpace();
entry->setField(TQString::tqfromLatin1("title"), t);
if(entry->field(TQString::tqfromLatin1("year")).isEmpty()) {
entry->setField(TQString::tqfromLatin1("year"), y);
}
}
}
TQString desc;
switch(coll->type()) {
case Data::Collection::Book:
case Data::Collection::ComicBook:
case Data::Collection::Bibtex:
desc = entry->field(TQString::tqfromLatin1("author"))
+ TQChar('/') + entry->field(TQString::tqfromLatin1("publisher"));
if(!entry->field(TQString::tqfromLatin1("cr_year")).isEmpty()) {
desc += TQChar('/') + entry->field(TQString::tqfromLatin1("cr_year"));
} else if(!entry->field(TQString::tqfromLatin1("pub_year")).isEmpty()){
desc += TQChar('/') + entry->field(TQString::tqfromLatin1("pub_year"));
}
break;
case Data::Collection::Video:
desc = entry->field(TQString::tqfromLatin1("studio"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("director"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("year"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("medium"));
break;
case Data::Collection::Album:
desc = entry->field(TQString::tqfromLatin1("artist"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("label"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("year"));
break;
case Data::Collection::Game:
desc = entry->field(TQString::tqfromLatin1("platform"))
+ TQChar('/')
+ entry->field(TQString::tqfromLatin1("year"));
break;
default:
break;
}
// strip HTML from comments, or plot in movies
// tentatively don't do this, looks like ECS 4 cleaned everything up
/*
if(coll->type() == Data::Collection::Video) {
TQString plot = entry->field(TQString::tqfromLatin1("plot"));
plot.remove(stripHTML);
entry->setField(TQString::tqfromLatin1("plot"), plot);
} else if(coll->type() == Data::Collection::Game) {
TQString desc = entry->field(TQString::tqfromLatin1("description"));
desc.remove(stripHTML);
entry->setField(TQString::tqfromLatin1("description"), desc);
} else {
TQString comments = entry->field(TQString::tqfromLatin1("comments"));
comments.remove(stripHTML);
entry->setField(TQString::tqfromLatin1("comments"), comments);
}
*/
// myDebug() << "AmazonFetcher::slotComplete() - " << entry->title() << endl;
SearchResult* r = new SearchResult(this, entry->title(), desc, entry->field(TQString::tqfromLatin1("isbn")));
m_entries.insert(r->uid, Data::EntryPtr(entry));
emit signalResultFound(r);
++m_numResults;
}
// we might have gotten aborted
if(!m_started) {
return;
}
// are there any additional results to get?
m_hasMoreResults = m_page * AMAZON_RETURNS_PER_REQUEST < m_total;
const int currentTotal = TQMIN(m_total, m_limit);
if(m_page * AMAZON_RETURNS_PER_REQUEST < currentTotal) {
int foundCount = (m_page-1) * AMAZON_RETURNS_PER_REQUEST + coll->entryCount();
message(i18n("Results from %1: %2/%3").tqarg(source()).tqarg(foundCount).tqarg(m_total), MessageHandler::tqStatus);
++m_page;
m_countOffset = 0;
doSearch();
} else if(m_value.tqcontains(';') > 9) {
search(m_key, m_value.section(';', 10));
} else {
m_countOffset = m_entries.count() % AMAZON_RETURNS_PER_REQUEST;
if(m_countOffset == 0) {
++m_page; // need to go to next page
}
stop();
}
}
Tellico::Data::EntryPtr AmazonFetcher::fetchEntry(uint uid_) {
Data::EntryPtr entry = m_entries[uid_];
if(!entry) {
kdWarning() << "AmazonFetcher::fetchEntry() - no entry in dict" << endl;
return 0;
}
TQStringList defaultFields = customFields().keys();
for(TQStringList::Iterator it = defaultFields.begin(); it != defaultFields.end(); ++it) {
if(!m_fields.tqcontains(*it)) {
entry->setField(*it, TQString());
}
}
// do what we can to remove useless keywords
const int type = Kernel::self()->collectionType();
switch(type) {
case Data::Collection::Book:
case Data::Collection::ComicBook:
case Data::Collection::Bibtex:
{
const TQString keywords = TQString::tqfromLatin1("keyword");
TQStringList oldWords = entry->fields(keywords, false);
StringSet words;
for(TQStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) {
// the amazon2tellico stylesheet separates keywords with '/'
TQStringList nodes = TQStringList::split('/', *it);
for(TQStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) {
if(*it2 == Latin1Literal("General") ||
*it2 == Latin1Literal("Subjects") ||
*it2 == Latin1Literal("Par prix") || // french stuff
*it2 == Latin1Literal("Divers") || // french stuff
(*it2).startsWith(TQChar('(')) ||
(*it2).startsWith(TQString::tqfromLatin1("Authors"))) {
continue;
}
words.add(*it2);
}
}
entry->setField(keywords, words.toList().join(TQString::tqfromLatin1("; ")));
}
entry->setField(TQString::tqfromLatin1("comments"), Tellico::decodeHTML(entry->field(TQString::tqfromLatin1("comments"))));
break;
case Data::Collection::Video:
{
const TQString genres = TQString::tqfromLatin1("genre");
TQStringList oldWords = entry->fields(genres, false);
StringSet words;
// only care about genres that have "Genres" in the amazon response
// and take the first word after that
for(TQStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) {
if((*it).tqfind(TQString::tqfromLatin1("Genres")) == -1) {
continue;
}
// the amazon2tellico stylesheet separates words with '/'
TQStringList nodes = TQStringList::split('/', *it);
for(TQStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) {
if(*it2 != Latin1Literal("Genres")) {
continue;
}
++it2;
if(it2 != nodes.end() && *it2 != Latin1Literal("General")) {
words.add(*it2);
}
break; // we're done
}
}
entry->setField(genres, words.toList().join(TQString::tqfromLatin1("; ")));
// language tracks get duplicated, too
TQStringList langs = entry->fields(TQString::tqfromLatin1("language"), false);
words.clear();
for(TQStringList::ConstIterator it = langs.begin(); it != langs.end(); ++it) {
words.add(*it);
}
entry->setField(TQString::tqfromLatin1("language"), words.toList().join(TQString::tqfromLatin1("; ")));
}
entry->setField(TQString::tqfromLatin1("plot"), Tellico::decodeHTML(entry->field(TQString::tqfromLatin1("plot"))));
break;
case Data::Collection::Album:
{
const TQString genres = TQString::tqfromLatin1("genre");
TQStringList oldWords = entry->fields(genres, false);
StringSet words;
// only care about genres that have "Styles" in the amazon response
// and take the first word after that
for(TQStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) {
if((*it).tqfind(TQString::tqfromLatin1("Styles")) == -1) {
continue;
}
// the amazon2tellico stylesheet separates words with '/'
TQStringList nodes = TQStringList::split('/', *it);
bool isStyle = false;
for(TQStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) {
if(!isStyle) {
if(*it2 == Latin1Literal("Styles")) {
isStyle = true;
}
continue;
}
if(*it2 != Latin1Literal("General")) {
words.add(*it2);
}
}
}
entry->setField(genres, words.toList().join(TQString::tqfromLatin1("; ")));
}
entry->setField(TQString::tqfromLatin1("comments"), Tellico::decodeHTML(entry->field(TQString::tqfromLatin1("comments"))));
break;
case Data::Collection::Game:
entry->setField(TQString::tqfromLatin1("description"), Tellico::decodeHTML(entry->field(TQString::tqfromLatin1("description"))));
break;
}
// clean up the title
parseTitle(entry, type);
// also sometimes table fields have rows but no values
Data::FieldVec fields = entry->collection()->fields();
TQRegExp blank(TQString::tqfromLatin1("[\\s:;]+")); // only white space, column separators and row separators
for(Data::FieldVec::Iterator fIt = fields.begin(); fIt != fields.end(); ++fIt) {
if(fIt->type() != Data::Field::Table) {
continue;
}
if(blank.exactMatch(entry->field(fIt))) {
entry->setField(fIt, TQString());
}
}
KURL imageURL;
switch(m_imageSize) {
case SmallImage:
imageURL = entry->field(TQString::tqfromLatin1("small-image"));
break;
case MediumImage:
imageURL = entry->field(TQString::tqfromLatin1("medium-image"));
break;
case LargeImage:
imageURL = entry->field(TQString::tqfromLatin1("large-image"));
break;
case NoImage:
default:
break;
}
// myDebug() << "AmazonFetcher::fetchEntry() - grabbing " << imageURL.prettyURL() << endl;
if(!imageURL.isEmpty()) {
TQString id = ImageFactory::addImage(imageURL, true);
// FIXME: need to add cover image field to bibtex collection
if(id.isEmpty()) {
message(i18n("The cover image could not be loaded."), MessageHandler::Warning);
} else { // amazon serves up 1x1 gifs occasionally, but that's caught in the image constructor
// all relevant collection types have cover fields
entry->setField(TQString::tqfromLatin1("cover"), id);
}
}
// don't want to show image urls in the fetch dialog
entry->setField(TQString::tqfromLatin1("small-image"), TQString());
entry->setField(TQString::tqfromLatin1("medium-image"), TQString());
entry->setField(TQString::tqfromLatin1("large-image"), TQString());
return entry;
}
void AmazonFetcher::initXSLTHandler() {
TQString xsltfile = locate("appdata", TQString::tqfromLatin1("amazon2tellico.xsl"));
if(xsltfile.isEmpty()) {
kdWarning() << "AmazonFetcher::initXSLTHandler() - can not locate amazon2tellico.xsl." << endl;
return;
}
KURL u;
u.setPath(xsltfile);
delete m_xsltHandler;
m_xsltHandler = new XSLTHandler(u);
if(!m_xsltHandler->isValid()) {
kdWarning() << "AmazonFetcher::initXSLTHandler() - error in amazon2tellico.xsl." << endl;
delete m_xsltHandler;
m_xsltHandler = 0;
return;
}
}
void AmazonFetcher::updateEntry(Data::EntryPtr entry_) {
// myDebug() << "AmazonFetcher::updateEntry()" << endl;
int type = entry_->collection()->type();
if(type == Data::Collection::Book || type == Data::Collection::ComicBook || type == Data::Collection::Bibtex) {
TQString isbn = entry_->field(TQString::tqfromLatin1("isbn"));
if(!isbn.isEmpty()) {
m_limit = 5; // no need for more
search(Fetch::ISBN, isbn);
return;
}
} else if(type == Data::Collection::Album) {
TQString a = entry_->field(TQString::tqfromLatin1("artist"));
if(!a.isEmpty()) {
search(Fetch::Person, a);
return;
}
}
// optimistically try searching for title and rely on Collection::sameEntry() to figure things out
TQString t = entry_->field(TQString::tqfromLatin1("title"));
if(!t.isEmpty()) {
search(Fetch::Title, t);
return;
}
myDebug() << "AmazonFetcher::updateEntry() - insufficient info to search" << endl;
emit signalDone(this); // always need to emit this if not continuing with the search
}
void AmazonFetcher::parseTitle(Data::EntryPtr entry, int collType) {
Q_UNUSED(collType);
// assume that everything in brackets or tqparentheses is extra
TQRegExp rx(TQString::tqfromLatin1("[\\(\\[](.*)[\\)\\]]"));
rx.setMinimal(true);
TQString title = entry->field(TQString::tqfromLatin1("title"));
int pos = rx.search(title);
while(pos > -1) {
if(parseTitleToken(entry, rx.cap(1))) {
title.remove(pos, rx.matchedLength());
--pos; // search again there
}
pos = rx.search(title, pos+1);
}
entry->setField(TQString::tqfromLatin1("title"), title.stripWhiteSpace());
}
bool AmazonFetcher::parseTitleToken(Data::EntryPtr entry, const TQString& token) {
// if res = true, then the token gets removed from the title
bool res = false;
if(token.tqfind(TQString::tqfromLatin1("widescreen"), 0, false /* case-insensitive*/) > -1 ||
token.tqfind(i18n("Widescreen"), 0, false) > -1) {
entry->setField(TQString::tqfromLatin1("widescreen"), TQString::tqfromLatin1("true"));
// res = true; leave it in the title
} else if(token.tqfind(TQString::tqfromLatin1("full screen"), 0, false) > -1) {
// skip, but go ahead and remove from title
res = true;
}
if(token.tqfind(TQString::tqfromLatin1("blu-ray"), 0, false) > -1) {
entry->setField(TQString::tqfromLatin1("medium"), i18n("Blu-ray"));
res = true;
} else if(token.tqfind(TQString::tqfromLatin1("hd dvd"), 0, false) > -1) {
entry->setField(TQString::tqfromLatin1("medium"), i18n("HD DVD"));
res = true;
}
if(token.tqfind(TQString::tqfromLatin1("director's cut"), 0, false) > -1 ||
token.tqfind(i18n("Director's Cut"), 0, false) > -1) {
entry->setField(TQString::tqfromLatin1("directors-cut"), TQString::tqfromLatin1("true"));
// res = true; leave it in the title
}
return res;
}
Tellico::Fetch::ConfigWidget* AmazonFetcher::configWidget(TQWidget* tqparent_) const {
return new AmazonFetcher::ConfigWidget(tqparent_, this);
}
AmazonFetcher::ConfigWidget::ConfigWidget(TQWidget* tqparent_, const AmazonFetcher* fetcher_/*=0*/)
: Fetch::ConfigWidget(tqparent_) {
TQGridLayout* l = new TQGridLayout(optionsWidget(), 4, 2);
l->setSpacing(4);
l->setColStretch(1, 10);
int row = -1;
TQLabel* label = new TQLabel(i18n("Co&untry: "), optionsWidget());
l->addWidget(label, ++row, 0);
m_siteCombo = new GUI::ComboBox(optionsWidget());
m_siteCombo->insertItem(i18n("United States"), US);
m_siteCombo->insertItem(i18n("United Kingdom"), UK);
m_siteCombo->insertItem(i18n("Germany"), DE);
m_siteCombo->insertItem(i18n("Japan"), JP);
m_siteCombo->insertItem(i18n("France"), FR);
m_siteCombo->insertItem(i18n("Canada"), CA);
connect(m_siteCombo, TQT_SIGNAL(activated(int)), TQT_SLOT(slotSetModified()));
connect(m_siteCombo, TQT_SIGNAL(activated(int)), TQT_SLOT(slotSiteChanged()));
l->addWidget(m_siteCombo, row, 1);
TQString w = i18n("Amazon.com provides data from several different localized sites. Choose the one "
"you wish to use for this data source.");
TQWhatsThis::add(label, w);
TQWhatsThis::add(m_siteCombo, w);
label->setBuddy(m_siteCombo);
label = new TQLabel(i18n("&Image size: "), optionsWidget());
l->addWidget(label, ++row, 0);
m_imageCombo = new GUI::ComboBox(optionsWidget());
m_imageCombo->insertItem(i18n("Small Image"), SmallImage);
m_imageCombo->insertItem(i18n("Medium Image"), MediumImage);
m_imageCombo->insertItem(i18n("Large Image"), LargeImage);
m_imageCombo->insertItem(i18n("No Image"), NoImage);
connect(m_imageCombo, TQT_SIGNAL(activated(int)), TQT_SLOT(slotSetModified()));
l->addWidget(m_imageCombo, row, 1);
w = i18n("The cover image may be downloaded as well. However, too many large images in the "
"collection may degrade performance.");
TQWhatsThis::add(label, w);
TQWhatsThis::add(m_imageCombo, w);
label->setBuddy(m_imageCombo);
label = new TQLabel(i18n("&Associate's ID: "), optionsWidget());
l->addWidget(label, ++row, 0);
m_assocEdit = new KLineEdit(optionsWidget());
connect(m_assocEdit, TQT_SIGNAL(textChanged(const TQString&)), TQT_SLOT(slotSetModified()));
l->addWidget(m_assocEdit, row, 1);
w = i18n("The associate's id identifies the person accessing the Amazon.com Web Services, and is included "
"in any links to the Amazon.com site.");
TQWhatsThis::add(label, w);
TQWhatsThis::add(m_assocEdit, w);
label->setBuddy(m_assocEdit);
l->setRowStretch(++row, 10);
if(fetcher_) {
m_siteCombo->setCurrentData(fetcher_->m_site);
m_assocEdit->setText(fetcher_->m_assoc);
m_imageCombo->setCurrentData(fetcher_->m_imageSize);
} else { // defaults
m_assocEdit->setText(TQString::tqfromLatin1(AMAZON_ASSOC_TOKEN));
m_imageCombo->setCurrentData(MediumImage);
}
addFieldsWidget(AmazonFetcher::customFields(), fetcher_ ? fetcher_->m_fields : TQStringList());
KAcceleratorManager::manage(optionsWidget());
}
void AmazonFetcher::ConfigWidget::saveConfig(KConfigGroup& config_) {
int n = m_siteCombo->currentData().toInt();
config_.writeEntry("Site", n);
TQString s = m_assocEdit->text().stripWhiteSpace();
if(!s.isEmpty()) {
config_.writeEntry("AssocToken", s);
}
n = m_imageCombo->currentData().toInt();
config_.writeEntry("Image Size", n);
saveFieldsConfig(config_);
slotSetModified(false);
}
TQString AmazonFetcher::ConfigWidget::preferredName() const {
return AmazonFetcher::siteData(m_siteCombo->currentData().toInt()).title;
}
void AmazonFetcher::ConfigWidget::slotSiteChanged() {
emit signalName(preferredName());
}
//static
Tellico::StringMap AmazonFetcher::customFields() {
StringMap map;
map[TQString::tqfromLatin1("keyword")] = i18n("Keywords");
return map;
}
#include "amazonfetcher.moc"
View Git Blame Copy Permalink