TDE
/
extra-dependencies
mirror of https://mirror.git.trinitydesktop.org/gitea/TDE/extra-dependencies.git
0
0
Fork 0
Code Issues Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '3b0c3b8206'
${ noResults }
extra-dependencies/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.cc

1085 lines
27 KiB
Raw Normal View History Unescape

DEB htdig: Added to repository. Signed-off-by: Slávek Banko <slavek.banko@axis.cz>
3 years ago
//
// HtHTTP.cc
//
// HtHTTP: Interface classes for HTTP messaging
//
// Including:
// - Generic class
// - Response message class
//
// Part of the ht://Dig package <http://www.htdig.org/>
// Copyright (c) 1995-2004 The ht://Dig Group
// For copyright details, see the file COPYING in your distribution
// or the GNU Library General Public License (LGPL) version 2 or later
// <http://www.gnu.org/copyleft/lgpl.html>
//
// $Id: HtHTTP.cc,v 1.27 2004/05/28 13:15:23 lha Exp $
//
#ifdef HAVE_CONFIG_H
#include "htconfig.h"
#endif /* HAVE_CONFIG_H */
#include "lib.h"
#include "Transport.h"
#include "HtHTTP.h"
#include <signal.h>
#include <sys/types.h>
#include <ctype.h>
#include <stdio.h> // for sscanf
// for setw()
#ifdef HAVE_STD
#include <iomanip>
#ifdef HAVE_NAMESPACES
using namespace std;
#endif
#else
#include <iomanip.h>
#endif /* HAVE_STD */
#if 1
typedef void (*SIGNAL_HANDLER) (...);
#else
typedef SIG_PF SIGNAL_HANDLER;
#endif
// User Agent
String HtHTTP::_user_agent = 0;
// Stats information
int HtHTTP::_tot_seconds = 0;
int HtHTTP::_tot_requests = 0;
int HtHTTP::_tot_bytes = 0;
// flag that manage the option of 'HEAD' before 'GET'
bool HtHTTP::_head_before_get = true;
// Handler of the CanParse function
int (* HtHTTP::CanBeParsed) (char *) = 0;
// Cookies jar
HtCookieJar *HtHTTP::_cookie_jar = 0; // Set to 0 by default
///////
// HtHTTP_Response class
//
// Response message sent by the remote HTTP server
///////
// Construction
HtHTTP_Response::HtHTTP_Response()
: _version(0),
_transfer_encoding(0),
_server(0),
_hdrconnection(0),
_content_language(0)
{
}
// Destruction
HtHTTP_Response::~HtHTTP_Response()
{
}
void HtHTTP_Response::Reset()
{
// Call the base class method in order to reset
// the base class attributes
Transport_Response::Reset();
// Initialize the version, transfer-encoding, location and server strings
_version.trunc();
_transfer_encoding.trunc();
_hdrconnection.trunc();
_server.trunc();
_content_language.trunc();
}
///////
// HtHTTP generic class
//
//
///////
// Construction
HtHTTP::HtHTTP(Connection& connection)
: Transport(&connection),
_Method(Method_GET), // Default Method Request
_bytes_read(0),
_accept_language(0),
_persistent_connection_allowed(true),
_persistent_connection_possible(false),
_send_cookies(true)
{
}
// Destruction
HtHTTP::~HtHTTP()
{
}
///////
// Manages the requesting process
///////
Transport::DocStatus HtHTTP::Request()
{
DocStatus result = Document_ok;
///////
// We make a double request (HEAD and, maybe, GET)
// Depending on the
///////
if (HeadBeforeGet() && // Option value to true
_Method == Method_GET) // Initial request method is GET
{
if (debug>3)
cout << " Making a HEAD call before the GET" << endl;
_Method = Method_HEAD;
result = HTTPRequest();
_Method = Method_GET;
}
if (result == Document_ok)
result = HTTPRequest();
if(result == Document_no_header
&& isPersistentConnectionAllowed())
{
// Sometimes, the parsing phase of the header of the response
// that the server gives us back, fails and a <no header>
// error is raised. This happens with HTTP/1.1 persistent
// connections, usually because the previous response stream
// has not yet been flushed, so the buffer still contains
// data regarding the last document retrieved. That sucks alot!
// The only thing to do is to lose persistent connections benefits
// for this document, so close the connection and 'GET' it again.
CloseConnection(); // Close a previous connection
if (debug>0)
cout << "! Impossible to get the HTTP header line." << endl
<< " Connection closed. Try to get it again." << endl;
result = HTTPRequest(); // Get the document again
}
return result;
}
///////
// Sends an HTTP 1/1 request
///////
Transport::DocStatus HtHTTP::HTTPRequest()
{
static Transport::DocStatus DocumentStatus;
bool ShouldTheBodyBeRead = true;
SetBodyReadingController(&HtHTTP::ReadBody);
// Reset the response
_response.Reset();
// Flush the connection
FlushConnection();
_bytes_read=0;
if( debug > 4)
cout << "Try to get through to host "
<< _url.host() << " (port " << _url.port() << ")" << endl;
ConnectionStatus result;
// Assign the timeout
AssignConnectionTimeOut();
// Assign number of retries
AssignConnectionRetries();
// Assign connection wait time
AssignConnectionWaitTime();
// Start the timer
_start_time.SettoNow();
result = EstablishConnection();
if(result != Connection_ok && result != Connection_already_up)
{
switch (result)
{
// Open failed
case Connection_open_failed:
if (debug>1)
cout << "Unable to open the connection with host: "
<< _url.host() << " (port " << _url.port() << ")" << endl;
CloseConnection();
return FinishRequest(Document_no_connection);
break;
// Server not reached
case Connection_no_server:
if (debug>1)
cout << "Unable to find the host: "
<< _url.host() << " (port " << _url.port() << ")" << endl;
CloseConnection();
return FinishRequest(Document_no_host);
break;
// Port not reached
case Connection_no_port:
if (debug>1)
cout << "Unable to connect with the port " << _url.port()
<< " of the host: " << _url.host() << endl;
CloseConnection();
return FinishRequest(Document_no_port);
break;
// Connection failed
case Connection_failed:
if (debug>1)
cout << "Unable to establish the connection with host: "
<< _url.host() << " (port " << _url.port() << ")" << endl;
CloseConnection();
return FinishRequest(Document_no_connection);
break;
// Other reason
default:
if (debug>1)
cout << "connection failed with unexpected result: result = "
<< (int)result << ", "
<< _url.host() << " (port " << _url.port() << ")" << endl;
CloseConnection();
return FinishRequest(Document_other_error);
break;
}
return FinishRequest(Document_other_error);
}
// Visual comments about the result of the connection
if (debug > 5)
switch(result)
{
case Connection_already_up:
cout << "Taking advantage of persistent connections" << endl;
break;
case Connection_ok:
cout << "New connection open successfully" << endl;
break;
default:
cout << "Unexptected value: " << (int)result << endl;
break;
}
String command;
switch(_Method)
{
case Method_GET:
command = "GET ";
break;
case Method_HEAD:
command = "HEAD ";
ShouldTheBodyBeRead = false;
break;
}
// Set the request command
SetRequestCommand(command);
if (debug > 6)
cout << "Request\n" << command;
// Writes the command
ConnectionWrite(command);
// Parse the header
if (ParseHeader() == -1) // Connection down
{
// The connection probably fell down !?!
if ( debug > 4 )
cout << setw(5) << Transport::GetTotOpen() << " - "
<< "Connection fell down ... let's close it" << endl;
CloseConnection(); // Let's close the connection which is down now
// Return that the connection has fallen down during the request
return FinishRequest(Document_connection_down);
}
if (_response._status_code == -1)
{
// Unable to retrieve the status line
if ( debug > 4 )
cout << "Unable to retrieve or parse the status line" << endl;
return FinishRequest(Document_no_header);
}
if (debug > 3)
{
cout << "Retrieving document " << _url.path() << " on host: "
<< _url.host() << ":" << _url.port() << endl;
cout << "Http version : " << _response._version << endl;
cout << "Server : " << _response._version << endl;
cout << "Status Code : " << _response._status_code << endl;
cout << "Reason : " << _response._reason_phrase << endl;
if (_response.GetAccessTime())
cout << "Access Time : " << _response.GetAccessTime()->GetRFC1123() << endl;
if (_response.GetModificationTime())
cout << "Modification Time : " << _response.GetModificationTime()->GetRFC1123() << endl;
cout << "Content-type : " << _response.GetContentType() << endl;
if (_response._transfer_encoding.length())
cout << "Transfer-encoding : " << _response._transfer_encoding << endl;
if (_response._content_language.length())
cout << "Content-Language : " << _response._content_language << endl;
if (_response._hdrconnection.length())
cout << "Connection : " << _response._hdrconnection << endl;
}
// Check if persistent connection are possible
CheckPersistentConnection(_response);
if (debug > 4)
cout << "Persistent connection: "
<< (_persistent_connection_possible ? "would be accepted" : "not accepted")
<< endl;
DocumentStatus = GetDocumentStatus(_response);
// We read the body only if the document has been found
if (DocumentStatus != Document_ok)
{
ShouldTheBodyBeRead=false;
}
// For now a chunked response MUST BE retrieved
if (mystrncasecmp ((char*)_response._transfer_encoding, "chunked", 7) == 0)
{
// Change the controller of the body reading
SetBodyReadingController(&HtHTTP::ReadChunkedBody);
}
// If "ShouldTheBodyBeRead" is set to true and
// If the document is parsable, we can read the body
// otherwise it is not worthwhile
if (ShouldTheBodyBeRead)
{
if ( debug > 4 )
cout << "Reading the body of the response" << endl;
// We use a int (HtHTTP::*)() function pointer
if ( (this->*_readbody)() == -1 )
{
// The connection probably fell down !?!
if ( debug > 4 )
cout << setw(5) << Transport::GetTotOpen() << " - "
<< "Connection fell down ... let's close it" << endl;
CloseConnection(); // Let's close the connection which is down now
// Return that the connection has fallen down during the request
return FinishRequest(Document_connection_down);
}
if ( debug > 6 )
cout << "Contents:" << endl << _response.GetContents();
// Check if the stream returned by the server has not been completely read
if (_response._document_length != _response._content_length &&
_response._document_length == _max_document_size)
{
// Max document size reached
if (debug > 4)
cout << "Max document size (" << GetRequestMaxDocumentSize()
<< ") reached ";
if (isPersistentConnectionUp())
{
// Only have to close persistent connection when we didn't read
// all the input. For now, we always read all chunked input...
if (mystrncasecmp ((char*)_response._transfer_encoding, "chunked", 7) != 0)
{
if (debug > 4)
cout << "- connection closed. ";
CloseConnection();
}
}
if (debug > 4)
cout << endl;
}
// Make sure our content-length makes sense, if none given...
if (_response._content_length < _response._document_length)
_response._content_length = _response._document_length;
}
else if ( debug > 4 )
cout << "Body not retrieved" << endl;
// Close the connection (if there's no persistent connection)
if( ! isPersistentConnectionUp() )
{
if ( debug > 4 )
cout << setw(5) << Transport::GetTotOpen() << " - "
<< "Connection closed (No persistent connection)" << endl;
CloseConnection();
}
else
{
// Persistent connection is active
// If the document is not parsable and we asked for it with a 'GET'
// method, the stream's not been completely read.
if (DocumentStatus == Document_not_parsable && _Method == Method_GET)
{
// We have to close the connection.
if ( debug > 4 )
cout << "Connection must be closed (stream not completely read)"
<< endl;
CloseConnection();
}
else
if ( debug > 4 )
cout << "Connection stays up ... (Persistent connection)" << endl;
}
// Check the doc_status and return a value
return FinishRequest(DocumentStatus);
}
HtHTTP::ConnectionStatus HtHTTP::EstablishConnection()
{
int result;
// Open the connection
result=OpenConnection();
if (!result)
return Connection_open_failed; // Connection failed
else if(debug > 4)
{
cout << setw(5) << Transport::GetTotOpen() << " - ";
if (result == -1)
cout << "Connection already open. No need to re-open." << endl;
else
cout << "Open of the connection ok" << endl;
}
if(result==1) // New connection open
{
// Assign the remote host to the connection
if ( !AssignConnectionServer() )
return Connection_no_server;
else if (debug > 4)
cout << "\tAssigned the remote host " << _url.host() << endl;
// Assign the port of the remote host
if ( !AssignConnectionPort() )
return Connection_no_port;
else if (debug > 4)
cout << "\tAssigned the port " << _url.port() << endl;
}
// Connect
if (! (result = Connect()))
return Connection_failed;
else if (result == -1) return Connection_already_up; // Persistent
else return Connection_ok; // New connection
}
// Set the string of the HTTP message request
void HtHTTP::SetRequestCommand(String &cmd)
{
// Initialize it
if (_useproxy) {
cmd << _url.get() << " HTTP/1.1\r\n";
} else
cmd << _url.path() << " HTTP/1.1\r\n";
// Insert the "virtual" host to which ask the document
cmd << "Host: " << _url.host();
if (_url.port() != 0 && _url.port() != _url.DefaultPort())
cmd << ":" << _url.port();
cmd << "\r\n";
// Insert the User Agent
if (_user_agent.length())
cmd << "User-Agent: " << _user_agent << "\r\n";
// Referer
if (_referer.get().length())
cmd << "Referer: " << _referer.get() << "\r\n";
// Accept-Language
if (_accept_language.length())
cmd << "Accept-language: " << _accept_language << "\r\n";
// Authentication
if (_credentials.length())
cmd << "Authorization: Basic " << _credentials << "\r\n";
// Proxy Authentication
if (_useproxy && _proxy_credentials.length())
cmd << "Proxy-Authorization: Basic " << _proxy_credentials << "\r\n";
// Accept-Encoding: waiting to handle the gzip and compress formats, we
// just send an empty header which, according to the HTTP 1/1 standard,
// should let the server know that we only accept the 'identity' case
// (no encoding of the document)
cmd << "Accept-Encoding: \r\n";
// A date has been passed to check if the server one is newer than
// the one we already own.
if(_modification_time && *_modification_time > 0)
{
_modification_time->ToGMTime();
cmd << "If-Modified-Since: " << _modification_time->GetRFC1123() << "\r\n";
}
///////
// Cookies! Let's go eat them! ;-)
///////
// The method returns all the valid cookies and writes them
// directly into the request string, as a list of headers
if (_send_cookies && _cookie_jar)
_cookie_jar->SetHTTPRequest_CookiesString(_url, cmd);
// Let's close the command
cmd << "\r\n";
}
//*****************************************************************************
// int HtHTTP::ParseHeader()
// Parse the header of the document
//
int HtHTTP::ParseHeader()
{
String line = 0;
int inHeader = 1;
if (_response._modification_time)
{
delete _response._modification_time;
_response._modification_time=0;
}
while (inHeader)
{
line.trunc();
if(! _connection->Read_Line(line, "\n"))
return -1; // Connection down
_bytes_read+=line.length();
line.chop('\r');
if (line.length() == 0)
inHeader = 0;
else
{
// Found a not-empty line
if (debug > 2)
cout << "Header line: " << line << endl;
// Status - Line check
char *token = line.get();
while (*token && !isspace(*token) && *token != ':')
++token;
while (*token && (isspace(*token) || *token == ':'))
++token;
if(!strncmp((char*)line, "HTTP/", 5))
{
// Here is the status-line
// store the HTTP version returned by the server
_response._version = strtok(line, " ");
// Store the status code
_response._status_code = atoi(strtok(0, " "));
// Store the reason phrase
_response._reason_phrase = strtok(0, "\n");
}
else if( ! mystrncasecmp((char*)line, "server:", 7))
{
// Server info
// Set the server info
token = strtok(token, "\n\t");
if (token && *token)
_response._server = token;
}
else if( ! mystrncasecmp((char*)line, "last-modified:", 14))
{
// Modification date sent by the server
// Set the response modification time
token = strtok(token, "\n\t");
if (token && *token)
_response._modification_time = NewDate(token);
}
else if( ! mystrncasecmp((char*)line, "date:", 5))
{
// Access date time sent by the server
// Set the response access time
token = strtok(token, "\n\t");
if (token && *token)
_response._access_time = NewDate(token);
}
else if( ! mystrncasecmp((char*)line, "content-type:", 13))
{
// Content - type
token = strtok(token, "\n\t");
if (token && *token)
_response._content_type = token;
}
else if( ! mystrncasecmp((char*)line, "content-length:", 15))
{
// Content - length
token = strtok(token, "\n\t");
if (token && *token)
_response._content_length = atoi(token);
}
else if( ! mystrncasecmp((char*)line, "transfer-encoding:", 18))
{
// Transfer-encoding
token = strtok(token, "\n\t");
if (token && *token)
_response._transfer_encoding = token;
}
else if( ! mystrncasecmp((char*)line, "location:", 9))
{
// Found a location directive - redirect in act
token = strtok(token, "\n\t");
if (token && *token)
_response._location = token;
}
else if( ! mystrncasecmp((char*)line, "connection:", 11))
{
// Ooops ... found a Connection clause
token = strtok(token, "\n\t");
if (token && *token)
_response._hdrconnection = token;
}
else if( ! mystrncasecmp((char*)line, "content-language:", 17))
{
// Found a content-language directive
token = strtok(token, "\n\t");
if (token && *token)
_response._content_language = token;
}
else if( ! mystrncasecmp((char*)line, "set-cookie:", 11))
{
// Found a cookie
// Are cookies enabled?
if (_send_cookies && _cookie_jar)
{
token = strtok(token, "\n\t");
if (token && *token)
{
// Insert the cookie into the jar
_cookie_jar->AddCookie(token, _url);
}
}
}
else
{
// Discarded
if (debug > 3)
cout << "Discarded header line: " << line << endl;
}
}
}
if (_response._modification_time == 0)
{
if (debug > 3)
cout << "No modification time returned: assuming now" << endl;
//Set the modification time
_response._modification_time = new HtDateTime;
_response._modification_time->ToGMTime(); // Set to GM time
}
return 1;
}
// Check for a document to be parsable
// It all depends on the content-type directive returned by the server
bool HtHTTP::isParsable(const char *content_type)
{
// Here I can decide what kind of document I can parse
// depending on the value of Transport:_default_parser_content_type
// and the rest are determined by the external_parser settings
if( ! mystrncasecmp (_default_parser_content_type.get(), content_type,
_default_parser_content_type.length()) )
return true;
// External function that checks if a document is parsable or not.
// CanBeParsed should point to a function that returns an int value,
// given a char * containing the content-type.
if (CanBeParsed && (*CanBeParsed)( (char *) content_type) )
return true;
return false;
}
// Check for a possibile persistent connection
// on the return message's HTTP version basis
void HtHTTP::CheckPersistentConnection(HtHTTP_Response &response)
{
const char *version = response.GetVersion();
if( ! mystrncasecmp ("HTTP/1.1", version, 8))
{
const char *connection = response.GetConnectionInfo();
if( ! mystrncasecmp ("close", connection, 5))
_persistent_connection_possible=false; // Server wants to close
else _persistent_connection_possible=true;
}
else
_persistent_connection_possible=false;
}
HtHTTP::DocStatus HtHTTP::FinishRequest (HtHTTP::DocStatus ds)
{
int seconds;
// Set the finish time
_end_time.SettoNow();
// Let's add the number of seconds needed by the request
seconds=HtDateTime::GetDiff(_end_time, _start_time);
_tot_seconds += seconds;
_tot_requests ++;
_tot_bytes += _bytes_read;
if (debug > 2)
cout << "Request time: " << seconds << " secs" << endl;
return ds;
}
HtHTTP::DocStatus HtHTTP::GetDocumentStatus(HtHTTP_Response &r)
{
// Let's give a look at the return status code
HtHTTP::DocStatus returnStatus=Document_not_found;
int statuscode;
statuscode=r.GetStatusCode();
if(statuscode==200)
{
returnStatus = Document_ok; // OK
// Is it parsable?
if (! isParsable ((const char*)r.GetContentType()) )
returnStatus=Document_not_parsable;
}
else if(statuscode > 200 && statuscode < 300)
returnStatus = Document_ok; // Successful 2xx
else if(statuscode==304)
returnStatus = Document_not_changed; // Not modified
else if(statuscode > 300 && statuscode < 400)
returnStatus = Document_redirect; // Redirection 3xx
else if(statuscode==401)
returnStatus = Document_not_authorized; // Unauthorized
// Exit the function
return returnStatus;
}
void HtHTTP::SetCredentials (const String& s)
{
Transport::SetHTTPBasicAccessAuthorizationString(_credentials, s);
}
void HtHTTP::SetProxyCredentials (const String& s)
{
Transport::SetHTTPBasicAccessAuthorizationString(_proxy_credentials, s);
}
int HtHTTP::ReadBody()
{
_response._contents = 0; // Initialize the string
char docBuffer[8192];
int bytesRead = 0;
int bytesToGo = _response._content_length;
if (bytesToGo < 0 || bytesToGo > _max_document_size)
bytesToGo = _max_document_size;
while (bytesToGo > 0)
{
int len = bytesToGo< (int)sizeof(docBuffer) ? bytesToGo : (int)sizeof(docBuffer);
bytesRead = _connection->Read(docBuffer, len);
if (bytesRead <= 0)
break;
_response._contents.append(docBuffer, bytesRead);
bytesToGo -= bytesRead;
_bytes_read+=bytesRead;
}
// Set document length
_response._document_length = _response._contents.length();
return bytesRead;
}
int HtHTTP::ReadChunkedBody()
{
// Chunked Transfer decoding
// as shown in the RFC2616 (HTTP/1.1) - 19.4.6
#define BSIZE 8192
int length = 0; // initialize the length
unsigned int chunk_size;
String ChunkHeader = 0;
char buffer[BSIZE+1];
int chunk, rsize;
_response._contents.trunc(); // Initialize the string
// Read chunk-size and CRLF
if (!_connection->Read_Line(ChunkHeader, "\r\n"))
return -1;
sscanf ((char *)ChunkHeader, "%x", &chunk_size);
if (debug>4)
cout << "Initial chunk-size: " << chunk_size << endl;
while (chunk_size > 0)
{
chunk = chunk_size;
do {
if (chunk > BSIZE) {
rsize = BSIZE;
if (debug>4)
cout << "Read chunk partial: left=" << chunk << endl;
} else {
rsize = chunk;
}
chunk -= rsize;
// Read Chunk data
if (_connection->Read(buffer, rsize) == -1)
return -1;
length+=rsize;
// Append the chunk-data to the contents of the response
// ... but not more than _max_document_size...
if (rsize > _max_document_size-_response._contents.length())
rsize = _max_document_size-_response._contents.length();
buffer[rsize] = 0;
_response._contents.append(buffer, rsize);
} while (chunk);
// if (_connection->Read(buffer, chunk_size) == -1)
// return -1;
// Read CRLF - to be ignored
if (!_connection->Read_Line(ChunkHeader, "\r\n"))
return -1;
// Read chunk-size and CRLF
if (!_connection->Read_Line(ChunkHeader, "\r\n"))
return -1;
sscanf ((char *)ChunkHeader, "%x", &chunk_size);
if (debug>4)
cout << "Chunk-size: " << chunk_size << endl;
}
ChunkHeader = 0;
// Ignoring next part of the body - the TRAILER
// (it contains further headers - not implemented)
// Set content length
_response._content_length = length;
// Set document length
_response._document_length = _response._contents.length();
return length;
}
///////
// Show the statistics
///////
ostream &HtHTTP::ShowStatistics (ostream &out)
{
Transport::ShowStatistics(out); // call the base class method
out << " HTTP Requests : " << GetTotRequests() << endl;
out << " HTTP KBytes requested : " << (double)GetTotBytes()/1024 << endl;
out << " HTTP Average request time : " << GetAverageRequestTime()
<< " secs" << endl;
out << " HTTP Average speed : " << GetAverageSpeed()/1024
<< " KBytes/secs" << endl;
return out;
}