187
192
return sanitizedHeaders;
195
static bool isPotentialSpoofingAttack(const HTTPProtocol::HTTPRequest& request, const KConfigGroup* config)
197
// kDebug(7113) << request.url << "response code: " << request.responseCode << "previous response code:" << request.prevResponseCode;
198
if (request.url.user().isEmpty()) {
202
// We already have cached authentication.
203
if (config->readEntry(QLatin1String("cached-www-auth"), false)) {
207
const QString userName = config->readEntry(QLatin1String("LastSpoofedUserName"), QString());
208
return ((userName.isEmpty() || userName != request.url.user()) && request.responseCode != 401 && request.prevResponseCode != 401);
190
211
// for a given response code, conclude if the response is going to/likely to have a response body
191
212
static bool canHaveResponseBody(int responseCode, KIO::HTTP_METHOD method)
247
268
return isValidProxy(u) && u.protocol() == QLatin1String("http");
271
static QIODevice* createPostBufferDeviceFor (KIO::filesize_t size)
274
if (size > static_cast<KIO::filesize_t>(s_MaxInMemPostBufSize))
275
device = new KTemporaryFile;
277
device = new QBuffer;
279
if (!device->open(QIODevice::ReadWrite))
250
285
QByteArray HTTPProtocol::HTTPRequest::methodString() const
252
287
if (!methodStringOverride.isEmpty())
253
return (methodStringOverride + QLatin1Char(' ')).toLatin1();
288
return (methodStringOverride + QLatin1Char(' ')).toLatin1();
486
523
if (config()->readEntry("SendLanguageSettings", true)) {
487
m_request.charsets = config()->readEntry( "Charsets", "iso-8859-1" );
488
if (!m_request.charsets.isEmpty()) {
489
m_request.charsets += QLatin1String(DEFAULT_PARTIAL_CHARSET_HEADER);
524
m_request.charsets = config()->readEntry("Charsets", DEFAULT_PARTIAL_CHARSET_HEADER);
525
if (!m_request.charsets.contains(QLatin1String("*;"), Qt::CaseInsensitive)) {
526
m_request.charsets += QLatin1String(",*;q=0.5");
491
m_request.languages = config()->readEntry( "Languages", DEFAULT_LANGUAGE_HEADER );
528
m_request.languages = config()->readEntry("Languages", DEFAULT_LANGUAGE_HEADER);
493
530
m_request.charsets.clear();
494
531
m_request.languages.clear();
606
646
// if data is required internally, don't finish,
607
647
// it is processed before we finish()
609
if ((m_request.responseCode == 204) &&
610
((m_request.method == HTTP_GET) || (m_request.method == HTTP_POST))) {
611
error(ERR_NO_CONTENT, QString());
652
if (m_request.responseCode == 204 &&
653
(m_request.method == HTTP_GET || m_request.method == HTTP_POST)) {
654
infoMessage(QLatin1String(""));
655
error(ERR_NO_CONTENT, QString());
618
662
bool HTTPProtocol::proceedUntilResponseHeader()
1689
1733
case 500: // hack: Apache mod_dav returns this instead of 403 (!)
1690
1734
// 403 Forbidden
1691
kError = ERR_ACCESS_DENIED;
1735
// ERR_ACCESS_DENIED
1692
1736
errorString = i18nc( "%1: request type", "Access was denied while attempting to %1.", action );
1695
1739
// 405 Method Not Allowed
1696
if ( m_request.method == DAV_MKCOL )
1698
kError = ERR_DIR_ALREADY_EXIST;
1740
if ( m_request.method == DAV_MKCOL ) {
1741
// ERR_DIR_ALREADY_EXIST
1699
1742
errorString = i18n("The specified folder already exists.");
1703
1746
// 409 Conflict
1704
kError = ERR_ACCESS_DENIED;
1747
// ERR_ACCESS_DENIED
1705
1748
errorString = i18n("A resource cannot be created at the destination "
1706
1749
"until one or more intermediate collections (folders) "
1707
1750
"have been created.");
1710
1753
// 412 Precondition failed
1711
if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE )
1713
kError = ERR_ACCESS_DENIED;
1754
if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE ) {
1755
// ERR_ACCESS_DENIED
1714
1756
errorString = i18n("The server was unable to maintain the liveness of "
1715
1757
"the properties listed in the propertybehavior XML "
1716
1758
"element or you attempted to overwrite a file while "
1717
1759
"requesting that files are not overwritten. %1",
1721
else if ( m_request.method == DAV_LOCK )
1723
kError = ERR_ACCESS_DENIED;
1762
} else if ( m_request.method == DAV_LOCK ) {
1763
// ERR_ACCESS_DENIED
1724
1764
errorString = i18n("The requested lock could not be granted. %1", ow );
1728
1768
// 415 Unsupported Media Type
1729
kError = ERR_ACCESS_DENIED;
1769
// ERR_ACCESS_DENIED
1730
1770
errorString = i18n("The server does not support the request type of the body.");
1734
kError = ERR_ACCESS_DENIED;
1774
// ERR_ACCESS_DENIED
1735
1775
errorString = i18nc( "%1: request type", "Unable to %1 because the resource is locked.", action );
1742
1782
// 502 Bad Gateway
1743
if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE )
1745
kError = ERR_WRITE_ACCESS_DENIED;
1783
if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE ) {
1784
// ERR_WRITE_ACCESS_DENIED
1746
1785
errorString = i18nc( "%1: request type", "Unable to %1 because the destination server refuses "
1747
1786
"to accept the file or folder.", action );
1751
1790
// 507 Insufficient Storage
1752
kError = ERR_DISK_FULL;
1753
1792
errorString = i18n("The destination resource does not have sufficient space "
1754
1793
"to record the state of the resource after the execution "
1755
1794
"of this method.");
1793
1830
case 500: // hack: Apache mod_dav returns this instead of 403 (!)
1794
1831
// 403 Forbidden
1795
1832
// 405 Method Not Allowed
1796
kError = ERR_ACCESS_DENIED;
1833
// ERR_ACCESS_DENIED
1797
1834
errorString = i18nc( "%1: request type", "Access was denied while attempting to %1.", action );
1800
1837
// 409 Conflict
1801
kError = ERR_ACCESS_DENIED;
1838
// ERR_ACCESS_DENIED
1802
1839
errorString = i18n("A resource cannot be created at the destination "
1803
1840
"until one or more intermediate collections (folders) "
1804
1841
"have been created.");
1808
kError = ERR_ACCESS_DENIED;
1845
// ERR_ACCESS_DENIED
1809
1846
errorString = i18nc( "%1: request type", "Unable to %1 because the resource is locked.", action );
1812
1849
// 502 Bad Gateway
1813
kError = ERR_WRITE_ACCESS_DENIED;
1850
// ERR_WRITE_ACCESS_DENIED;
1814
1851
errorString = i18nc( "%1: request type", "Unable to %1 because the destination server refuses "
1815
"to accept the file or folder.", action );
1852
"to accept the file or folder.", action );
1818
1855
// 507 Insufficient Storage
1819
kError = ERR_DISK_FULL;
1820
1857
errorString = i18n("The destination resource does not have sufficient space "
1821
1858
"to record the state of the resource after the execution "
1822
1859
"of this method.");
2390
2427
kDebug(7113) << "needs validation, performing conditional get.";
2391
2428
/* conditional get */
2392
2429
if (!m_request.cacheTag.etag.isEmpty())
2393
header += QLatin1String("If-None-Match: ")+m_request.cacheTag.etag+QLatin1String("\r\n");
2430
header += QLatin1String("If-None-Match: ") + m_request.cacheTag.etag + QLatin1String("\r\n");
2395
2432
if (m_request.cacheTag.lastModifiedDate != -1) {
2396
QString httpDate = formatHttpDate(m_request.cacheTag.lastModifiedDate);
2433
const QString httpDate = formatHttpDate(m_request.cacheTag.lastModifiedDate);
2397
2434
header += QLatin1String("If-Modified-Since: ") + httpDate + QLatin1String("\r\n");
2398
2435
setMetaData(QLatin1String("modified"), httpDate);
2408
2445
header += QLatin1String("\r\n");
2410
2447
if (m_request.allowTransferCompression)
2411
header += QLatin1String("Accept-Encoding: x-gzip, x-deflate, gzip, deflate\r\n");
2448
header += QLatin1String("Accept-Encoding: gzip, deflate, x-gzip, x-deflate\r\n");
2413
2450
if (!m_request.charsets.isEmpty())
2414
2451
header += QLatin1String("Accept-Charset: ") + m_request.charsets + QLatin1String("\r\n");
2454
2491
header += QLatin1String("\r\n");
2494
// DoNotTrack feature...
2495
if (config()->readEntry("DoNotTrack", false))
2496
header += QLatin1String("DNT: 1\r\n");
2457
2498
// Remember that at least one failed (with 401 or 407) request/response
2458
2499
// roundtrip is necessary for the server to tell us that it requires
2460
// We proactively add authentication headers if we have cached credentials
2461
// to avoid the extra roundtrip where possible.
2462
// (TODO: implement this caching)
2500
// authentication. However, we proactively add authentication headers if when
2501
// we have cached credentials to avoid the extra roundtrip where possible.
2463
2502
header += authenticationHeader();
2465
2504
if ( m_protocol == "webdav" || m_protocol == "webdavs" )
2630
2671
m_mimeType = QLatin1String("application/x-gzpostscript");
2674
// Prefer application/x-xz-compressed-tar over application/x-xz for LMZA compressed
2675
// tar files. Arch Linux AUR servers notoriously send the wrong mimetype for this.
2676
else if(m_mimeType == QLatin1String("application/x-xz")) {
2677
if (m_request.url.path().endsWith(QLatin1String(".tar.xz")) ||
2678
m_request.url.path().endsWith(QLatin1String(".txz"))) {
2679
m_mimeType = QLatin1String("application/x-xz-compressed-tar");
2633
2683
// Some webservers say "text/plain" when they mean "application/x-bzip"
2634
2684
else if ((m_mimeType == QLatin1String("text/plain")) || (m_mimeType == QLatin1String("application/octet-stream"))) {
2635
QString ext = m_request.url.path().right(4).toUpper();
2636
if (ext == QLatin1String(".BZ2"))
2685
const QString ext = QFileInfo(m_request.url.path()).suffix().toUpper();
2686
if (ext == QLatin1String("BZ2"))
2637
2687
m_mimeType = QLatin1String("application/x-bzip");
2638
else if (ext == QLatin1String(".PEM"))
2688
else if (ext == QLatin1String("PEM"))
2639
2689
m_mimeType = QLatin1String("application/x-x509-ca-cert");
2640
else if (ext == QLatin1String(".SWF"))
2690
else if (ext == QLatin1String("SWF"))
2641
2691
m_mimeType = QLatin1String("application/x-shockwave-flash");
2642
else if (ext == QLatin1String(".PLS"))
2692
else if (ext == QLatin1String("PLS"))
2643
2693
m_mimeType = QLatin1String("audio/x-scpls");
2644
else if (ext == QLatin1String(".WMV"))
2694
else if (ext == QLatin1String("WMV"))
2645
2695
m_mimeType = QLatin1String("video/x-ms-wmv");
2696
else if (ext == QLatin1String("WEBM"))
2697
m_mimeType = QLatin1String("video/webm");
2647
2699
kDebug(7113) << "after fixup" << m_mimeType;
2747
//Return true if the term was found, false otherwise. Advance *pos.
2748
//If (*pos + strlen(term) >= end) just advance *pos to end and return false.
2749
//This means that users should always search for the shortest terms first.
2750
static bool consume(const char input[], int *pos, int end, const char *term)
2752
// note: gcc/g++ is quite good at optimizing away redundant strlen()s
2754
if (idx + (int)strlen(term) >= end) {
2758
if (strncasecmp(&input[idx], term, strlen(term)) == 0) {
2759
*pos = idx + strlen(term);
2697
2766
* This function will read in the return header from the server. It will
2842
2908
// immediately act on most response codes...
2910
// Protect users against bogus username intended to fool them into visiting
2911
// sites they had no intention of visiting.
2912
if (isPotentialSpoofingAttack(m_request, config())) {
2913
// kDebug(7113) << "**** POTENTIAL ADDRESS SPOOFING:" << m_request.url;
2914
const int result = messageBox(WarningYesNo,
2915
i18nc("@warning: Security check on url "
2916
"being accessed", "You are about to "
2917
"log in to the site \"%1\" with the "
2918
"username \"%2\", but the website "
2919
"does not require authentication. "
2920
"This may be an attempt to trick you."
2921
"<p>Is \"%1\" the site you want to visit?",
2922
m_request.url.host(), m_request.url.user()),
2923
i18nc("@title:window", "Confirm Website Access"));
2924
if (result == KMessageBox::No) {
2925
error(ERR_USER_CANCELED, m_request.url.url());
2928
setMetaData(QLatin1String("{internal~currenthost}LastSpoofedUserName"), m_request.url.user());
2844
2931
if (m_request.responseCode != 200 && m_request.responseCode != 304) {
2845
2932
m_request.cacheTag.ioMode = NoCache;
2963
3050
if (auth->realm().isEmpty() && !auth->supportsPathMatching())
2964
3051
setMetaData(QLatin1String("{internal~allhosts}proxy-auth-realm"), authinfo.realmValue);
2966
cacheAuthentication(authinfo);
3053
if (authinfo.keepPassword) {
3054
cacheAuthentication(authinfo);
2967
3056
kDebug(7113) << "Caching authentication for" << m_request.url;
2969
3058
// Update our server connection state which includes www and proxy username and password.
2984
3073
// either we have a http response line -> try to parse the header, fail if it doesn't work
2985
3074
// or we have garbage -> fail.
2986
3075
HeaderTokenizer tokenizer(buffer);
2987
headerSize = tokenizer.tokenize(idx, sizeof(buffer));
3076
tokenizer.tokenize(idx, sizeof(buffer));
2989
3078
// Note that not receiving "accept-ranges" means that all bets are off
2990
3079
// wrt the server supporting ranges.
3518
3610
prevLineEnd - prevLinePos));
3519
3611
prevLinePos = nextLinePos;
3522
3614
// IMPORTNAT: Do not remove this line because forwardHttpResponseHeader
3523
// is called below. This line is added to make http response headers are
3524
// available by the time the content mimetype information is transmitted
3525
// to the job. If the line below is removed, the KIO-QNAM integration
3526
// will not work properly when attempting to put ioslaves on hold.
3615
// is called below. This line is here to ensure the response headers are
3616
// available to the client before it receives mimetype information.
3617
// The support for putting ioslaves on hold in the KIO-QNAM integration
3618
// will break if this line is removed.
3527
3619
setMetaData(QLatin1String("HTTP-Headers"), m_responseHeaders.join(QString(QLatin1Char('\n'))));
3530
3622
// Let the app know about the mime-type iff this is not a redirection and
3531
3623
// the mime-type string is not empty.
3532
if (!m_isRedirection &&
3624
if (!m_isRedirection && m_request.responseCode != 204 &&
3533
3625
(!m_mimeType.isEmpty() || m_request.method == HTTP_HEAD) &&
3534
3626
(m_isLoadingErrorPage || !authRequiresAnotherRoundtrip)) {
3535
3627
kDebug(7113) << "Emitting mimetype " << m_mimeType;
3536
3628
mimeType( m_mimeType );
3539
// Do not move the function call below before doing any redirection.
3540
// Otherwise it might mess up some sites. See BR# 150904.
3541
// IMPORTANT: Do not remove it either thinking it duplicates what is done
3542
// above. Otherwise, the http response headers will not be available if
3543
// this ioslave is put on hold.
3631
// IMPORTANT: Do not move the function call below before doing any
3632
// redirection. Otherwise it might mess up some sites, see BR# 150904.
3544
3633
forwardHttpResponseHeader();
3546
3635
if (m_request.method == HTTP_HEAD)
3885
bool HTTPProtocol::sendCachedBody()
3887
infoMessage(i18n("Sending data to %1" , m_request.url.host()));
3889
QByteArray cLength ("Content-Length: ");
3890
cLength += QByteArray::number(m_POSTbuf->size());
3891
cLength += "\r\n\r\n";
3893
kDebug(7113) << "sending cached data (size=" << m_POSTbuf->size() << ")";
3895
// Send the content length...
3896
bool sendOk = (write(cLength.data(), cLength.size()) == (ssize_t) cLength.size());
3898
kDebug( 7113 ) << "Connection broken when sending "
3899
<< "content length: (" << m_request.url.host() << ")";
3900
error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3904
// Make sure the read head is at the beginning...
3908
while (!m_POSTbuf->atEnd()) {
3909
const QByteArray buffer = m_POSTbuf->read(s_MaxInMemPostBufSize);
3910
sendOk = (write(buffer.data(), buffer.size()) == (ssize_t) buffer.size());
3912
kDebug(7113) << "Connection broken when sending message body: ("
3913
<< m_request.url.host() << ")";
3914
error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3796
3922
bool HTTPProtocol::sendBody()
3798
infoMessage( i18n( "Requesting data to send" ) );
3800
int readFromApp = -1;
3802
// m_POSTbuf will NOT be empty iff authentication was required before posting
3803
// the data OR a re-connect is requested from ::readResponseHeader because the
3804
// connection was lost for some reason.
3805
if (m_POSTbuf.isEmpty())
3807
kDebug(7113) << "POST'ing live data...";
3812
m_POSTbuf.append(buffer);
3814
dataReq(); // Request for data
3815
readFromApp = readData(buffer);
3816
} while (readFromApp > 0);
3820
kDebug(7113) << "POST'ing saved data...";
3824
if (readFromApp < 0)
3826
error(ERR_ABORTED, m_request.url.host());
3924
// If we have cached data, the it is either a repost or a DAV request so send
3925
// the cached data...
3927
return sendCachedBody();
3929
if (m_iPostDataSize == NO_SIZE) {
3930
// Try the old approach of retireving content data from the job
3931
// before giving up.
3932
if (retrieveAllData())
3933
return sendCachedBody();
3935
error(ERR_POST_NO_SIZE, m_request.url.host());
3830
infoMessage(i18n("Sending data to %1" , m_request.url.host()));
3832
const QByteArray cLength = QByteArray("Content-Length: ") + QByteArray::number(m_POSTbuf.size()) + "\r\n\r\n";
3939
kDebug(7113) << "sending data (size=" << m_iPostDataSize << ")";
3941
infoMessage(i18n("Sending data to %1", m_request.url.host()));
3943
QByteArray cLength ("Content-Length: ");
3944
cLength += QByteArray::number(m_iPostDataSize);
3945
cLength += "\r\n\r\n";
3833
3947
kDebug(7113) << cLength.trimmed();
3835
3949
// Send the content length...
3836
bool sendOk = (write(cLength.constData(), cLength.size()) == (ssize_t) cLength.size());
3950
bool sendOk = (write(cLength.data(), cLength.size()) == (ssize_t) cLength.size());
3838
3952
// The server might have closed the connection due to a timeout, or maybe
3839
3953
// some transport problem arose while the connection was idle.
3852
//kDebug(7113) << "POST DATA:" << QString::fromLocal8Bit(m_POSTbuf);
3853
sendOk = (write(m_POSTbuf.data(), m_POSTbuf.size()) == (ssize_t) m_POSTbuf.size());
3856
kDebug(7113) << "Connection broken when sending message body: ("
3857
<< m_request.url.host() << ")";
3858
error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3966
totalSize(m_iPostDataSize);
3969
KIO::filesize_t bytesSent = 0;
3975
const int bytesRead = readData(buffer);
3978
if (bytesRead == 0) {
3979
sendOk = (bytesSent == m_iPostDataSize);
3983
// On error return false...
3984
if (bytesRead < 0) {
3985
error(ERR_ABORTED, m_request.url.host());
3990
// Cache the POST data in case of a repost request.
3991
cachePostData(buffer);
3993
// This will only happen if transmitting the data fails, so we will simply
3994
// cache the content locally for the potential re-transmit...
3998
if (write(buffer.data(), bytesRead) == static_cast<ssize_t>(bytesRead)) {
3999
bytesSent += bytesRead;
4000
processedSize(bytesSent); // Send update status...
4004
kDebug(7113) << "Connection broken while sending POST content to" << m_request.url.host();
4005
error(ERR_CONNECTION_BROKEN, m_request.url.host());
3865
4012
void HTTPProtocol::httpClose( bool keepAlive )
4502
4651
HTTPProtocol::CacheTag::CachePlan HTTPProtocol::CacheTag::plan(time_t maxCacheAge) const
4504
4653
//notable omission: we're not checking cache file presence or integrity
4505
if (policy == KIO::CC_CacheOnly || policy == KIO::CC_Cache) {
4655
case KIO::CC_Refresh:
4656
// Conditional GET requires the presence of either an ETag or
4657
// last modified date.
4658
if (lastModifiedDate != -1 || !etag.isEmpty()) {
4659
return ValidateCached;
4662
case KIO::CC_Reload:
4663
return IgnoreCached;
4664
case KIO::CC_CacheOnly:
4506
4666
return UseCached;
4507
} else if (policy == KIO::CC_Refresh) {
4508
return ValidateCached;
4509
} else if (policy == KIO::CC_Reload) {
4510
return IgnoreCached;
4512
Q_ASSERT(policy == CC_Verify);
4671
Q_ASSERT((policy == CC_Verify || policy == CC_Refresh));
4513
4672
time_t currentDate = time(0);
4514
if ((servedDate != -1 && currentDate > servedDate + maxCacheAge) ||
4673
if ((servedDate != -1 && currentDate > (servedDate + maxCacheAge)) ||
4515
4674
(expireDate != -1 && currentDate > expireDate)) {
4516
4675
return ValidateCached;
4968
5127
m_request.cacheTag.file->write(d);
5130
void HTTPProtocol::cachePostData(const QByteArray& data)
5133
m_POSTbuf = createPostBufferDeviceFor(qMax(m_iPostDataSize, static_cast<KIO::filesize_t>(data.size())));
5138
m_POSTbuf->write (data.constData(), data.size());
5141
void HTTPProtocol::clearPostDataBuffer()
5150
bool HTTPProtocol::retrieveAllData()
5153
m_POSTbuf = createPostBufferDeviceFor(s_MaxInMemPostBufSize + 1);
5157
error (ERR_OUT_OF_MEMORY, m_request.url.host());
5164
const int bytesRead = readData(buffer);
5166
if (bytesRead < 0) {
5167
error(ERR_ABORTED, m_request.url.host());
5171
if (bytesRead == 0) {
5175
m_POSTbuf->write(buffer.constData(), buffer.size());
4971
5181
// The above code should be kept in sync
4972
5182
// with the code in http_cache_cleaner.cpp
4988
5198
// If no relam metadata, then make sure path matching is turned on.
4989
5199
authinfo.verifyPath = (authinfo.realmValue.isEmpty());
4991
if (checkCachedAuthentication(authinfo)) {
5201
const bool useCachedAuth = (m_request.responseCode == 401 || !(config()->readEntry("no-preemptive-auth-reuse", false)));
5202
if (useCachedAuth && checkCachedAuthentication(authinfo)) {
4992
5203
const QByteArray cachedChallenge = authinfo.digestInfo.toLatin1();
4993
5204
if (!cachedChallenge.isEmpty()) {
4994
5205
m_wwwAuth = KAbstractHttpAuthentication::newAuth(cachedChallenge, config());