2
* This file is part of nzbget
4
* Copyright (C) 2012-2013 Andrey Prygunkov <hugbug@users.sourceforge.net>
6
* This program is free software; you can redistribute it and/or modify
7
* it under the terms of the GNU General Public License as published by
8
* the Free Software Foundation; either version 2 of the License, or
9
* (at your option) any later version.
11
* This program is distributed in the hope that it will be useful,
12
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
* GNU General Public License for more details.
16
* You should have received a copy of the GNU General Public License
17
* along with this program; if not, write to the Free Software
18
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21
* $Date: 2013-05-14 22:20:52 +0200 (Tue, 14 May 2013) $
47
#include "WebDownloader.h"
52
extern Options* g_pOptions;
54
WebDownloader::WebDownloader()
56
debug("Creating WebDownloader");
59
m_szOutputFilename = NULL;
62
m_bConfirmedLength = false;
63
m_eStatus = adUndefined;
64
m_szOriginalFilename = NULL;
65
SetLastUpdateTimeNow();
68
WebDownloader::~WebDownloader()
70
debug("Destroying WebDownloader");
80
if (m_szOutputFilename)
82
free(m_szOutputFilename);
84
if (m_szOriginalFilename)
86
free(m_szOriginalFilename);
90
void WebDownloader::SetOutputFilename(const char* v)
92
m_szOutputFilename = strdup(v);
95
void WebDownloader::SetInfoName(const char* v)
97
m_szInfoName = strdup(v);
100
void WebDownloader::SetURL(const char * szURL)
102
m_szURL = strdup(szURL);
105
void WebDownloader::SetStatus(EStatus eStatus)
111
void WebDownloader::Run()
113
debug("Entering WebDownloader-loop");
115
SetStatus(adRunning);
117
int iRemainedDownloadRetries = g_pOptions->GetRetries() > 0 ? g_pOptions->GetRetries() : 1;
118
int iRemainedConnectRetries = iRemainedDownloadRetries > 10 ? iRemainedDownloadRetries : 10;
120
EStatus Status = adFailed;
122
while (!IsStopped() && iRemainedDownloadRetries > 0 && iRemainedConnectRetries > 0)
124
SetLastUpdateTimeNow();
128
if ((((Status == adFailed) && (iRemainedDownloadRetries > 1)) ||
129
((Status == adConnectError) && (iRemainedConnectRetries > 1)))
130
&& !IsStopped() && !(g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2()))
132
detail("Waiting %i sec to retry", g_pOptions->GetRetryInterval());
134
while (!IsStopped() && (msec < g_pOptions->GetRetryInterval() * 1000) &&
135
!(g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2()))
142
if (IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2())
148
if (Status == adFinished || Status == adFatalError || Status == adNotFound)
153
if (Status != adConnectError)
155
iRemainedDownloadRetries--;
159
iRemainedConnectRetries--;
163
if (Status != adFinished && Status != adRetry)
168
if (Status == adFailed)
172
detail("Download %s cancelled", m_szInfoName);
176
error("Download %s failed", m_szInfoName);
180
if (Status == adFinished)
182
detail("Download %s completed", m_szInfoName);
187
debug("Exiting WebDownloader-loop");
190
WebDownloader::EStatus WebDownloader::Download()
192
EStatus Status = adRunning;
196
Status = CreateConnection(&url);
197
if (Status != adRunning)
202
m_pConnection->SetSuppressErrors(false);
205
bool bConnected = m_pConnection->Connect();
206
if (!bConnected || IsStopped())
209
return adConnectError;
212
// Okay, we got a Connection. Now start downloading.
213
detail("Downloading %s", m_szInfoName);
217
Status = DownloadHeaders();
219
if (Status == adRunning)
221
Status = DownloadBody();
231
if (Status != adFinished)
233
// Download failed, delete broken output file
234
remove(m_szOutputFilename);
241
WebDownloader::EStatus WebDownloader::CreateConnection(URL *pUrl)
243
if (!pUrl->IsValid())
245
error("URL is not valid: %s", pUrl->GetAddress());
249
int iPort = pUrl->GetPort();
250
if (iPort == 0 && !strcasecmp(pUrl->GetProtocol(), "http"))
254
if (iPort == 0 && !strcasecmp(pUrl->GetProtocol(), "https"))
259
if (strcasecmp(pUrl->GetProtocol(), "http") && strcasecmp(pUrl->GetProtocol(), "https"))
261
error("Unsupported protocol in URL: %s", pUrl->GetAddress());
266
if (!strcasecmp(pUrl->GetProtocol(), "https"))
268
error("Program was compiled without TLS/SSL-support. Cannot download using https protocol. URL: %s", pUrl->GetAddress());
273
bool bTLS = !strcasecmp(pUrl->GetProtocol(), "https");
275
m_pConnection = new Connection(pUrl->GetHost(), iPort, bTLS);
280
void WebDownloader::SendHeaders(URL *pUrl)
285
snprintf(tmp, 1024, "GET %s HTTP/1.0\r\n", pUrl->GetResource());
287
m_pConnection->WriteLine(tmp);
289
snprintf(tmp, 1024, "User-Agent: nzbget/%s\r\n", Util::VersionRevision());
291
m_pConnection->WriteLine(tmp);
293
snprintf(tmp, 1024, "Host: %s\r\n", pUrl->GetHost());
295
m_pConnection->WriteLine(tmp);
297
m_pConnection->WriteLine("Accept: */*\r\n");
299
m_pConnection->WriteLine("Accept-Encoding: gzip\r\n");
301
m_pConnection->WriteLine("Connection: close\r\n");
302
m_pConnection->WriteLine("\r\n");
305
WebDownloader::EStatus WebDownloader::DownloadHeaders()
307
EStatus Status = adRunning;
309
m_bConfirmedLength = false;
310
const int LineBufSize = 1024*10;
311
char* szLineBuf = (char*)malloc(LineBufSize);
313
bool bFirstLine = true;
319
SetLastUpdateTimeNow();
322
char* line = m_pConnection->ReadLine(szLineBuf, LineBufSize, &iLen);
326
Status = CheckResponse(szLineBuf);
327
if (Status != adRunning)
334
// Have we encountered a timeout?
339
warn("URL %s failed: Unexpected end of file", m_szInfoName);
345
debug("Header: %s", line);
347
// detect body of response
348
if (*line == '\r' || *line == '\n')
350
if (m_iContentLen == -1 && !m_bGZip)
352
warn("URL %s: Content-Length is not submitted by server, cannot verify whether the file is complete", m_szInfoName);
365
WebDownloader::EStatus WebDownloader::DownloadBody()
367
EStatus Status = adRunning;
371
const int LineBufSize = 1024*10;
372
char* szLineBuf = (char*)malloc(LineBufSize);
376
m_pGUnzipStream = NULL;
379
m_pGUnzipStream = new GUnzipStream(1024*10);
386
SetLastUpdateTimeNow();
390
m_pConnection->ReadBuffer(&szBuffer, &iLen);
393
iLen = m_pConnection->TryRecv(szLineBuf, LineBufSize);
394
szBuffer = szLineBuf;
397
// Have we encountered a timeout?
400
if (m_iContentLen == -1)
408
warn("URL %s failed: Unexpected end of file", m_szInfoName);
414
// write to output file
415
if (!Write(szBuffer, iLen))
417
Status = adFatalError;
423
if (iWrittenLen == m_iContentLen || (m_iContentLen == -1 && m_bGZip && m_bConfirmedLength))
435
delete m_pGUnzipStream;
444
if (!bEnd && Status == adRunning && !IsStopped())
446
warn("URL %s failed: file incomplete", m_szInfoName);
458
WebDownloader::EStatus WebDownloader::CheckResponse(const char* szResponse)
464
warn("URL %s: Connection closed by remote host", m_szInfoName);
466
return adConnectError;
469
const char* szHTTPResponse = strchr(szResponse, ' ');
470
if (strncmp(szResponse, "HTTP", 4) || !szHTTPResponse)
472
warn("URL %s failed: %s", m_szInfoName, szResponse);
478
if (!strncmp(szHTTPResponse, "400", 3) || !strncmp(szHTTPResponse, "499", 3))
480
warn("URL %s failed: %s", m_szInfoName, szHTTPResponse);
481
return adConnectError;
483
else if (!strncmp(szHTTPResponse, "404", 3))
485
warn("URL %s failed: %s", m_szInfoName, szHTTPResponse);
488
else if (!strncmp(szHTTPResponse, "200", 3))
495
// unknown error, no special handling
496
warn("URL %s failed: %s", m_szInfoName, szResponse);
501
void WebDownloader::ProcessHeader(const char* szLine)
503
if (!strncmp(szLine, "Content-Length: ", 16))
505
m_iContentLen = atoi(szLine + 16);
506
m_bConfirmedLength = true;
509
if (!strncmp(szLine, "Content-Encoding: gzip", 22))
514
if (!strncmp(szLine, "Content-Disposition: ", 21))
516
ParseFilename(szLine);
520
void WebDownloader::ParseFilename(const char* szContentDisposition)
523
// Content-Disposition: attachment; filename="fname.ext"
524
// Content-Disposition: attachement;filename=fname.ext
525
// Content-Disposition: attachement;filename=fname.ext;
526
const char *p = strstr(szContentDisposition, "filename");
540
while (*p == ' ') p++;
543
strncpy(fname, p, 1024);
544
fname[1024-1] = '\0';
546
char *pe = fname + strlen(fname) - 1;
547
while ((*pe == ' ' || *pe == '\n' || *pe == '\r' || *pe == ';') && pe > fname) {
552
WebUtil::HttpUnquote(fname);
554
if (m_szOriginalFilename)
556
free(m_szOriginalFilename);
558
m_szOriginalFilename = strdup(Util::BaseFileName(fname));
560
debug("OriginalFilename: %s", m_szOriginalFilename);
563
bool WebDownloader::Write(void* pBuffer, int iLen)
565
if (!m_pOutFile && !PrepareFile())
573
m_pGUnzipStream->Write(pBuffer, iLen);
578
GUnzipStream::EStatus eGZStatus = m_pGUnzipStream->Read(&pOutBuf, &iOutLen);
580
if (eGZStatus == GUnzipStream::zlError)
582
error("URL %s: GUnzip failed", m_szInfoName);
586
if (iOutLen > 0 && fwrite(pOutBuf, 1, iOutLen, m_pOutFile) <= 0)
591
if (eGZStatus == GUnzipStream::zlFinished)
593
m_bConfirmedLength = true;
602
return fwrite(pBuffer, 1, iLen, m_pOutFile) > 0;
605
bool WebDownloader::PrepareFile()
607
// prepare file for writing
609
const char* szFilename = m_szOutputFilename;
610
m_pOutFile = fopen(szFilename, "wb");
613
error("Could not %s file %s", "create", szFilename);
616
if (g_pOptions->GetWriteBufferSize() > 0)
618
setvbuf(m_pOutFile, (char *)NULL, _IOFBF, g_pOptions->GetWriteBufferSize());
624
void WebDownloader::LogDebugInfo()
627
#ifdef HAVE_CTIME_R_3
628
ctime_r(&m_tLastUpdateTime, szTime, 50);
630
ctime_r(&m_tLastUpdateTime, szTime);
633
debug(" Web-Download: status=%i, LastUpdateTime=%s, filename=%s", m_eStatus, szTime, Util::BaseFileName(m_szOutputFilename));
636
void WebDownloader::Stop()
638
debug("Trying to stop WebDownloader");
640
m_mutexConnection.Lock();
643
m_pConnection->SetSuppressErrors(true);
644
m_pConnection->Cancel();
646
m_mutexConnection.Unlock();
647
debug("WebDownloader stopped successfully");
650
bool WebDownloader::Terminate()
652
Connection* pConnection = m_pConnection;
653
bool terminated = Kill();
654
if (terminated && pConnection)
656
debug("Terminating connection");
657
pConnection->SetSuppressErrors(true);
658
pConnection->Cancel();
659
pConnection->Disconnect();
665
void WebDownloader::FreeConnection()
669
debug("Releasing connection");
670
m_mutexConnection.Lock();
671
if (m_pConnection->GetStatus() == Connection::csCancelled)
673
m_pConnection->Disconnect();
675
delete m_pConnection;
676
m_pConnection = NULL;
677
m_mutexConnection.Unlock();
added
removed
WebDownloader.cpp
