← Back to branch summary

~ubuntu-branches/ubuntu/quantal/apt-cacher-ng/quantal-backports

« back to all changes in this revision

Viewing changes to source/mirror.cc

  • Committer: Bazaar Package Importer
  • Author(s): Eduard Bloch
  • Date: 2011-08-01 23:40:11 UTC
  • mfrom: (1.1.29 upstream) (29.1.13 sid)
  • Revision ID: james.westby@ubuntu.com-20110801234011-2ouft5pay71773vs
Tags: 0.6.4-1
http://bugs.debian.org/628995
http://bugs.debian.org/616091
* New upstream version
  + fixes potential pipeline freeze (closes: #628995)
  + supports "soft" blacklisting of bad hosts when important files are
    missing (see keyfile in documentation, closes: #616091)

Show diffs side-by-side

added added

removed removed

Lines of Context:
source/mirror.cc
 
1
/*
 
2
 * mirror.cpp
 
3
 *
 
4
 *  Created on: 25.11.2010
 
5
 *      Author: ed
 
6
 */
 
7
 
 
8
 
 
9
#define LOCAL_DEBUG
 
10
#include "debug.h"
 
11
 
 
12
#include "mirror.h"
 
13
#include "header.h"
 
14
#include "dirwalk.h"
 
15
#include "meta.h"
 
16
#include "acfg.h"
 
17
 
 
18
#include <fnmatch.h>
 
19
 
 
20
pkgmirror::pkgmirror(int fd): tCacheMan(fd),
 
21
m_bCalcSize(false), m_bSkipIxUpdate(false), m_bDoDownload(false),
 
22
m_totalneed(0)
 
23
{
 
24
        m_sTypeName="Mirroring";
 
25
}
 
26
 
 
27
pkgmirror::~pkgmirror()
 
28
{
 
29
}
 
30
 
 
31
bool pkgmirror::ProcessRegular(const string &sPath, const struct stat &stinfo)
 
32
{
 
33
        if (endsWithSzAr(sPath, ".head"))
 
34
                return true;
 
35
 
 
36
        if (sPath.size() <= CACHE_BASE_LEN) // heh?
 
37
                return false;
 
38
 
 
39
        if (0 == sPath.compare(CACHE_BASE_LEN, 1, "_"))
 
40
                return true; // not for us, also includes _import
 
41
 
 
42
        AddIFileCandidate(sPath.substr(CACHE_BASE_LEN));
 
43
 
 
44
        return ! CheckAbortCondition();
 
45
}
 
46
 
 
47
tStrDeq GetVariants(cmstring &mine)
 
48
{
 
49
        tStrDeq ret;
 
50
        string base;
 
51
        for(cmstring *p=suxeWempty; p<suxeWempty+_countof(suxeWempty); p++)
 
52
        {
 
53
                if(endsWith(mine, *p))
 
54
                {
 
55
                        base=mine.substr(0, mine.size()-p->size());
 
56
                        break;
 
57
                }
 
58
        }
 
59
        for(cmstring *p=suxeWempty; p<suxeWempty+_countof(suxeWempty); p++)
 
60
        {
 
61
                string cand=base+*p;
 
62
                if(cand!=mine)
 
63
                        ret.push_back(cand);
 
64
        }
 
65
        return ret;
 
66
}
 
67
 
 
68
void pkgmirror::Action(const string &cmd)
 
69
{
 
70
        SendChunk("<b>Locating index files, scanning...</b><br>\n");
 
71
 
 
72
        SetCommonUserFlags(cmd);
 
73
        m_bErrAbort=false; // does not f...ing matter, do what we can
 
74
 
 
75
        m_bCalcSize=(cmd.find("calcSize=cs")!=stmiss);
 
76
        m_bDoDownload=(cmd.find("doDownload=dd")!=stmiss);
 
77
        m_bSkipIxUpdate=(cmd.find("skipIxUp=si")!=stmiss);
 
78
 
 
79
        DirectoryWalk(acfg::cachedir, this);
 
80
 
 
81
        if(CheckAbortCondition())
 
82
                return;
 
83
 
 
84
        if(m_indexFilesRel.empty())
 
85
        {
 
86
                SendChunk("<font size=0 color=red>No index files detected. Unable to continue, cannot map files to internal locations.</font>");
 
87
                return;
 
88
        }
 
89
 
 
90
        if(CheckAbortCondition())
 
91
                return;
 
92
 
 
93
        if(!m_bSkipIxUpdate)
 
94
                UpdateIndexFiles();
 
95
 
 
96
 
 
97
        if(CheckAbortCondition())
 
98
                return;
 
99
 
 
100
        tStrSet srcs;
 
101
        class __srcpicker : public ifileprocessor
 
102
        {
 
103
        public:
 
104
                tStrSet *pSrcs;
 
105
                tStrVec matchList;
 
106
                __srcpicker(tStrSet *x) : pSrcs(x)
 
107
                {
 
108
                        Tokenize(acfg::mirrorsrcs, SPACECHARS, matchList);
 
109
                };
 
110
                void TryAdd(cmstring &s)
 
111
                {
 
112
                        for(tStrVecIterConst it=matchList.begin(); it!=matchList.end(); it++)
 
113
                                if(0==fnmatch(it->c_str(), s.c_str(), FNM_PATHNAME))
 
114
                                {
 
115
                                        pSrcs->insert(s);
 
116
                                        break;
 
117
                                }
 
118
                }
 
119
                void HandlePkgEntry(const tRemoteFileInfo &entry, bool)
 
120
                {
 
121
                        TryAdd(entry.sDirectory+entry.sFileName);
 
122
                }
 
123
        } picker(&srcs);
 
124
 
 
125
        mstring sErr;
 
126
 
 
127
        for(tS2IDX::iterator it=m_indexFilesRel.begin(); it!=m_indexFilesRel.end();it++)
 
128
        {
 
129
                if(endsWithSzAr(it->first, "Release"))
 
130
                {
 
131
                        if(!GetFlags(it->first).uptodate)
 
132
                                if(!DownloadIdxFile(it->first, sErr))
 
133
                                        SendFmt() << "<font color=\"red\">" << sErr << "</font><br>\n";
 
134
                        ParseAndProcessIndexFile(picker, it->first, EIDX_RELEASE);
 
135
                }
 
136
                else
 
137
                        picker.TryAdd(it->first);
 
138
        }
 
139
 
 
140
        SendChunk("<b>Identifying relevant index files...</b><br>");
 
141
 
 
142
        // unless found in release files, get the from the local system
 
143
        for (tStrVecIterConst it = picker.matchList.begin(); it != picker.matchList.end(); it++)
 
144
        {
 
145
                tStrDeq paths(ExpandFilePattern(CACHE_BASE+it->c_str(), false));
 
146
                for(tStrDeq::iterator j=paths.begin(); j!=paths.end(); j++)
 
147
                        picker.TryAdd(*j);
 
148
        }
 
149
 
 
150
        restart_clean: // start over if the set changed while having a hot iterator
 
151
        for(tStrSet::iterator it=srcs.begin(); it!=srcs.end(); it++)
 
152
        {
 
153
                if(GetFlags(*it).uptodate) // this is the one
 
154
                {
 
155
                tStrDeq bros(GetVariants(*it));
 
156
                int nDeleted=0;
 
157
                for(tStrDeq::iterator b=bros.begin(); b!=bros.end(); b++)
 
158
                        nDeleted+=srcs.erase(*b);
 
159
                if(nDeleted)
 
160
                        goto restart_clean;
 
161
                }
 
162
        }
 
163
 
 
164
        // now there may still be something like Sources and Sources.bz2 if they
 
165
        // were added by Release file scan. Choose the prefered simply by extension.
 
166
 
 
167
        restart_clean2: // start over if the set changed while having a hot iterator
 
168
        for (const string *p = suxeByCompSize; p < suxeByCompSize
 
169
        + _countof(suxeByCompSize); p++)
 
170
        {
 
171
                for (tStrSet::iterator it = srcs.begin(); it != srcs.end(); it++)
 
172
                {
 
173
                        if(endsWith(*it, *p))
 
174
                        {
 
175
                                tStrDeq bros(GetVariants(*it));
 
176
                        int nDeleted=0;
 
177
                        for(tStrDeq::iterator b=bros.begin(); b!=bros.end(); b++)
 
178
                                nDeleted+=srcs.erase(*b);
 
179
                        if(nDeleted)
 
180
                                goto restart_clean2;
 
181
                        }
 
182
                }
 
183
        }
 
184
 
 
185
        for (tStrSet::iterator it=srcs.begin(); it!=srcs.end(); it++)
 
186
        {
 
187
                SendFmt() << "File list: " << *it<< "<br>";
 
188
                if(!GetFlags(*it).uptodate)
 
189
                        DownloadIdxFile(*it, sErr);
 
190
 
 
191
                if(CheckAbortCondition())
 
192
                        return;
 
193
        }
 
194
 
 
195
        off_t total=0;
 
196
 
 
197
        if (m_bCalcSize)
 
198
        {
 
199
 
 
200
                SendChunk("<b>Counting downloadable content size...</b><br>");
 
201
                for (tStrSet::iterator it = srcs.begin(); it != srcs.end(); it++)
 
202
                {
 
203
                        class __cnt: public ifileprocessor
 
204
                        {
 
205
                        public:
 
206
                                __cnt() :
 
207
                                        sum(0), have(0)
 
208
                                {
 
209
                                }
 
210
                                ;
 
211
                                off_t sum, have;
 
212
                                void HandlePkgEntry(const tRemoteFileInfo &entry, bool)
 
213
                                {
 
214
                                        sum += entry.fpr.size;
 
215
                                        have += GetFileSize(CACHE_BASE + entry.sDirectory + entry.sFileName, 0);
 
216
                                        //off_t have=GetFileSize(CACHE_BASE+entry.sDirectory+entry.sFileName, 0);
 
217
                                        //sum+=(entry.fpr.size-have);
 
218
                                }
 
219
                        } counter;
 
220
 
 
221
                        ParseAndProcessIndexFile(counter, *it, GuessIndexType(*it));
 
222
 
 
223
                        SendFmt() << *it << ": " << offttosH(counter.sum - counter.have) << "/" << offttosH(
 
224
                                        counter.sum) << "<br>\n";
 
225
                        total += counter.sum;
 
226
                        m_totalneed += (counter.sum - counter.have);
 
227
 
 
228
                        if(CheckAbortCondition())
 
229
                                return;
 
230
                }
 
231
                SendFmt() << "Total size: " << offttosH(total) << ", missing: up to "
 
232
                                << offttosH(m_totalneed) << "<br>\n";
 
233
        }
 
234
 
 
235
        if(m_bDoDownload && (m_totalneed>0 || !m_bCalcSize))
 
236
        {
 
237
                SendChunk("<b>Starting download...</b><br>");
 
238
 
 
239
                for (tStrSet::iterator it=srcs.begin(); it!=srcs.end(); it++)
 
240
                {
 
241
                        if(CheckAbortCondition())
 
242
                                return;
 
243
 
 
244
                        ParseAndProcessIndexFile(*this, *it, GuessIndexType(*it));
 
245
                }
 
246
        }
 
247
 
 
248
}
 
249
 
 
250
void pkgmirror::UpdateFingerprint(const mstring &sPathRel, off_t nOverrideSize,
 
251
                        uint8_t *pOverrideSha1, uint8_t *pOverrideMd5)
 
252
{
 
253
 
 
254
}
 
255
 
 
256
void pkgmirror::HandlePkgEntry(const tRemoteFileInfo &entry, bool bUncompressForChecksum)
 
257
{
 
258
        off_t davor(0);
 
259
        if(m_bVerbose)
 
260
                davor=GetFileSize(CACHE_BASE + entry.sDirectory + entry.sFileName, 0);
 
261
 
 
262
        DownloadSimple(entry.sDirectory+entry.sFileName);
 
263
 
 
264
        if(m_bVerbose)
 
265
        {
 
266
                off_t diff=(GetFileSize(CACHE_BASE + entry.sDirectory + entry.sFileName, 0)-davor);
 
267
                if(m_totalneed && diff)
 
268
                {
 
269
                        m_totalneed-=diff;
 
270
                        SendFmt() << "Remaining download size: " << offttosH(m_totalneed)<< "<br>\n";
 
271
                }
 
272
        }
 
273
}