« back to all changes in this revision
Viewing changes to index/Index.cpp
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Bazaar Package Importer
- Author(s): Jonas Meurer
- Date: 2004-09-26 16:27:51 UTC
- Revision ID: james.westby@ubuntu.com-20040926162751-z1ohcjltv7ojtg6z
Tags: upstream-1.2
Import upstream version 1.2
- files added:
- common
- imgs
- index
- libesort
- prune
- render
- tools
- ui
added
removed
index/Index.cpp
1
/* $Id: Index.cpp,v 1.31 2004/08/27 17:53:44 terpstra Exp $
2
*
3
* index.cpp - Insert all the keywords from the given email
4
*
5
* Copyright (C) 2002 - Wesley W. Terpstra
6
*
7
* License: GPL
8
*
9
* Authors: 'Wesley W. Terpstra' <wesley@terpstra.ca>
10
*
11
* This program is free software; you can redistribute it and/or modify
12
* it under the terms of the GNU General Public License as published by
13
* the Free Software Foundation; version 2.
14
*
15
* This program is distributed in the hope that it will be useful,
16
* but WITHOUT ANY WARRANTY; without even the implied warranty of
17
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18
* GNU General Public License for more details.
19
*
20
* You should have received a copy of the GNU General Public License
21
* along with this program; if not, write to the Free Software
22
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23
*/
24
25
#define _XOPEN_SOURCE 500
26
#define _FILE_OFFSET_BITS 64
27
28
#include <mimelib/headers.h>
29
#include <mimelib/datetime.h>
30
#include <mimelib/addrlist.h>
31
#include <mimelib/address.h>
32
#include <mimelib/group.h>
33
#include <mimelib/mboxlist.h>
34
#include <mimelib/mailbox.h>
35
#include <mimelib/text.h>
36
#include <mimelib/param.h>
37
#include <mimelib/enum.h>
38
#include <mimelib/body.h>
39
#include <mimelib/bodypart.h>
40
#include <mimelib/utility.h>
41
42
#include <CharsetEscape.h>
43
#include <Keys.h>
44
#include <md5.h>
45
46
#include "Index.h"
47
48
#include <string>
49
#include <vector>
50
#include <iostream>
51
52
#include <unistd.h>
53
#include <iconv.h>
54
#include <cerrno>
55
56
using namespace std;
57
58
#define MAX_MESSAGE_ID 80
59
60
void utf8Truncate(string& str, string::size_type len)
61
{
62
if (str.length() < len) return;
63
64
// look for nasty utf-8 stuff that's dangling and crop it
65
while (len && ((unsigned char)str[len-1]) >= 0x80 &&
66
((unsigned char)str[len-1]) <= 0xBF)
67
--len;
68
// now rewind off potential utf-8 start bytes
69
while (len && ((unsigned char)str[len-1]) >= 0xC0)
70
--len;
71
72
// len is now at the end of a complete multi-byte element or ascii
73
74
str.resize(len);
75
}
76
77
// first = address, second = name
78
pair<string, string> pickAddress(DwAddress* a, const char* charset)
79
{
80
for (; a != 0; a = a->Next())
81
{
82
if (a->IsGroup())
83
{
84
DwGroup* g = dynamic_cast<DwGroup*>(a);
85
if (g)
86
{
87
pair<string, string> out =
88
pickAddress(
89
g->MailboxList().FirstMailbox(),
90
charset);
91
if (out.first != "") return out;
92
}
93
}
94
else
95
{
96
DwMailbox* m = dynamic_cast<DwMailbox*>(a);
97
if (m)
98
{
99
string name = m->FullName().c_str();
100
name = decode_header(name, charset);
101
DwString addr = m->LocalPart() + "@" + m->Domain();
102
103
// fucked address? (one cannot safely cut this)
104
if (addr.length() > 128 ||
105
m->LocalPart() == "" || m->Domain() == "")
106
{
107
addr = "";
108
}
109
110
for (size_t i = 0; i < addr.length(); ++i)
111
{
112
if (addr[i] <= 0x20 || addr[i] >= 0x7f)
113
{ // fucked up address
114
addr = "";
115
break;
116
}
117
}
118
119
// prune any optional quotes
120
if (name.length() >= 2 && name[0] == '"')
121
name = name.substr(1, name.length()-2);
122
123
if (addr != "")
124
return pair<string, string>(addr.c_str(), name);
125
}
126
}
127
}
128
129
return pair<string, string>("", "");
130
}
131
132
int Index::index_author()
133
{
134
// one always has headers, but not always this function:
135
// if (message.hasHeaders())
136
137
charset = "ISO-8859-1"; // a good default as any
138
139
if (message.Headers().HasContentType())
140
{
141
DwParameter* p = message.Headers().ContentType().FirstParameter();
142
while (p)
143
{
144
if (p->Attribute() == "charset")
145
charset = p->Value().c_str();
146
p = p->Next();
147
}
148
}
149
150
// pickAddress only gives an author_name if it gave an author_email
151
152
if (message.Headers().HasReplyTo())
153
{
154
pair<string, string> addr = pickAddress(
155
message.Headers().ReplyTo().FirstAddress(),
156
charset.c_str());
157
158
author_email = addr.first;
159
author_name = addr.second;
160
161
// Some evil mailing lists set reply-to the list.
162
if (author_email == list.address)
163
{
164
author_email = "";
165
author_name = "";
166
}
167
}
168
169
// Given a reply-to that is not the list, we allow the from to
170
// provide a fullname under the assumption it is the same person.
171
172
if (message.Headers().HasFrom())
173
{
174
pair<string, string> addr = pickAddress(
175
message.Headers().From().FirstMailbox(),
176
charset.c_str());
177
178
if (!author_email.length()) author_email = addr.first;
179
if (!author_name .length()) author_name = addr.second;
180
}
181
182
// ditto
183
184
if (message.Headers().HasSender())
185
{
186
pair<string, string> addr = pickAddress(
187
&message.Headers().Sender(),
188
charset.c_str());
189
190
if (!author_email.length()) author_email = addr.first;
191
if (!author_name .length()) author_name = addr.second;
192
}
193
194
utf8Truncate(author_name, 100);
195
// - nothing longer than 128 could get here (from above)
196
// - one can never safely truncate an email address
197
// utf8Truncate(author_email, 100);
198
199
return 0;
200
}
201
202
// Doesn't vary with charset
203
inline bool lu_isspace(char x)
204
{
205
return x == ' ' || x == '\n' || x == '\r' || x == '\t';
206
}
207
208
void build_message_hash(const char* str, unsigned char* hash)
209
{
210
MD5Context ctx;
211
212
MD5Init(&ctx);
213
MD5Update(&ctx, (const unsigned char*)str, strlen(str));
214
215
unsigned char buf[16];
216
MD5Final(buf, &ctx);
217
218
hash[0] = buf[0] ^ buf[4] ^ buf[ 8] ^ buf[12];
219
hash[1] = buf[1] ^ buf[5] ^ buf[ 9] ^ buf[13];
220
hash[2] = buf[2] ^ buf[6] ^ buf[10] ^ buf[14];
221
hash[3] = buf[3] ^ buf[7] ^ buf[11] ^ buf[15];
222
}
223
224
int feed_writer(const char* keyword, void* arg)
225
{
226
Index* i = (Index*)arg;
227
228
string x(LU_KEYWORD);
229
x += keyword;
230
x += '\0';
231
x += i->id.raw();
232
233
return i->writer->insert(x);
234
}
235
236
int Index::index_id(bool userdate, time_t server)
237
{
238
time_t stamp = server;
239
string messageId;
240
unsigned char hash[4];
241
242
// if (message.hasHeaders())
243
244
if (message.Headers().HasDate())
245
{
246
time_t user = message.Headers().Date().AsUnixTime();
247
248
/* User time must be earlier; there is delivery delay!
249
* However, more than 7 day delivery time is unlikely.
250
*/
251
if ((user <= server && server < user+7*60*60*24) ||
252
userdate || // trusting the userdate?
253
server <= 0) // server is on crack?
254
stamp = user;
255
}
256
257
if (stamp <= 0)
258
{ // this is crazy; I don't care if they agree: it's wrong
259
stamp = 1; // liers all have timestamp 1970-01-01 00:00:01
260
}
261
262
if (message.Headers().HasMessageId())
263
{
264
vector<string> ids = extract_message_ids(
265
message.Headers().MessageId().AsString().c_str());
266
267
if (!ids.empty())
268
messageId = ids.front();
269
}
270
271
if (messageId.length())
272
{
273
// Constant message-id across import, and threadable
274
build_message_hash(messageId.c_str(), hash);
275
}
276
else if (author_email.length())
277
{
278
// This means no proper threading.
279
// At least the message-id is constant across import.
280
build_message_hash(author_email.c_str(), hash);
281
}
282
else
283
{
284
// Can't make any guarantees; just import it.
285
hash[0] = random() % 256;
286
hash[1] = random() % 256;
287
hash[2] = random() % 256;
288
hash[3] = random() % 256;
289
}
290
291
id = MessageId(stamp, hash);
292
293
if (messageId.length() && writer->insert(
294
LU_KEYWORD +
295
string(LU_KEYWORD_MESSAGE_ID) +
296
messageId +
297
'\0' +
298
id.raw()) != 0)
299
{
300
cerr << "Failed to insert message id keyword!" << endl;
301
return -1;
302
}
303
304
if (writer->insert(
305
LU_KEYWORD +
306
string(LU_KEYWORD_EVERYTHING) +
307
'\0' +
308
id.raw()) != 0)
309
{
310
cerr << "Failed to the any keyword!" << endl;
311
return -1;
312
}
313
314
return 0;
315
}
316
317
int Index::index_summary(bool check, bool& exist)
318
{
319
string prefix = LU_SUMMARY + id.raw();
320
321
if (message.Headers().HasSubject())
322
{
323
subject = message.Headers().Subject().AsString().c_str();
324
subject = decode_header(subject, charset.c_str());
325
}
326
327
if (subject == "")
328
subject = "[...]";
329
330
string mbox = prefix + LU_MESSAGE_MBOX + list.mbox + '\0';
331
332
if (check)
333
{
334
// Check for existance
335
auto_ptr<ESort::Walker> w(writer->seek(mbox, "", ESort::Forward));
336
337
if (w->advance() == -1)
338
{ // was it just eof?
339
if (errno != 0) return -1;
340
}
341
else
342
{ // if it suceeded. then ... it is already in there
343
exist = true;
344
return 0;
345
}
346
}
347
348
unsigned char buf[12];
349
off_t o = off;
350
long l = len;
351
int i;
352
353
for (i = 7; i >= 0; --i)
354
{
355
buf[i] = (o & 0xFF);
356
o >>= 8;
357
}
358
for (i = 11; i >= 8; --i)
359
{
360
buf[i] = (l & 0xFF);
361
l >>= 8;
362
}
363
364
// Don't let crazy stuff in there.
365
utf8Truncate(subject, 200);
366
367
if (writer->insert(prefix + LU_MESSAGE_AUTHOR_EMAIL + author_email) != 0 ||
368
writer->insert(prefix + LU_MESSAGE_AUTHOR_NAME + author_name) != 0 ||
369
writer->insert(prefix + LU_MESSAGE_SUBJECT + subject) != 0 ||
370
writer->insert(mbox + string((char*)buf, 12)) != 0)
371
{
372
cerr << "Failed to insert summary keys" << endl;
373
return -1;
374
}
375
376
return 0;
377
}
378
379
int Index::index_threading()
380
{
381
string shash = subject_hash(subject.c_str());
382
string suffix;
383
384
unsigned char hash[4];
385
386
if (writer->insert(
387
LU_KEYWORD
388
LU_KEYWORD_THREAD +
389
shash +
390
'\0' +
391
id.raw()) != 0)
392
{
393
cerr << "Failed to insert threading keyword" << endl;
394
return -1;
395
}
396
397
// if (message.hasHeaders())
398
399
if (message.Headers().HasInReplyTo())
400
{
401
vector<string> ids = extract_message_ids(
402
message.Headers().InReplyTo().AsString().c_str());
403
404
// first in-reply-to is most relevant
405
for (vector<string>::iterator i = ids.begin(); i != ids.end(); ++i)
406
{
407
build_message_hash(i->c_str(), hash);
408
409
// keep it reasonable; too many reply-tos is bad
410
if (suffix.length() < 200)
411
suffix.append((const char*)hash, 4);
412
}
413
}
414
415
if (message.Headers().HasReferences())
416
{
417
vector<string> ids = extract_message_ids(
418
message.Headers().References().AsString().c_str());
419
420
// last references is most recently added (most likely irt)
421
for (vector<string>::reverse_iterator i = ids.rbegin();
422
i != ids.rend(); ++i)
423
{
424
build_message_hash(i->c_str(), hash);
425
// keep it reasonable; too many reply-tos is bad
426
if (suffix.length() < 200)
427
suffix.append((const char*)hash, 4);
428
}
429
}
430
431
if (writer->insert(
432
LU_THREADING
433
+ shash
434
+ id.raw()
435
+ suffix) != 0)
436
{
437
cerr << "Failed to insert threading keys" << endl;
438
return -1;
439
}
440
441
if (writer->insert(
442
LU_NEW_TOPICS
443
+ list.mbox + '\0'
444
+ id.raw().substr(0, 4)
445
+ shash) != 0)
446
{
447
cerr << "Failed to insert new topics keys" << endl;
448
return -1;
449
}
450
451
return 0;
452
}
453
454
int Index::index_control(time_t import)
455
{
456
bool ok = true;
457
if (writer->insert(
458
LU_KEYWORD
459
LU_KEYWORD_LIST +
460
list.mbox +
461
'\0' +
462
id.raw()) != 0) ok = false;
463
464
/* emulated group and language searches are impossibly slow.
465
* these keywords are a must for large archives.
466
* see the regroupable option in the stock lurker.conf
467
*/
468
if (writer->insert(
469
LU_KEYWORD
470
LU_KEYWORD_GROUP +
471
list.group +
472
'\0' +
473
id.raw()) != 0) ok = false;
474
475
if (writer->insert(
476
LU_KEYWORD
477
LU_KEYWORD_LANGUAGE +
478
list.language +
479
'\0' +
480
id.raw()) != 0) ok = false;
481
482
MessageId importStamp(import);
483
if (writer->insert(
484
LU_CACHE +
485
importStamp.raw().substr(0, 4) +
486
id.raw()) != 0) ok = false;
487
488
if (author_email.length())
489
{
490
if (my_keyword_digest_string(
491
author_email.c_str(), author_email.length(),
492
LU_KEYWORD_AUTHOR, &feed_writer, this, 1) != 0)
493
ok = false;
494
}
495
496
if (author_name.length())
497
{
498
if (my_keyword_digest_string(
499
author_name.c_str(), author_name.length(),
500
LU_KEYWORD_AUTHOR, &feed_writer, this, 1) != 0)
501
ok = false;
502
}
503
504
if (subject.length())
505
{
506
if (my_keyword_digest_string(
507
subject.c_str(), subject.length(),
508
LU_KEYWORD_SUBJECT, &feed_writer, this, 1) != 0)
509
ok = false;
510
}
511
512
if (message.Headers().HasInReplyTo())
513
{
514
vector<string> ids = extract_message_ids(
515
message.Headers().InReplyTo().AsString().c_str());
516
for (vector<string>::iterator i = ids.begin(); i != ids.end(); ++i)
517
if (writer->insert(
518
LU_KEYWORD
519
LU_KEYWORD_REPLY_TO +
520
*i + '\0' + id.raw()) != 0)
521
ok = false;
522
}
523
524
#if 0 // this is questionable...
525
if (message.Headers().HasReferences())
526
{
527
vector<string> ids = extract_message_ids(
528
message.Headers().References().AsString().c_str());
529
for (vector<string>::iterator i = ids.begin(); i != ids.end(); ++i)
530
if (writer->insert(
531
LU_KEYWORD
532
LU_KEYWORD_REPLY_TO +
533
*i + '\0' + id.raw()) != 0)
534
ok = false;
535
}
536
#endif
537
538
if (!ok)
539
{
540
cerr << "Failed to insert control keys" << endl;
541
return -1;
542
}
543
544
return 0;
545
}
546
547
int Index::index_entity(DwEntity& e, const string& charset)
548
{
549
DwString text;
550
if (e.Headers().HasContentTransferEncoding())
551
{
552
switch (e.Headers().ContentTransferEncoding().AsEnum())
553
{
554
case DwMime::kCteQuotedPrintable:
555
DwDecodeQuotedPrintable(e.Body().AsString(), text);
556
break;
557
558
case DwMime::kCteBase64:
559
DwDecodeBase64(e.Body().AsString(), text);
560
break;
561
562
case DwMime::kCteNull:
563
case DwMime::kCteUnknown:
564
case DwMime::kCte7bit:
565
case DwMime::kCte8bit:
566
case DwMime::kCteBinary:
567
text = e.Body().AsString();
568
break;
569
}
570
}
571
else
572
{
573
text = e.Body().AsString();
574
}
575
576
CharsetEscape decode(charset.c_str());
577
string utf8 = decode.write(text.c_str(), text.length());
578
579
if (my_keyword_digest_string(
580
utf8.c_str(), utf8.length(),
581
LU_KEYWORD_WORD, &feed_writer, this, 1) != 0)
582
{
583
cerr << "Failed to index un-typed segment" << endl;
584
return -1;
585
}
586
587
return 0;
588
}
589
590
int Index::index_keywords(DwEntity& e, const string& parentCharset)
591
{
592
string charset = parentCharset;
593
594
if (e.Headers().HasContentType())
595
{
596
DwMediaType& mt = e.Headers().ContentType();
597
598
for (DwParameter* p = mt.FirstParameter(); p; p = p->Next())
599
{
600
DwString attr = p->Attribute();
601
attr.ConvertToLowerCase(); // case insens
602
if (attr == "charset") charset = p->Value().c_str();
603
}
604
}
605
606
// if (e.hasHeaders() &&
607
if (e.Headers().HasContentType())
608
{
609
DwMediaType& t = e.Headers().ContentType();
610
switch (t.Type())
611
{
612
case DwMime::kTypeMessage:
613
if (e.Body().Message())
614
index_keywords(*e.Body().Message(), charset);
615
break;
616
617
case DwMime::kTypeMultipart:
618
// index all alternatives in multipart
619
for (DwBodyPart* p = e.Body().FirstBodyPart(); p != 0; p = p->Next())
620
index_keywords(*p, charset);
621
break;
622
623
case DwMime::kTypeText:
624
if (t.Subtype() == DwMime::kSubtypePlain)
625
{
626
if (index_entity(e, charset) != 0) return -1;
627
}
628
break;
629
}
630
}
631
else
632
{
633
if (index_entity(e, charset) != 0) return -1;
634
}
635
636
return 0;
637
}
638
639
int Index::index(bool userdate, time_t envelope, time_t import, bool check, bool& exist)
640
{
641
exist = false;
642
643
// cout << message.Headers().Subject().AsString().c_str() << endl;
644
645
if (index_author() < 0) return -1;
646
if (index_id(userdate, envelope) < 0) return -1;
647
if (index_summary(check, exist) < 0) return -1;
648
649
if (exist) return 0;
650
651
if (index_threading( ) < 0) return -1;
652
if (index_control (import) < 0) return -1;
653
if (index_keywords (message, "ISO-8859-1") < 0) return -1;
654
655
return 0;
656
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1