1
/* This file is part of Strigi Desktop Search
3
* Copyright (C) 2007 Flavio Castelli <flavio.castelli@gmail.com>
5
* This library is free software; you can redistribute it and/or
6
* modify it under the terms of the GNU Library General Public
7
* License as published by the Free Software Foundation; either
8
* version 2 of the License, or (at your option) any later version.
10
* This library is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
* Library General Public License for more details.
15
* You should have received a copy of the GNU Library General Public License
16
* along with this library; see the file COPYING.LIB. If not, write to
17
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18
* Boston, MA 02110-1301, USA.
20
#include "fslistener.h"
22
#include "combinedindexmanager.h"
24
#include "eventlistenerqueue.h"
25
#include "filelister.h"
26
#include "indexreader.h"
27
#include "pollinglistener.h"
28
#include "../strigilogging.h"
31
#include <sys/resource.h>
32
#include <sys/select.h>
33
#include <sys/types.h>
39
using namespace Strigi;
43
* @param path string containing path to check
44
* Removes the terminating char to path.
45
* Under Windows that char is '\', '/' under *nix
47
string fixPath (string path)
49
if ( path.c_str() == NULL || path.length() == 0 )
55
size_t l= temp.length();
56
char* t = (char*)temp.c_str();
57
for (size_t i=0;i<l;i++){
61
temp[0] = tolower(temp.at(0));
66
if (temp[temp.length() - 1 ] == separator)
67
return temp.substr(0, temp.size() - 1);
73
void calculateDiff(set<string> actualDirs, set<string> reindexDirs,
74
set<string>& dirsDeleted,set<string>& dirsCreated)
76
set<string>::iterator iter;
77
stringstream strDirsDeleted;
78
stringstream strDirsCreated;
79
stringstream strDirsActual;
80
stringstream strDirsReindex;
82
strDirsDeleted << "Dirs deleted:";
83
strDirsCreated << "Dirs created:";
84
strDirsActual << "Actual dirs:";
85
strDirsReindex << "Reindex dirs:";
90
for (iter = actualDirs.begin(); iter != actualDirs.end(); ++iter)
91
strDirsActual << "\n\t-" << *iter;
93
for (iter = reindexDirs.begin(); iter != reindexDirs.end(); ++iter)
94
strDirsReindex << "\n\t-" << *iter;
96
STRIGI_LOG_DEBUG ("strigi.calculateDiff", strDirsActual.str())
97
STRIGI_LOG_DEBUG ("strigi.calculateDiff", strDirsReindex.str())
99
for (iter = actualDirs.begin(); iter != actualDirs.end(); ++iter) {
100
set<string>::iterator match = reindexDirs.find (*iter);
101
if (match == reindexDirs.end()) {
102
dirsDeleted.insert (*iter);
103
strDirsDeleted << "\n\t-" << *iter;
106
reindexDirs.erase (match);
109
for (iter = reindexDirs.begin(); iter != reindexDirs.end(); ++iter) {
110
dirsCreated.insert (*iter);
111
strDirsCreated << "\n\t-" << *iter;
114
STRIGI_LOG_DEBUG ("strigi.calculateDiff", strDirsCreated.str())
115
STRIGI_LOG_DEBUG ("strigi.calculateDiff", strDirsDeleted.str())
121
File(string name, time_t mtime)
130
bool operator< (const File& f1, const File& f2)
132
if (f1.m_name.compare(f2.m_name) < 0)
142
MatchFile( string name)
145
bool operator ()(const File& f)
146
{ return (m_name == f.m_name); }
149
FsEvent::FsEvent (const string path, const string name)
151
m_file = fixPath(path);
158
// improved multimap, stl multimap class doesn't provide a method that returns
159
// all the available keys
160
typedef map< string, set<File> > iMultimap;
163
FsListener::FsListener(const char* name, set<string>& indexedDirs)
164
: EventListener(name)
168
m_bInitialized = false;
169
m_bBootstrapped = false;
170
m_bReindexReq = false;
172
m_pollingListener = 0;
174
for (set<string>::iterator iter = indexedDirs.begin();
175
iter != indexedDirs.end(); ++iter)
176
m_indexedDirs.insert (fixPath (*iter));
178
STRIGI_MUTEX_INIT (&m_reindexLock);
181
FsListener::~FsListener()
183
if (m_pollingListener) {
184
m_pollingListener->stop();
185
delete m_pollingListener;
186
m_pollingListener = 0;
188
STRIGI_MUTEX_DESTROY (&m_reindexLock);
191
void* FsListener::run(void*)
193
while (getState() != Stopping) {
194
if (!m_bBootstrapped)
196
else if (reindexReq())
201
if (getState() == Working)
205
STRIGI_LOG_DEBUG ("strigi.FsListener.run",
206
string("exit state: ") + getStringState());
210
bool FsListener::reindexReq()
214
STRIGI_MUTEX_LOCK (&m_reindexLock);
216
STRIGI_MUTEX_UNLOCK (&m_reindexLock);
221
void FsListener::getReindexDirs(set<string>& reindexDirs)
223
STRIGI_MUTEX_LOCK (&m_reindexLock);
224
m_bReindexReq = false;
225
reindexDirs = m_reindexDirs;
226
m_reindexDirs.clear();
227
STRIGI_MUTEX_UNLOCK (&m_reindexLock);
230
void FsListener::bootstrap()
232
STRIGI_LOG_DEBUG ("strigi.FsListener.bootstrap","BOOTSTRAP INIT");
235
vector<Event*> events;
237
set<string> bootstrapDirs;
240
getReindexDirs ( bootstrapDirs);
242
bootstrapDirs = m_indexedDirs;
244
for (set<string>::iterator iter = bootstrapDirs.begin();
245
iter != bootstrapDirs.end(); ++iter)
247
toWatch.insert (*iter);
249
DirLister lister(m_pAnalyzerConfiguration);
251
vector<pair<string, struct stat> > dirs;
253
lister.startListing (*iter);
254
int ret = lister.nextDir(path, dirs);
260
vector<pair<string, struct stat> >::iterator iter;
262
for (iter = dirs.begin(); iter != dirs.end(); ++iter) {
263
struct stat stats = iter->second;
265
if (S_ISDIR(stats.st_mode)) { //dir
266
toWatch.insert (iter->first);
268
else if (S_ISREG(stats.st_mode)) { //file
269
iMultimap::iterator mIter = files.find (path);
270
File file (iter->first, stats.st_mtime);
271
if (mIter != files.end())
272
(mIter->second).insert (file);
276
files.insert(make_pair(path, temp));
281
ret = lister.nextDir(path, dirs);
286
msg << "there're " << files.size();
287
msg << " keys inside iMultimap";
288
STRIGI_LOG_DEBUG ("strigi.FsListener.bootstrap", msg.str())
290
for (iMultimap::iterator iter = files.begin(); iter != files.end(); ++iter)
292
map <string, time_t> indexedFiles;
294
string path = iter->first;
296
// retrieve all indexed files contained by path
297
m_pManager->indexReader()->getChildren (path, indexedFiles);
298
msg << "there're " << indexedFiles.size();
299
msg << " indexed files associated to dir " << path;
301
STRIGI_LOG_DEBUG ("strigi.FsListener.bootstrap", msg.str())
303
if (indexedFiles.empty()) {
304
string temp = path + '/';
306
m_pManager->indexReader()->getChildren (temp, indexedFiles);
307
msg << "there're " << indexedFiles.size();
308
msg << " indexed files associated to dir " << temp;
310
STRIGI_LOG_DEBUG ("strigi.FsListener.bootstrap", msg.str())
313
if (!indexedFiles.empty()) {
314
// find differences between fs files and indexed ones
315
map<string, time_t>::iterator it = indexedFiles.begin();
316
for ( ; it != indexedFiles.end(); it++) {
317
MatchFile finder (it->first);
318
set<File>::iterator match;
319
match = find_if( (iter->second).begin(),
320
(iter->second).end(), finder);
322
if (match == (iter->second).end()) {
323
// indexed file has been deleted from filesystem
324
events.push_back (new Event (Event::DELETED, iter->first));
326
else if ((*match).m_mtime > it->second) {
327
// file has been updated
328
events.push_back (new Event (Event::UPDATED, iter->first));
331
// file has not been modified since index time
332
(iter->second).erase( match);
340
for (set<File>::iterator it = (iter->second).begin();
341
it != (iter->second).end(); ++it)
344
events.push_back (new Event (Event::CREATED, file.m_name));
350
for (vector<Event*>::iterator iter = events.begin();
351
iter != events.end(); ++iter) {
358
if (events.size() > 0)
359
m_pEventQueue->addEvents (events);
361
addWatches (toWatch);
363
m_bBootstrapped = true;
364
m_indexedDirs = bootstrapDirs;
368
void FsListener::reindex()
370
STRIGI_LOG_DEBUG ("strigi.FsListener.reindex","REINDEX INIT");
372
if (m_pEventQueue == NULL) {
373
STRIGI_LOG_ERROR ("strigi.FsListener.reindex",
374
"m_pEventQueue == NULL!");
378
set<string> reindexDirs;
379
set<string> dirsDeleted;
380
set<string> dirsCreated;
381
set<string> dirsMonitored;
382
vector<Event*> events;
387
getReindexDirs(reindexDirs);
389
calculateDiff(m_indexedDirs, reindexDirs, dirsDeleted, dirsCreated);
391
for (set<string>::iterator iter = dirsCreated.begin();
392
iter != dirsCreated.end() && !reindexReq(); ++iter)
394
DirLister lister(m_pAnalyzerConfiguration);
396
vector<pair<string, struct stat> > dirs;
398
lister.startListing (*iter);
399
int ret = lister.nextDir(path, dirs);
402
vector<pair<string, struct stat> >::iterator iter;
403
for (iter = dirs.begin(); iter != dirs.end(); iter++) {
404
struct stat stats = iter->second;
405
if (S_ISDIR(stats.st_mode)) {//dir
407
recursivelyMonitor (iter->first, toWatch, events);
409
addWatches (toWatch);
410
dirsMonitored.insert (iter->first);
412
else if (S_ISREG(stats.st_mode)) {
414
events.push_back (new Event (Event::CREATED, iter->first));
417
ret = lister.nextDir(path, dirs);
422
// another reindex request arrived, undo last actions
423
for (vector<Event*>::iterator iter = events.begin();
424
iter != events.end(); ++iter)
428
dirsRemoved (dirsMonitored, events);
431
// finish reindex operation
432
dirsRemoved (dirsDeleted, events);
434
if (events.size() > 0)
435
m_pEventQueue->addEvents (events);
438
m_indexedDirs = reindexDirs;
442
void FsListener::watch ()
444
if (m_pEventQueue == NULL) {
445
STRIGI_LOG_ERROR ("strigi.FsListener.watch",
446
"m_pEventQueue == NULL!");
450
vector <Event*> events;
452
while (pendingEvent()) {
453
FsEvent* fsevent = retrieveEvent();
455
if (!isEventValid(fsevent)) {
456
STRIGI_LOG_WARNING ("strigi.FsListener.watch",
457
"discarded invalid event");
461
if (isEventInteresting(fsevent)) {
462
STRIGI_LOG_DEBUG ("strigi.FsListener.watch",
463
fsevent->description());
465
switch (fsevent->type()) {
466
case FsEvent::UPDATE:
468
Event* event = new Event (Event::UPDATED, fsevent->file());
469
events.push_back (event);
472
case FsEvent::DELETE:
474
if (fsevent->regardsDir())
475
dirRemoved (fsevent->file(), events);
477
Event* event = new Event (Event::DELETED,
479
events.push_back (event);
483
case FsEvent::CREATE:
485
if (fsevent->regardsDir()) {
487
recursivelyMonitor (fsevent->file(), toWatch, events);
488
addWatches( toWatch);
491
Event* event = new Event (Event::CREATED,
493
events.push_back (event);
498
STRIGI_LOG_DEBUG ("strigi.FsListener.watch",
499
"unknown event type")
507
if (events.size() > 0) {
508
STRIGI_LOG_DEBUG ("strigi.FsListener.watch",
509
"adding events to processing queue")
510
m_pEventQueue->addEvents (events);
514
void FsListener::recursivelyMonitor (string dir, set<string>& toWatch,
515
vector<Event*>& events)
517
STRIGI_LOG_DEBUG ("FsListener.recursivelyMonitor","going to monitor " + dir)
518
DirLister lister(m_pAnalyzerConfiguration);
520
vector<pair<string, struct stat> > fsitems;
522
toWatch.insert (dir);
524
lister.startListing (dir);
525
int ret = lister.nextDir(path, fsitems);
528
for (vector<pair<string,struct stat> >::iterator iter = fsitems.begin();
529
iter != fsitems.end(); ++iter)
531
struct stat stats = iter->second;
533
if (S_ISDIR(stats.st_mode)) //dir
534
recursivelyMonitor(iter->first, toWatch, events);
535
else if (S_ISREG(stats.st_mode)) {
537
Event* event = new Event (Event::CREATED, iter->first);
538
events.push_back (event);
541
ret = lister.nextDir(path, fsitems);
545
void FsListener::addWatches(const set<string> &watches)
547
set<string>::iterator iter;
551
if (!m_bInitialized) {
552
STRIGI_LOG_ERROR ("strigi.FsListener.addWatches",
553
"Listener has not been initialized, unable"
558
for (iter = watches.begin(); iter != watches.end(); ++iter) {
559
if (!addWatch (*iter)) {
560
// adding watch failed, we've to use polling in order to watch it
561
toPool.insert(*iter);
564
watched.insert (*iter);
570
if (m_pollingListener == NULL)
572
m_pollingListener = new PollingListener();
573
m_pollingListener->setEventListenerQueue( m_pEventQueue);
574
m_pollingListener->setCombinedIndexManager( m_pManager);
575
m_pollingListener->setIndexerConfiguration(m_pAnalyzerConfiguration);
576
//TODO: start with a low priority?
577
m_pollingListener->start( );
580
m_pollingListener->addWatches( toPool);
581
m_pollingDirs.insert (toPool.begin(), toPool.end());
585
void FsListener::dirsRemoved (set<string> dirs, vector<Event*>& events)
587
for (set<string>::iterator iter = dirs.begin();
588
iter != dirs.end(); ++iter)
590
string path = fixPath(*iter);
591
set<string>::iterator match = m_pollingDirs.find (path);
592
if (match == m_pollingDirs.end())
593
dirRemoved (path, events);
595
m_pollingListener->rmWatch (*match);
599
void FsListener::setIndexedDirectories (const set<string> &dirs)
602
set<string> fixedDirs;
604
// fix path, all dir must end with a '/'
605
for (set<string>::iterator iter = dirs.begin(); iter != dirs.end(); ++iter)
606
fixedDirs.insert (fixPath (*iter));
608
STRIGI_MUTEX_LOCK (&m_reindexLock);
609
m_reindexDirs = fixedDirs;
610
m_bReindexReq = true;
611
STRIGI_MUTEX_UNLOCK (&m_reindexLock);
614
msg << fixedDirs.size() << " dirs specified";
615
STRIGI_LOG_DEBUG ("FsListener.setIndexedDirectories", msg.str())
618
void FsListener::dumpEvents()
620
unsigned int counter = 1;
621
for (vector<FsEvent*>::iterator iter = m_events.begin();
622
iter != m_events.end(); ++iter)
624
FsEvent* event = *iter;
627
msg << "Event " << counter << "/" << m_events.size() << ": ";
628
msg << event->description();
629
STRIGI_LOG_DEBUG("strigi.FsListener.dumpEvents", msg.str())