« back to all changes in this revision
Viewing changes to Main.cs
- browse files at revision 30
- compare with another revision
- download diff
- download tarball
- view history from revision 30
- Committer: thindil
- Date: 2010-04-04 08:42:41 UTC
- Revision ID: thindil2@gmail.com-20100404084241-p0z5fqho81xwtjes
added ToDo list
- files added:
- TODO
- files removed:
- 7Zip
- misc
- files modified:
- AssemblyInfo.cs
added
removed
Main.cs
1
1
//
2
// Copyright (C) 2009,2010,2011 Bartek thindil Jasicki
2
// Copyright (C) 2009,2010 Bartek thindil Jasicki
3
3
//
4
4
// This file is part of Grubng
5
5
//
22
22
using System.Text.RegularExpressions;
23
23
using System.Net;
24
24
using System.Runtime.InteropServices;
25
using Grubng;
25
26
26
27
27
namespace urlsdb
28
28
{
41
41
public static void Main(string[] args)
42
42
{
43
43
SetProcessName("grub-urlsdb");
44
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(MainUnhandledException);
45
44
//If no arguments for program, show help
46
45
if (args.Length == 0)
47
46
{
48
Console.WriteLine("Syntax: mono urlsdb.exe [OPTION] [COMMAND]");
49
Console.WriteLine("Options:");
50
Console.WriteLine("--quiet - no console output (quiet mode, only error messages)");
51
Console.WriteLine("Commands:");
52
47
Console.WriteLine("createconf - create configuration file for program");
53
48
Console.WriteLine("updateconf - update configuration file for program");
54
49
Console.WriteLine("createdb - create database");
55
50
Console.WriteLine("updatedb - update database");
56
51
Console.WriteLine("filldb [FILENAME] - put URL's to database from FILENAME file");
57
Console.WriteLine("fastfilldb [FILENAME] - put URL's to database from FILENAME file. For difference with filldb look at README.txt");
58
52
Console.WriteLine("backup [FILENAME] - create database backup to FILENAME file (only URL's)");
59
53
Console.WriteLine("addurl [URL] - add URL to database");
60
54
Console.WriteLine("delurl [URL] - delete URL from database and index");
61
55
Console.WriteLine("delurls [FILENAME] - delete URL's (list from FILENAME) from database and index");
62
56
Console.WriteLine("createwu - create workunits from URL's from database.");
63
Console.WriteLine("maintain - delete URL's from database (with HTTP status codes: 204, 301, 400-408, 500, 503), optimize database and Solr, create compressed backup of URL's");
57
Console.WriteLine("maintain - delete URL's from database (with HTTP status codes: 301 and 400-406), optimize tables and optimize Solr");
64
58
Console.WriteLine("optimizedb - optimize and flush database");
65
59
Console.WriteLine("optimizesolr - optimize Solr index (if Solr is enabled)");
66
60
return;
67
61
}
68
string command;
69
Utils util;
70
int pos = 1;
71
bool quiet = false;
72
if (args[0] != "--quiet")
73
{
74
command = args[0];
75
}
76
else
77
{
78
command = args[1];
79
pos = 2;
80
quiet = true;
81
}
82
util = new Utils(quiet);
83
62
//Create configuration file
84
if (command == "createconf")
63
if (args[0] == "createconf")
85
64
{
86
util.Message("Creating configuration file ... ", false);
87
Config.CreateConfig();
88
util.Message("done.", true);
89
util.Message("Please now edit it for set correct settings", true);
65
Console.Write("Creating configuration file ... ");
66
Config.CreateConfig();
67
Console.WriteLine("done.");
68
Console.WriteLine("Please now edit it for set correct settings");
90
69
return;
91
70
}
92
71
//Update configuration file
93
if (command == "updateconf")
72
if (args[0] == "updateconf")
94
73
{
95
util.Message("Updating configuration file ... ", false);
74
Console.Write("Updating configuration file ... ");
96
75
Config.UpdateConfig();
97
util.Message("done.", true);
98
util.Message("Please now edit it for set correct settings", true);
76
Console.WriteLine("done.");
77
Console.WriteLine("Please now edit it for set correct settings");
99
78
return;
100
79
}
101
80
//Create database
102
if (command == "createdb")
81
if (args[0] == "createdb")
103
82
{
104
util.Message("Creating database ... ", false);
83
Console.Write("Creating database ... ");
105
84
Database.CreateDB();
106
util.Message("done.", true);
85
Console.WriteLine("done.");
107
86
return;
108
87
}
109
88
Database db = new Database();
89
db.Open();
110
90
//Update database
111
if (command == "updatedb")
91
if (args[0] == "updatedb")
112
92
{
113
util.Message("Updating database ... ", false);
93
Console.Write("Updating database ... ");
114
94
db.UpdateDB();
115
util.Message("done.", true);
95
Console.WriteLine("done.");
96
db.Close();
116
97
return;
117
98
}
118
99
//Backup URL's to plain text file
119
Files file = new Files(quiet);
120
if (command == "backup")
100
if (args[0] == "backup")
121
101
{
122
if (args.Length == pos)
102
if (args.Length == 1)
123
103
{
124
104
Console.WriteLine("Please enter backup file name.");
125
105
return;
126
106
}
127
file.Backup(args[pos]);
107
Console.Write("Creating database backup ... ");
108
int offset = 0;
109
//string urls;
110
StreamWriter writer = new StreamWriter(args[1]);
111
StringBuilder urls2 = new StringBuilder();
112
while (true)
113
{
114
urls2.Append(db.SelectURLs(offset));
115
if (urls2.Length == 0)
116
{
117
break;
118
}
119
urls2.Append(Environment.NewLine);
120
writer.Write(urls2);
121
writer.Flush();
122
urls2.Remove(0, urls2.Length);
123
offset += 250000;
124
}
125
writer.Close();
126
writer.Dispose();
127
Console.WriteLine("done.");
128
128
}
129
129
ParseURLs parseurl = new ParseURLs();
130
130
int amount = 0;
131
131
string record, hash1, hash2 = String.Empty;
132
//Check if selected file exists
133
if ((command == "filldb") || (command == "fastfilldb") || (command == "delurls"))
134
{
135
if (args.Length == 1)
136
{
137
Console.WriteLine("Please enter file name with urls.");
138
return;
139
}
140
if (!File.Exists(args[pos]))
141
{
142
Console.WriteLine("File {0} not exist.", args[pos]);
143
return;
144
}
145
}
146
132
//Insert URL's from file to database
147
if ((command == "filldb") || (command == "fastfilldb"))
133
if (args[0] == "filldb")
148
134
{
149
if (command == "filldb")
150
{
151
file.AddURLs(args[pos], false);
152
}
153
else
154
{
155
file.AddURLs(args[pos], true);
156
}
135
if (args.Length == 1)
136
{
137
Console.WriteLine("Please enter file name with urls.");
138
return;
139
}
140
if (!File.Exists(args[1]))
141
{
142
Console.WriteLine("File {0} not exist.", args[1]);
143
return;
144
}
145
StreamReader reader = new StreamReader(args[1]);
146
int i = 0;
147
while (!reader.EndOfStream)
148
{
149
record = ParseURLs.ParseURL(reader.ReadLine());
150
if (record.Length > 0)
151
{
152
hash1 = parseurl.GetHash(record);
153
if (!record.StartsWith("www."))
154
{
155
hash2 = parseurl.GetHash("www." + record);
156
}
157
else
158
{
159
hash2 = parseurl.GetHash(record.Substring(4));
160
}
161
amount += db.InstertURL(hash1, record, hash2);
162
}
163
i ++;
164
}
165
reader.Close();
166
reader.Dispose();
167
Console.WriteLine("{0} URL's was inserted from {1} URL's.", amount.ToString(), i.ToString());
157
168
}
158
169
//Add single URL to database
159
if (command == "addurl")
170
if (args[0] == "addurl")
160
171
{
161
172
if (args.Length == 1)
162
173
{
163
174
Console.WriteLine("Please enter URL to add.");
164
175
return;
165
176
}
166
record = ParseURLs.ParseURL(args[pos]);
177
record = ParseURLs.ParseURL(args[1]);
167
178
if (record.Length > 0)
168
179
{
169
180
hash1 = parseurl.GetHash(record);
179
190
}
180
191
if (amount == 0)
181
192
{
182
util.Message("URL already exist in database.", true);
193
Console.WriteLine("URL already exist in database.");
183
194
}
184
195
else
185
196
{
186
util.Message("URL added to database.", true);
197
Console.WriteLine("URL added to database.");
187
198
}
188
199
}
189
200
bool solrenabled = false;
192
203
solrenabled = true;
193
204
}
194
205
//Delete single URL from database
195
if (command == "delurl")
206
if (args[0] == "delurl")
196
207
{
197
208
if (args.Length == 1)
198
209
{
199
210
Console.WriteLine("Please enter URL to delete.");
200
211
return;
201
212
}
202
amount = db.DeleteURL(parseurl.GetHash(args[pos]));
213
amount = db.DeleteURL(parseurl.GetHash(args[1]));
203
214
if (solrenabled)
204
215
{
205
string solrcommand = "<delete><id>http://" +
206
System.Security.SecurityElement.Escape(ParseURLs.ParseURL(args[pos])) +
207
"</id></delete>";
216
string solrcommand = "<delete><id>http://" + ParseURLs.ParseURL(args[1]) + "</id></delete>";
208
217
solrcommand = Regex.Replace(solrcommand, @"[\p{IsC}]", String.Empty);
209
MainClass.SendCommand(solrcommand, "Solr");
218
MainClass.SendToSolr(solrcommand);
210
219
solrcommand = "<commit/>";
211
MainClass.SendCommand(solrcommand, "Solr");
220
MainClass.SendToSolr(solrcommand);
212
221
}
213
222
if (amount == 0)
214
223
{
215
util.Message("URL not exist in database.", true);
224
Console.WriteLine("URL not exist in database.");
216
225
}
217
226
else
218
227
{
219
util.Message("URL deleted from database.", true);
228
Console.WriteLine("URL deleted from database.");
220
229
}
221
230
}
222
parseurl.Dispose();
223
231
//Delete URL's from file from database
224
if (command == "delurls")
232
if (args[0] == "delurls")
225
233
{
226
util.Message("Deleting URL's from database ... ", false);
227
FileInfo finfo = new FileInfo(args[pos]);
228
int length = (int)finfo.Length;
229
finfo = null;
230
StreamReader reader = new StreamReader(args[pos]);
231
StringBuilder solrcommand = new StringBuilder();
232
solrcommand.Append("<delete>");
233
int i = 0, amount2 = 0;
234
string scommand = String.Empty;
235
System.Collections.Generic.List<string> records = new System.Collections.Generic.List<string>();
236
util.ProgressStart(length, String.Empty);
234
if (args.Length == 1)
235
{
236
Console.WriteLine("Please enter file name with urls.");
237
return;
238
}
239
if (!File.Exists(args[1]))
240
{
241
Console.WriteLine("File {0} not exist.", args[1]);
242
return;
243
}
244
StreamReader reader = new StreamReader(args[1]);
245
string solrcommand = "<delete>";
246
int i = 0;
237
247
while (!reader.EndOfStream)
238
248
{
239
for (int j = 0; j < 25000; j++)
240
{
241
record = reader.ReadLine();
242
if (record == null)
243
{
244
break;
245
}
246
records.Add(record);
247
}
248
amount += db.DeleteURLs(records);
249
foreach (string record3 in records)
250
{
251
if (solrenabled)
252
{
253
solrcommand.Append("<id>http://");
254
solrcommand.Append(System.Security.SecurityElement.Escape(ParseURLs.ParseURL(record3)));
255
solrcommand.Append("</id>");
256
amount2 ++;
257
if (amount2 == 2000)
258
{
259
solrcommand.Append("</delete>");
260
scommand = Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
261
MainClass.SendCommand(scommand, "Solr");
262
MainClass.SendCommand("<commit/>", "Solr");
263
solrcommand.Remove(0, solrcommand.Length);
264
solrcommand.Append("<delete>");
265
amount2 = 0;
266
}
267
}
268
util.Curamount += record3.Length;
269
i ++;
270
}
271
records.Clear();
272
records.TrimExcess();
249
record = reader.ReadLine();
250
amount += db.DeleteURL(parseurl.GetHash(record));
251
solrcommand += "<id>http://" + ParseURLs.ParseURL(record) + "</id>";
252
i ++;
273
253
}
274
records = null;
275
254
reader.Close();
276
255
reader.Dispose();
277
256
if (solrenabled)
278
257
{
279
solrcommand.Append("</delete>");
280
scommand = Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
281
MainClass.SendCommand(scommand, "Solr");
282
MainClass.SendCommand("<commit/>", "Solr");
258
solrcommand += "</delete>";
259
solrcommand = Regex.Replace(solrcommand, @"[\p{IsC}]", String.Empty);
260
MainClass.SendToSolr(solrcommand);
261
solrcommand = "<commit/>";
262
MainClass.SendToSolr(solrcommand);
283
263
}
284
util.ProgressStop();
285
util.Message(String.Empty, true);
286
util.Message(amount.ToString() + " URL's was deleted from " + i.ToString() + " URL's.", true);
264
Console.WriteLine("{0} URL's was deleted from {1} URL's.", amount.ToString(), i.ToString());
287
265
}
288
266
//Create workunits from URL's from database
289
if (command == "createwu")
267
if (args[0] == "createwu")
290
268
{
291
util.Message("Creating workunits ... ", false);
292
269
string urls, path, host;
293
270
string wupass = Config.ReadConfig("/configuration/workunitspassword");
294
271
string useragent = Config.ReadConfig("/configuration/useragent");
376
353
writer.Close();
377
354
writer.Dispose();
378
355
File.Delete(tempwu);
379
util.Message("done.", true);
380
356
}
357
db.Close();
381
358
//Maintenance work on URL's database and Solr
382
if ((command == "maintain") || (command == "optimizedb") || (command == "optimizesolr"))
383
{
384
MainClass.SendCommand("serverenabled=N,enableupload=N", "Upload server");
385
}
386
if (command == "maintain")
387
{
388
string[] codes = new string[] {"204", "301", "400", "401", "402", "403", "404", "405", "406", "408", "500", "503"};
359
if (args[0] == "maintain")
360
{
361
string[] codes = new string[] {"204", "301", "400", "401", "402", "403", "404", "405", "406", "408", "503"};
389
362
foreach (string code in codes)
390
363
{
391
util.Message("Deleting URL's with HTTP status code " + code, true);
364
Console.WriteLine("Deleting URL's with HTTP status code {0}", code);
365
db.Open();
392
366
amount = db.CleanURLs(code, solrenabled);
393
util.Message(amount.ToString() + " URL's was deleted.", true);
367
db.Close();
368
Console.WriteLine("{0} URL's was deleted.", amount);
394
369
}
395
370
}
396
//Optimize database
397
if ((command == "optimizedb") || (command == "maintain"))
371
//Optimize and flush database
372
if ((args[0] == "optimizedb") || (args[0] == "maintain"))
398
373
{
399
util.Message("Optimizing database ... ", false);
374
Console.Write("Optimizing database ... ");
375
db.Open();
400
376
db.OptimizeDB();
401
util.Message("done.", true);
377
db.Close();
378
Console.WriteLine("done.");
402
379
}
403
380
//Optimize Solr index
404
if ((command == "optimizesolr") || (command == "maintain"))
381
if ((args[0] == "optimizesolr") || (args[0] == "maintain"))
405
382
{
406
383
if (!solrenabled)
407
384
{
408
385
return;
409
386
}
410
util.Message("Optimizing Solr ... ", false);
411
MainClass.SendCommand("<optimize/>", "Solr");
412
util.Message("done.", true);
413
}
414
//Create database backup during maintenance work
415
if (command == "maintain")
416
{
417
file.Backup(DateTime.UtcNow.ToString("yyyy" + "MM" + "dd") + ".txt");
418
}
419
if ((command == "maintain") || (command == "optimizedb") || (command == "optimizesolr"))
420
{
421
MainClass.SendCommand("serverenabled=Y,enableupload=Y", "Upload server");
422
}
423
util.Dispose();
387
Console.Write("Optimizing Solr ... ");
388
MainClass.SendToSolr("<optimize/>");
389
Console.WriteLine("done.");
390
}
424
391
}
425
392
426
393
/// <summary>
437
404
if (NativeMethods.prctl (15 /* PR_SET_NAME */, Encoding.ASCII.GetBytes (name + "\0"),
438
405
IntPtr.Zero, IntPtr.Zero, IntPtr.Zero) != 0) {
439
406
throw new ApplicationException ("Error setting process name: " +
440
Mono.Unix.Native.Stdlib.GetLastError().ToString());
407
Mono.Unix.Native.Stdlib.GetLastError ());
441
408
}
442
409
}
443
410
catch (EntryPointNotFoundException)
448
415
}
449
416
450
417
/// <summary>
451
/// Function send XML Update command to remote server
418
/// Function send XML Update command to Solr
452
419
/// </summary>
453
420
/// <param name="command">
454
/// A <see cref="System.String"/> command to send.
421
/// A <see cref="System.String"/> XML Update command to send
455
422
/// </param>
456
/// <param name="destination">
457
/// A <see cref="System.String"/> destination for command (Solr, upload server).
458
public static void SendCommand(string command, string destination)
423
public static void SendToSolr(string command)
459
424
{
460
HttpWebRequest request;
461
if (destination == "Solr")
462
{
463
request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/solraddress"));
464
request.Credentials = new NetworkCredential(Config.ReadConfig("/configuration/solrusername"),
465
Config.ReadConfig("configuration/solrpassword"));
466
request.PreAuthenticate = true;
467
request.ContentType = "text/xml";
468
}
469
else
470
{
471
request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/uploadaddress"));
472
command = "config," + Config.ReadConfig("/configuration/uploadusername") + "," +
473
Config.ReadConfig("/configuration/uploadpassword") + ",set," + command;
474
}
425
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/solraddress"));
475
426
request.Proxy = null;
427
request.Credentials = new NetworkCredential(Config.ReadConfig("/configuration/solrusername"),
428
Config.ReadConfig("configuration/solrpassword"));
429
request.PreAuthenticate = true;
430
request.ContentType = "text/xml";
476
431
request.Method = "POST";
477
432
request.Timeout = 1000000000;
433
Stream streamw1 = request.GetRequestStream();
478
434
byte[] buffer = Encoding.UTF8.GetBytes(command);
479
using (Stream streamw1 = request.GetRequestStream())
480
{
481
streamw1.Write(buffer, 0, buffer.Length);
482
streamw1.Close();
483
}
435
streamw1.Write(buffer, 0, buffer.Length);
436
streamw1.Close();
437
streamw1.Dispose();
484
438
try
485
439
{
486
440
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
493
447
{
494
448
if (e.Response != null)
495
449
{
496
using (Stream stream = e.Response.GetResponseStream())
450
Stream stream = e.Response.GetResponseStream();
451
FileStream errorfile = new FileStream("error.log", FileMode.Append);
452
string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
453
buffer = Encoding.UTF8.GetBytes(tdate + " Solr error:" + Environment.NewLine);
454
errorfile.Write(buffer, 0, buffer.Length);
455
errorfile.Flush();
456
buffer = new byte[1024];
457
int bytesRead = 0;
458
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != 0)
497
459
{
498
using (FileStream errorfile = new FileStream("error.log", FileMode.Append))
499
{
500
string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
501
buffer = Encoding.UTF8.GetBytes(tdate + " " + destination + " error:" + Environment.NewLine);
502
errorfile.Write(buffer, 0, buffer.Length);
503
errorfile.Flush();
504
buffer = new byte[1024];
505
int bytesRead = 0;
506
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != 0)
507
{
508
errorfile.Write(buffer, 0, bytesRead);
509
errorfile.Flush();
510
}
511
errorfile.Close();
512
}
513
stream.Close();
460
errorfile.Write(buffer, 0, bytesRead);
461
errorfile.Flush();
514
462
}
463
stream.Close();
464
stream.Dispose();
465
errorfile.Close();
466
errorfile.Dispose();
515
467
e.Response.Close();
516
468
}
517
469
}
518
470
}
519
520
/// <summary>
521
/// Main function for catch unhandled exceptions. Write informations about exception to error.log.
522
/// </summary>
523
/// <param name="sender">
524
/// A <see cref="System.Object"/> unused.
525
/// </param>
526
/// <param name="args">
527
/// A <see cref="UnhandledExceptionEventArgs"/> provide informations about error (source, stacktrace,
528
/// general info about error)
529
/// </param>
530
static void MainUnhandledException(object sender, UnhandledExceptionEventArgs args)
531
{
532
if (sender != null)
533
{
534
sender = null;
535
}
536
Exception e = (Exception)args.ExceptionObject;
537
string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
538
using (FileStream errorlog = File.Open("error.log", FileMode.Append))
539
{
540
using (StreamWriter logstream = new StreamWriter(errorlog))
541
{
542
logstream.WriteLine(tdate + " Caught: " + e.GetType().ToString() + " " + e.Message);
543
logstream.WriteLine(" Source: " + e.Source);
544
logstream.WriteLine(" StackTrace: " + e.StackTrace);
545
logstream.WriteLine(" TargetSite: " + e.TargetSite);
546
logstream.Close();
547
}
548
}
549
}
550
471
}
551
472
552
473
/// <summary>
Loggerhead is a web-based interface for Breezy
Version: 2.0.1