~grubng-dev/grubng/tools-urlsdb

//  
//  Copyright (C) 2009,2010,2011 Bartek thindil Jasicki
// 
//  This file is part of Grubng
// 
//  Grubng is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
// 
//  You should have received a copy of the GNU General Public License
//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
// 
using System;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Net;
using System.Runtime.InteropServices;
using Grubng;

namespace urlsdb
{
	/// <summary>
	/// Main program class
	/// </summary>
	static class MainClass
	{
		/// <summary>
		/// Main program function. Add, delete URL's from database and Solr, create workunits, config file
		/// and URL's database.
		/// </summary>
		/// <param name="args">
		/// A <see cref="System.String"/> option to program. Run program without option for get help.
		/// </param>
		public static void Main(string[] args)
		{
			SetProcessName("grub-urlsdb");
			AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(MainUnhandledException);
			//If no arguments for program, show help
			if (args.Length == 0)
			{
				Console.WriteLine("Syntax: mono urlsdb.exe [OPTION] [COMMAND]");
				Console.WriteLine("Options:");
				Console.WriteLine("--quiet - no console output (quiet mode, only error messages)");
				Console.WriteLine("Commands:");
				Console.WriteLine("createconf - create configuration file for program");
				Console.WriteLine("updateconf - update configuration file for program");
				Console.WriteLine("createdb - create database");
				Console.WriteLine("updatedb - update database");
				Console.WriteLine("filldb [FILENAME] - put URL's to database from FILENAME file");
				Console.WriteLine("fastfilldb [FILENAME] - put URL's to database from FILENAME file. For difference with filldb look at README.txt");
				Console.WriteLine("backup [FILENAME] - create database backup to FILENAME file (only URL's)");
				Console.WriteLine("addurl [URL] - add URL to database");
				Console.WriteLine("delurl [URL] - delete URL from database and index");
				Console.WriteLine("delurls [FILENAME] - delete URL's (list from FILENAME) from database and index");
				Console.WriteLine("createwu - create workunits from URL's from database.");
				Console.WriteLine("maintain - delete URL's from database (with HTTP status codes: 204, 301, 400-408, 500, 503), optimize database and Solr, create compressed backup of URL's");
				Console.WriteLine("optimizedb - optimize and flush database");
				Console.WriteLine("optimizesolr - optimize Solr index (if Solr is enabled)");
				return;
			}
			string command;
			Utils util;
			int pos = 1;
			bool quiet = false;
			if (args[0] != "--quiet")
			{
				command = args[0];
			}
			else
			{
				command = args[1];
				pos = 2;
				quiet = true;
			}
			util = new Utils(quiet);
			//Create configuration file
			if (command == "createconf")
			{
				util.Message("Creating configuration file ... ", false);
				Config.CreateConfig();
				util.Message("done.", true);
				util.Message("Please now edit it for set correct settings", true);
				return;
			}
			//Update configuration file
			if (command == "updateconf")
			{
				util.Message("Updating configuration file ... ", false);
				Config.UpdateConfig();
				util.Message("done.", true);
				util.Message("Please now edit it for set correct settings", true);
				return;
			}
			//Create database
			if (command == "createdb")
			{
				util.Message("Creating database ... ", false);
				Database.CreateDB();
				util.Message("done.", true);
				return;
			}
			Database db = new Database();
			//Update database
			if (command == "updatedb")
			{
				util.Message("Updating database ... ", false);
				db.UpdateDB();
				util.Message("done.", true);
				return;
			}
			//Backup URL's to plain text file
			Files file = new Files(quiet);
			if (command == "backup")
			{
				if (args.Length == pos)
				{
					Console.WriteLine("Please enter backup file name.");
					return;
				}
				file.Backup(args[pos]);
			}
			ParseURLs parseurl = new ParseURLs();
			int amount = 0;
			string record, hash1, hash2 = String.Empty;
			//Check if selected file exists
			if ((command == "filldb") || (command == "fastfilldb") || (command == "delurls"))
			{
				if (args.Length == 1)
				{
					Console.WriteLine("Please enter file name with urls.");
					return;
				}
				if (!File.Exists(args[pos]))
				{
					Console.WriteLine("File {0} not exist.", args[pos]);
					return;
				}
			}
			//Insert URL's from file to database
			if ((command == "filldb") || (command == "fastfilldb"))
			{
				if (command == "filldb")
				{
					file.AddURLs(args[pos], false);
				}
				else
				{
					file.AddURLs(args[pos], true);
				}
			}
			//Add single URL to database
			if (command == "addurl")
			{
				if (args.Length == 1)
				{
					Console.WriteLine("Please enter URL to add.");
					return;
				}
				record = ParseURLs.ParseURL(args[pos]);
				if (record.Length > 0)
				{
					hash1 = parseurl.GetHash(record);
					if (!record.StartsWith("www."))
					{
						hash2 = parseurl.GetHash("www." + record);
					}
					else
					{
						hash2 = parseurl.GetHash(record.Substring(4));
					}
					amount = db.InstertURL(hash1, record, hash2);
				}
				if (amount == 0)
				{
					util.Message("URL already exist in database.", true);
				}
				else
				{
					util.Message("URL added to database.", true);
				}
			}
			bool solrenabled = false;
			if (Config.ReadConfig("/configuration/enablesolr") == "Y")
			{
				solrenabled = true;
			}
			//Delete single URL from database
			if (command == "delurl")
			{
				if (args.Length == 1)
				{
					Console.WriteLine("Please enter URL to delete.");
					return;
				}
				amount = db.DeleteURL(parseurl.GetHash(args[pos]));
				if (solrenabled)
				{
					string solrcommand = "<delete><id>http://" + 
						System.Security.SecurityElement.Escape(ParseURLs.ParseURL(args[pos])) + 
						                                       "</id></delete>";
					solrcommand = Regex.Replace(solrcommand, @"[\p{IsC}]", String.Empty);
					MainClass.SendCommand(solrcommand, "Solr");
					solrcommand = "<commit/>";
					MainClass.SendCommand(solrcommand, "Solr");
				}
				if (amount == 0)
				{
					util.Message("URL not exist in database.", true);
				}
				else
				{
					util.Message("URL deleted from database.", true);
				}
			}
			parseurl.Dispose();
			//Delete URL's from file from database
			if (command == "delurls")
			{
				util.Message("Deleting URL's from database ... ", false);
				FileInfo finfo = new FileInfo(args[pos]);
				int length = (int)finfo.Length;
				finfo = null;
				StreamReader reader = new StreamReader(args[pos]);
				StringBuilder solrcommand = new StringBuilder();
				solrcommand.Append("<delete>");
				int i = 0, amount2 = 0;
				string scommand = String.Empty;
				System.Collections.Generic.List<string> records = new System.Collections.Generic.List<string>();
				util.ProgressStart(length, String.Empty);
				while (!reader.EndOfStream)
				{
					for (int j = 0; j < 25000; j++)
					{
						record = reader.ReadLine();
						if (record == null)
						{
							break;
						}
						records.Add(record);
					}
					amount += db.DeleteURLs(records);
					foreach (string record3 in records)
					{
						if (solrenabled)
						{
							solrcommand.Append("<id>http://");
							solrcommand.Append(System.Security.SecurityElement.Escape(ParseURLs.ParseURL(record3)));
							solrcommand.Append("</id>");
							amount2 ++;
							if (amount2 == 2000)
							{
								solrcommand.Append("</delete>");
								scommand = Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
								MainClass.SendCommand(scommand, "Solr");
								MainClass.SendCommand("<commit/>", "Solr");
								solrcommand.Remove(0, solrcommand.Length);
								solrcommand.Append("<delete>");
								amount2 = 0;
							}
						}
						util.Curamount += record3.Length;
						i ++;
					}
					records.Clear();
					records.TrimExcess();
				}
				records = null;
				reader.Close();
				reader.Dispose();
				if (solrenabled)
				{
					solrcommand.Append("</delete>");
					scommand = Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
					MainClass.SendCommand(scommand, "Solr");
					MainClass.SendCommand("<commit/>", "Solr");
				}
				util.ProgressStop();
				util.Message(String.Empty, true);
				util.Message(amount.ToString() + " URL's was deleted from " + i.ToString() + " URL's.", true);
			}
			//Create workunits from URL's from database
			if (command == "createwu")
			{
				util.Message("Creating workunits ... ", false);
				string urls, path, host;
				string wupass = Config.ReadConfig("/configuration/workunitspassword");
				string useragent = Config.ReadConfig("/configuration/useragent");
				int urlsamount = Convert.ToInt32(Config.ReadConfig("/configuration/urlsamount"));
				string httpversion = Config.ReadConfig("/configuration/httpversion");
				string accept = Config.ReadConfig("/configuration/accept");
				string[] urlsArray, urlparts;
				int offset = 0;
				urls = db.SelectURLs(-1);
				if (urls.Length == 0)
				{
					return;
				}
				string workunitsdirectory = Config.ReadConfig("/configuration/workunitsdirectory");
				string tempwu = workunitsdirectory + "wu.temp";
				StreamWriter writer = new StreamWriter(tempwu);
				DateTime modtime;
				urlsArray = urls.Split(new char[] {'\n'});
				StringBuilder key = new StringBuilder();
				System.Security.Cryptography.SHA1Managed SHhash = new System.Security.Cryptography.SHA1Managed();
				byte[] PureHash = Encoding.UTF8.GetBytes(wupass);
				byte[] HashValue = SHhash.ComputeHash(PureHash);
				foreach(byte b in HashValue) 
				{
					key.Append(String.Format("{0:x2}", b));
				}
				foreach (string url in urlsArray)
				{
					urlparts = url.Split(new char[] {' '});
					if (urlparts.Length != 2)
					{
						continue;
					}
					if (urlparts[0].IndexOf('/') == -1)
					{
						path = "/";
						host = urlparts[0];
					}
					else
					{
						path = urlparts[0].Substring(url.IndexOf('/'));
						host = urlparts[0].Remove(url.IndexOf('/'));
					}
					writer.Write("GET " + path + " HTTP/" + httpversion +"\r\n");
					writer.Write("Host: " + host + "\r\n");
					writer.Write("User-Agent: " + useragent + "\r\n");
					writer.Write("Accept: " + accept +"\r\n");
					if (urlparts[1] != "0")
					{
						modtime = new DateTime(Convert.ToInt32(urlparts[1].Substring(0, 4)), 
						                       Convert.ToInt32(urlparts[1].Substring(4, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(6, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(8, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(10, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(12)));
						writer.Write("If-Modified-Since: " + modtime.ToString("r") + "\r\n");
					}
					writer.Write("\r\n");
					writer.Flush();
					PureHash = Encoding.UTF8.GetBytes(key.ToString() + " " + host + " " + path);
					key.Remove(0, key.Length);
					HashValue = SHhash.ComputeHash(PureHash);
					foreach(byte b in HashValue) 
					{
						key.Append(String.Format("{0:x2}", b));
					}
					offset ++;
					if (offset == urlsamount)
					{
						writer.Close();
						writer.Dispose();
						File.Copy(tempwu, workunitsdirectory + key.ToString() + ".wu1", true);
						File.Delete(tempwu);
						key.Remove(0, key.Length);
						PureHash = Encoding.UTF8.GetBytes(wupass);
						HashValue = SHhash.ComputeHash(PureHash);
						foreach(byte b in HashValue) 
						{
							key.Append(String.Format("{0:x2}", b));
						}
						writer = new StreamWriter(tempwu);
						offset = 0;
					}
				}
				writer.Close();
				writer.Dispose();
				File.Delete(tempwu);
				util.Message("done.", true);
			}
			//Maintenance work on URL's database and Solr
			if ((command == "maintain") || (command == "optimizedb") || (command == "optimizesolr"))
			{
				MainClass.SendCommand("serverenabled=N,enableupload=N", "Upload server");
			}
			if (command == "maintain")
			{
				string[] codes = new string[] {"204", "301", "400", "401", "402", "403", "404", "405", "406", "408", "500", "503"};
				foreach (string code in codes)
				{
					util.Message("Deleting URL's with HTTP status code " + code, true);
					amount = db.CleanURLs(code, solrenabled);
					util.Message(amount.ToString() + " URL's was deleted.", true);
				}
			}
			//Optimize database
			if ((command == "optimizedb") || (command == "maintain"))
			{
				util.Message("Optimizing database ... ", false);
				db.OptimizeDB();
				util.Message("done.", true);
			}
			//Optimize Solr index
			if ((command == "optimizesolr") || (command == "maintain"))
			{
				if (!solrenabled)
				{
					return;
				}
				util.Message("Optimizing Solr ... ", false);
				MainClass.SendCommand("<optimize/>", "Solr");
				util.Message("done.", true);
			}
			//Create database backup during maintenance work
			if (command == "maintain")
			{
				file.Backup(DateTime.UtcNow.ToString("yyyy" + "MM" + "dd") + ".txt");
			}
			if ((command == "maintain") || (command == "optimizedb") || (command == "optimizesolr"))
			{
				MainClass.SendCommand("serverenabled=Y,enableupload=Y", "Upload server");
			}
			util.Dispose();
		}
		
		/// <summary>
		/// Function set process name on Unix systems. Code borrowed from 
		/// http://abock.org/2006/02/09/changing-process-name-in-mono/
		/// </summary>
		/// <param name="name">
		/// A <see cref="System.String"/> new name of process
		/// </param>
		public static void SetProcessName (string name)
		{
			try 
			{
				if (NativeMethods.prctl (15 /* PR_SET_NAME */, Encoding.ASCII.GetBytes (name + "\0"),
				           IntPtr.Zero, IntPtr.Zero, IntPtr.Zero) != 0) {
					throw new ApplicationException ("Error setting process name: " +
					                                Mono.Unix.Native.Stdlib.GetLastError().ToString());
				}
			} 
			catch (EntryPointNotFoundException) 
			{
				NativeMethods.setproctitle (Encoding.ASCII.GetBytes ("%s\0"),
				              Encoding.ASCII.GetBytes (name + "\0"));
			}
		}
		
		/// <summary>
		/// Function send XML Update command to remote server
		/// </summary>
		/// <param name="command">
		/// A <see cref="System.String"/> command to send.
		/// </param>
		/// <param name="destination">
		/// A <see cref="System.String"/> destination for command (Solr, upload server).
		public static void SendCommand(string command, string destination)
		{
			HttpWebRequest request;
			if (destination == "Solr")
			{
				request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/solraddress"));
				request.Credentials = new NetworkCredential(Config.ReadConfig("/configuration/solrusername"), 
				                                            Config.ReadConfig("configuration/solrpassword"));
				request.PreAuthenticate = true;
				request.ContentType = "text/xml";
			}
			else
			{
				request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/uploadaddress"));
				command = "config," + Config.ReadConfig("/configuration/uploadusername") + "," + 
				                      Config.ReadConfig("/configuration/uploadpassword") + ",set," + command;
			}
			request.Proxy = null;
			request.Method = "POST";
			request.Timeout = 1000000000;
			byte[] buffer = Encoding.UTF8.GetBytes(command);
			using (Stream streamw1 = request.GetRequestStream())
			{
				streamw1.Write(buffer, 0, buffer.Length);
				streamw1.Close();
			}
			try
			{
				HttpWebResponse response = (HttpWebResponse)request.GetResponse();
				if (response != null)
				{
					response.Close();
				}
			}
			catch (WebException e)
			{
				if (e.Response != null)
				{
					using (Stream stream = e.Response.GetResponseStream())
					{
						using (FileStream errorfile = new FileStream("error.log", FileMode.Append))
						{
							string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
							buffer = Encoding.UTF8.GetBytes(tdate + " " + destination + " error:" + Environment.NewLine);
							errorfile.Write(buffer, 0, buffer.Length);
							errorfile.Flush();
							buffer = new byte[1024];
							int bytesRead = 0;
							while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != 0)
							{
								errorfile.Write(buffer, 0, bytesRead);
								errorfile.Flush();
							}
							errorfile.Close();
						}
						stream.Close();
					}
					e.Response.Close();
				}
			}
		}
		
		/// <summary>
		/// Main function for catch unhandled exceptions. Write informations about exception to error.log.
		/// </summary>
		/// <param name="sender">
		/// A <see cref="System.Object"/> unused.
		/// </param>
		/// <param name="args">
		/// A <see cref="UnhandledExceptionEventArgs"/> provide informations about error (source, stacktrace,
		/// general info about error)
		/// </param>
		static void MainUnhandledException(object sender, UnhandledExceptionEventArgs args)
		{
			if (sender != null)
			{
				sender = null;
			}
			Exception e = (Exception)args.ExceptionObject;
			string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
			using (FileStream errorlog = File.Open("error.log", FileMode.Append))
			{
				using (StreamWriter logstream = new StreamWriter(errorlog))
				{
					logstream.WriteLine(tdate + " Caught: " + e.GetType().ToString() + " " + e.Message);
					logstream.WriteLine("  Source: " + e.Source);
					logstream.WriteLine("  StackTrace: " + e.StackTrace);
					logstream.WriteLine("  TargetSite: " + e.TargetSite);
					logstream.Close();
				}
			}
		}
	}
	
	/// <summary>
	/// Provide p/invoke for native Unix methods
	/// </summary>
	internal static class NativeMethods
	{
		[DllImport ("libc")] // GNU/Linux
		public static extern int prctl (int option, byte [] arg2, IntPtr arg3,
		                                 IntPtr arg4, IntPtr arg5);

		[DllImport ("libc")] // BSD
		public static extern void setproctitle (byte [] fmt, byte [] str_arg);
	}
}