~grubng-dev/grubng/tools-urlsdb

//  
//  Copyright (C) 2009 Bartek thindil Jasicki
// 
//  This file is part of Grubng
// 
//  Grubng is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
// 
//  You should have received a copy of the GNU General Public License
//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
// 
using System;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Net;

namespace urlsdb
{
	class MainClass
	{
		public static void Main(string[] args)
		{
			//If no arguments for program, show help
			if (args.Length == 0)
			{
				Console.WriteLine("createconf - create configuration file for program");
				Console.WriteLine("updateconf - update configuration file for program");
				Console.WriteLine("createdb - create database");
				Console.WriteLine("filldb [FILENAME] - put URL's to database from FILENAME file");
				Console.WriteLine("backup [FILENAME] - create database backup to FILENAME file (only URL's)");
				Console.WriteLine("addurl [URL] - add URL to database");
				Console.WriteLine("delurl [URL] - delete URL from database and index");
				Console.WriteLine("delurls [FILENAME] - delete URL's (list from FILENAME) from database and index");
				Console.WriteLine("createwu [PASSWORD] - create workunits from URL's from database. You must set this same password for upload server.");
				Console.WriteLine("maintain - delete URL's from database (with HTTP status codes: 301 and 400-406), optimize tables and optimize Solr");
				Console.WriteLine("optimizedb - optimize and flush database");
				Console.WriteLine("optimizesolr - optimize Solr index (if Solr is enabled)");
				return;
			}
			//Create configuration file
			if (args[0] == "createconf")
			{
				Console.Write("Creating configuration file ... ");
				Config.CreateConfig();				
				Console.WriteLine("done.");
				Console.WriteLine("Please now edit it for set correct settings");
				return;
			}
			//Update configuration file
			if (args[0] == "updateconf")
			{
				Console.Write("Updating configuration file ... ");
				Config.UpdateConfig();
				Console.WriteLine("done.");
				Console.WriteLine("Please now edit it for set correct settings");
			}
			//Create database
			if (args[0] == "createdb")
			{
				Console.Write("Creating database ... ");
				Database.CreateDB();
				Console.WriteLine("done.");
				return;
			}
			Database db = new Database();
			db.Open();
			//Backup URL's to plain text file
			if (args[0] == "backup")
			{
				Console.Write("Creating database backup ...");
				int offset = 0;
				//string urls;
				StreamWriter writer = new StreamWriter(args[1]);
				StringBuilder urls2 = new StringBuilder();
				while (true)
				{
					urls2.Append(db.SelectURLs(offset));
					if (urls2.Length == 0)
					{
						break;
					}
					urls2.Append(Environment.NewLine);
					writer.Write(urls2);
					writer.Flush();
					urls2.Remove(0, urls2.Length);
					offset += 1000000;
				}
				writer.Close();
				writer.Dispose();
				Console.WriteLine("done.");
			}
			System.Security.Cryptography.SHA1Managed SHhash = new System.Security.Cryptography.SHA1Managed();
			byte[] HashValue, PureHash;
			System.Text.StringBuilder key = new System.Text.StringBuilder();
			//Insert URL's from file to database
			if (args[0] == "filldb")
			{
				StreamReader reader = new StreamReader(args[1]);
				string record;
				int amount = 0, i = 0;
				while (!reader.EndOfStream)
				{
					record = reader.ReadLine();
					PureHash = Encoding.UTF8.GetBytes(record);
					HashValue = SHhash.ComputeHash(PureHash);
					foreach(byte b in HashValue) 
					{
						key.Append(String.Format("{0:x2}", b));
					}
					amount += db.InstertURL(key.ToString(), record);
					key.Remove(0, key.Length);
					i ++;
				}
				reader.Close();
				reader.Dispose();
				Console.WriteLine("{0} URL's was inserted from {1} URL's.", amount.ToString(), i.ToString());
			}
			//Add single URL to database
			if (args[0] == "addurl")
			{
				PureHash = Encoding.UTF8.GetBytes(args[1]);
				HashValue = SHhash.ComputeHash(PureHash);
				foreach(byte b in HashValue) 
				{
					key.Append(String.Format("{0:x2}", b));
				}
				int amount = db.InstertURL(key.ToString(), args[1]);
				if (amount == 0)
				{
					Console.WriteLine("URL already exist in database.");
				}
				else
				{
					Console.WriteLine("URL added to database.");
				}
			}
			bool solrenabled = false;
			if (Config.ReadConfig("/configuration/enablesolr") == "Y")
			{
				solrenabled = true;
			}
			//Delete single URL from database
			if (args[0] == "delurl")
			{
				PureHash = Encoding.UTF8.GetBytes(args[1]);
				HashValue = SHhash.ComputeHash(PureHash);
				foreach(byte b in HashValue) 
				{
					key.Append(String.Format("{0:x2}", b));
				}
				int amount = db.DeleteURL(key.ToString());
				if (solrenabled)
				{
					string solrcommand = "<delete><id>http://" + args[1] + "</id></delete>";
					solrcommand = Regex.Replace(solrcommand, @"[\p{IsC}]", String.Empty);
					MainClass.SendToSolr(solrcommand);
					solrcommand = "<commit/>";
					MainClass.SendToSolr(solrcommand);
				}
				if (amount == 0)
				{
					Console.WriteLine("URL not exist in database.");
				}
				else
				{
					Console.WriteLine("URL deleted from database.");
				}
			}
			//Delete URL's from file from database
			if (args[0] == "delurls")
			{
				StreamReader reader = new StreamReader(args[1]);
				string record, solrcommand = String.Empty;
				int amount = 0, i = 0;
				while (!reader.EndOfStream)
				{
					record = reader.ReadLine();
					PureHash = Encoding.UTF8.GetBytes(record);
					HashValue = SHhash.ComputeHash(PureHash);
					foreach(byte b in HashValue) 
					{
						key.Append(String.Format("{0:x2}", b));
					}
					amount += db.DeleteURL(key.ToString());
					solrcommand += "<delete><id>http://" + record + "</id></delete>";
					i ++;
					key.Remove(0, key.Length);
				}
				reader.Close();
				reader.Dispose();
				if (solrenabled)
				{
					solrcommand = Regex.Replace(solrcommand, @"[\p{IsC}]", String.Empty);
					MainClass.SendToSolr(solrcommand);
					solrcommand = "<commit/>";
					MainClass.SendToSolr(solrcommand);
				}
				Console.WriteLine("{0} URL's was deleted from {1} URL's.", amount.ToString(), i.ToString());
			}
			//Create workunits from URL's from database
			if (args[0] == "createwu")
			{
				string urls, path, host;
				string[] urlsArray, urlparts;
				int offset = 0;
				urls = db.SelectURLs(-1);
				if (urls.Length == 0)
				{
					return;
				}
				string workunitsdirectory = Config.ReadConfig("/configuration/workunitsdirectory");
				string tempwu = workunitsdirectory + "wu.temp";
				StreamWriter writer = new StreamWriter(tempwu);
				DateTime modtime;
				urlsArray = urls.Split(new char[] {'\n'});
				PureHash = Encoding.UTF8.GetBytes(args[1]);
				HashValue = SHhash.ComputeHash(PureHash);
				foreach(byte b in HashValue) 
				{
					key.Append(String.Format("{0:x2}", b));
				}
				foreach (string url in urlsArray)
				{
					urlparts = url.Split(new char[] {' '});
					if (urlparts.Length != 2)
					{
						continue;
					}
					if (urlparts[0].IndexOf('/') == -1)
					{
						path = "/";
						host = urlparts[0];
					}
					else
					{
						path = urlparts[0].Substring(url.IndexOf('/'));
						host = urlparts[0].Remove(url.IndexOf('/'));
					}
					writer.Write("GET " + path + " HTTP/1.0\r\n");
					writer.Write("Host: " + host + "\r\n");
					writer.Write("User-Agent: GrubNG 0.1 (http://grub.org)\r\n");
					writer.Write("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n");
					if (urlparts[1] != "0")
					{
						modtime = new DateTime(Convert.ToInt32(urlparts[1].Substring(0, 4)), 
						                       Convert.ToInt32(urlparts[1].Substring(4, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(6, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(8, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(10, 2)),
						                       Convert.ToInt32(urlparts[1].Substring(12)));
						writer.Write("If-Modified-Since: " + modtime.ToString("r") + "\r\n");
					}
					writer.Write("\r\n");
					writer.Flush();
					PureHash = Encoding.UTF8.GetBytes(key.ToString() + " " + host + " " + path);
					key.Remove(0, key.Length);
					HashValue = SHhash.ComputeHash(PureHash);
					foreach(byte b in HashValue) 
					{
						key.Append(String.Format("{0:x2}", b));
					}
					offset ++;
					if (offset == 250)
					{
						writer.Close();
						writer.Dispose();
						File.Copy(tempwu, workunitsdirectory + key.ToString() + ".wu1", true);
						File.Delete(tempwu);
						key.Remove(0, key.Length);
						PureHash = Encoding.UTF8.GetBytes(args[1]);
						HashValue = SHhash.ComputeHash(PureHash);
						foreach(byte b in HashValue) 
						{
							key.Append(String.Format("{0:x2}", b));
						}
						writer = new StreamWriter(tempwu);
						offset = 0;
					}
				}
				writer.Close();
				writer.Dispose();
				File.Delete(tempwu);
			}
			db.Close();
			//Maintenance work on URL's database and Solr
			if (args[0] == "maintain")
			{
				string[] codes = new string[] {"204", "301", "400", "401", "402", "403", "404", "405", "406", "408", "503"};
				int amount;
				foreach (string code in codes)
				{
					Console.WriteLine("Deleting URL's with HTTP status code {0}", code);
					db.Open();
					amount = db.CleanURLs(code);
					db.Close();
					Console.WriteLine("{0} URL's was deleted.", amount);
				}
			}
			//Optimize and flush database
			if ((args[0] == "optimizedb") || (args[0] == "maintain"))
			{
				Console.Write("Optimizing database ... ");
				db.Open();
				db.OptimizeDB();
				db.Close();
				Console.WriteLine("done.");
			}
			//Optimize Solr index
			if ((args[0] == "optimizesolr") || (args[0] == "maintain"))
			{
				if (!solrenabled)
				{
					return;
				}
				Console.Write("Optimizing Solr ... ");
				MainClass.SendToSolr("<optimize/>");
				Console.WriteLine("done.");
			}
		}
		
		/// <summary>
		/// Function send XML Update command to Solr
		/// </summary>
		/// <param name="command">
		/// A <see cref="System.String"/> XML Update command to send
		/// </param>
		public static void SendToSolr(string command)
		{
			HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Config.ReadConfig("/configuration/solraddress"));
			request.Proxy = null;
			request.Credentials = new NetworkCredential(Config.ReadConfig("/configuration/solruser"), 
			                                            Config.ReadConfig("configuration/solrpassword"));
			request.PreAuthenticate = true;
			request.ContentType = "text/xml";
			request.Method = "POST";
			request.Timeout = 1000000000;
			Stream streamw1 = request.GetRequestStream();
			byte[] buffer = Encoding.UTF8.GetBytes(command);
			streamw1.Write(buffer, 0, buffer.Length);
			streamw1.Close();
			streamw1.Dispose();
			HttpWebResponse response = (HttpWebResponse)request.GetResponse();
			response.Close();
		}
	}
}