~grubng-dev/grubng/tools-urlsdb

//  
//  Copyright (C) 2011 Bartek thindil Jasicki
// 
//  This file is part of Grubng
// 
//  Grubng is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
// 
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
// 
//  You should have received a copy of the GNU General Public License
//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
// 
using System;
using Grubng;
using System.Text;
using System.IO;
using System.IO.Compression;

namespace urlsdb
{
	/// <summary>
	/// Provide function for manipulate files.
	/// </summary>
	internal sealed class Files : IDisposable
	{
		/// <summary>
		/// Object for database operations.
		/// </summary>
		Database db;
		/// <summary>
		/// Object for show messages to user.
		/// </summary>
		Utils util;
		/// <summary>
		/// If true, all disposable fields are disposed.
		/// </summary>
		bool disposed;
		
		/// <summary>
		/// Standard class constructor.
		/// </summary>
		/// <param name="quiet">
		/// A <see cref="System.Boolean"/> if true, don't show messages to user.
		/// </param>
		public Files (bool quiet)
		{
			this.db = new Database();
			this.util = new Utils(quiet);
		}
		
		/// <summary>
		/// Function create database backup to selected file.
		/// </summary>
		/// <param name="filename">
		/// A <see cref="System.String"/> name of backup file.
		/// </param>
		public void Backup(string filename)
		{
			if (this.disposed)
			{
				throw new ObjectDisposedException(GetType().Name);
			}
			this.util.Message("Creating database backup ... ", false);
			using (StreamWriter writer = new StreamWriter(filename))
			{
				StringBuilder urls2 = new StringBuilder();
				this.util.ProgressStart(0, "URL's was saved");
				while (true)
				{
					urls2.Append(this.db.SelectURLs(this.util.Curamount));
					if (urls2.Length == 0)
					{
						break;
					}
					urls2.Append(Environment.NewLine);
					writer.Write(urls2);
					writer.Flush();
					urls2.Remove(0, urls2.Length);
					this.util.Curamount += 100000;
					GC.GetTotalMemory(true);
				}
				this.util.ProgressStop();
				writer.Close();
			}
			this.util.Message(" done.", true);
			this.util.Message("Compressing backup file ... ", false);
			using (FileStream file = File.OpenRead(filename))
			{
				using (FileStream workunit = File.Create(filename + ".lzma"))
				{
					long length = file.Length;
					if (length > 4194304)
					{
						length = 4194304;
					}
					object[] properties = {	(int)(length),
						(int)(2),
						(int)(3),
						(int)(0),
						(int)(2),
						(int)(128),
						"bt4",
						false
					};
					using (Grubng.Encoder encoder = new Grubng.Encoder())
					{
						CoderPropID[] propIDs = {CoderPropID.DictionarySize,
							CoderPropID.PosStateBits,
							CoderPropID.LitContextBits,
							CoderPropID.LitPosBits,
							CoderPropID.Algorithm,
							CoderPropID.NumFastBytes,
							CoderPropID.MatchFinder,
							CoderPropID.EndMarker
						};
						encoder.SetCoderProperties(propIDs, properties);
						encoder.WriteCoderProperties(workunit);
						length = file.Length;
						for (int i = 0; i < 8; i++)
						{
							workunit.WriteByte((Byte)(length >> (8 * i)));
						}
						encoder.Code(file, workunit);
						properties = null;
					}
				}
			}
			File.Delete(filename);
			this.util.Message("done", true);
		}
		
		/// <summary>
		/// Function read list of URL's from file and send they to database.
		/// </summary>
		/// <param name="filename">
		/// A <see cref="System.String"/> filename with URL's.
		/// </param>
		/// <param name="fast">
		/// A <see cref="System.Boolean"/> if true, do fast insert URL's to database.
		/// </param>
		public void AddURLs(string filename, bool fast)
		{
			if (this.disposed)
			{
				throw new ObjectDisposedException(GetType().Name);
			}
			FileInfo finfo = new FileInfo(filename);
			if (finfo.Extension == ".gz")
			{
				this.util.Message("Decompressing file ... ", false);
				string newfilename = filename.Remove(filename.Length - 3);
				using (FileStream file2 = File.OpenWrite(newfilename))
				{
					using (FileStream file = File.OpenRead(filename))
					{
						using (GZipStream gzstream = new GZipStream(file, CompressionMode.Decompress))
						{
							byte[] buffer = new byte[1024];
							int bytesRead = 0;
							while ((bytesRead = gzstream.Read(buffer, 0, buffer.Length)) != 0)
							{
								file2.Write(buffer, 0, bytesRead);
								file2.Flush();
							}
						}
					}
				}
				filename = newfilename;
				finfo = new FileInfo(filename);
				this.util.Message("done", true);
			}
			this.util.Message("Uploading URL's to database ... ", false);
			int length = (int)finfo.Length;
			finfo = null;
			this.util.ProgressStart(length, String.Empty);
			int i = 0;
			System.Collections.Generic.List<string> records = new System.Collections.Generic.List<string>();
			System.Collections.Generic.Dictionary<string, string[]> urls = new System.Collections.Generic.Dictionary<string, string[]>();
			int amount = 0;
			string record, hash1, hash2, record2 = String.Empty;
			using (StreamReader reader = new StreamReader(filename))
			{
				using (ParseURLs parseurl = new ParseURLs())
				{
					while (!reader.EndOfStream)
					{
						for (int j = 0; j < 25000; j++)
						{
							record2 = reader.ReadLine();
							if (record2 == null)
							{
								break;
							}
							if (records.Contains(record2))
							{
								continue;
							}
							if ((record2.StartsWith("www.")) && (records.Contains(record2.Substring(4))))
							{
								continue;
							}
							if ((!record2.StartsWith("www.")) && (record2.Contains("www." + record2)))
							{
								continue;
							}
							records.Add(record2);
						}
						//Remove bad URL's and create hash for each
						foreach (string record3 in records)
						{
							record = ParseURLs.ParseURL(record3);
							i ++;
							if (record.Length == 0)
							{
								continue;
							}
							hash1 = parseurl.GetHash(record);
							if (!record.StartsWith("www."))
							{
								hash2 = parseurl.GetHash("www." + record);
							}
							else
							{
								hash2 = parseurl.GetHash(record.Substring(4));
							}
							if (!urls.ContainsKey(record))
							{
								urls.Add(record, new string[] {hash1, hash2});
							}
							this.util.Curamount += record3.Length;
						}
						if (!fast)
						{
							amount += this.db.InsertURLs(urls, false);
						}
						else
						{
							amount += this.db.InsertURLs(urls, true);
						}
						urls.Clear();
						records.Clear();
						records.TrimExcess();
					}
				}
			}
			records = null;
			this.util.ProgressStop();
			this.util.Message(String.Empty, true);
			this.util.Message(amount.ToString() + " URL's was inserted from " + i.ToString() + " URL's.", true);
		}
		
		/// <summary>
		/// Function dispose unmanaged resources.
		/// </summary>
		public void Dispose()
		{
			if (this.util != null)
			{
				this.util.Dispose();
			}
			this.disposed = true;
		}
	}
}