← Back to branch summary

~grubng-dev/grubng/tools-urlsdb

« back to all changes in this revision

Viewing changes to Files.cs

  • Committer: thindil
  • Date: 2011-04-21 10:23:53 UTC
  • Revision ID: thindil2@gmail.com-20110421102353-f6szsgmqmfo8hq5j
some code optimization

Show diffs side-by-side

added added

removed removed

Lines of Context:
Files.cs
27
27
        /// <summary>
28
28
        /// Provide function for manipulate files.
29
29
        /// </summary>
30
 
        internal sealed class Files
 
30
        internal sealed class Files : IDisposable
31
31
        {
 
32
                /// <summary>
 
33
                /// Object for database operations.
 
34
                /// </summary>
32
35
                Database db;
 
36
                /// <summary>
 
37
                /// Object for show messages to user.
 
38
                /// </summary>
33
39
                Utils util;
 
40
                /// <summary>
 
41
                /// If true, all disposable fields are disposed.
 
42
                /// </summary>
 
43
                bool disposed;
34
44
                
 
45
                /// <summary>
 
46
                /// Standard class constructor.
 
47
                /// </summary>
 
48
                /// <param name="quiet">
 
49
                /// A <see cref="System.Boolean"/> if true, don't show messages to user.
 
50
                /// </param>
35
51
                public Files (bool quiet)
36
52
                {
37
53
                        this.db = new Database();
46
62
                /// </param>
47
63
                public void Backup(string filename)
48
64
                {
 
65
                        if (this.disposed)
 
66
                        {
 
67
                                throw new ObjectDisposedException(GetType().Name);
 
68
                        }
49
69
                        this.util.Message("Creating database backup ... ", false);
50
 
                        StreamWriter writer = new StreamWriter(filename);
51
 
                        StringBuilder urls2 = new StringBuilder();
52
 
                        this.util.ProgressStart(0, "URL's was saved");
53
 
                        while (true)
 
70
                        using (StreamWriter writer = new StreamWriter(filename))
54
71
                        {
55
 
                                urls2.Append(this.db.SelectURLs(this.util.Curamount));
56
 
                                if (urls2.Length == 0)
 
72
                                StringBuilder urls2 = new StringBuilder();
 
73
                                this.util.ProgressStart(0, "URL's was saved");
 
74
                                while (true)
57
75
                                {
58
 
                                        break;
 
76
                                        urls2.Append(this.db.SelectURLs(this.util.Curamount));
 
77
                                        if (urls2.Length == 0)
 
78
                                        {
 
79
                                                break;
 
80
                                        }
 
81
                                        urls2.Append(Environment.NewLine);
 
82
                                        writer.Write(urls2);
 
83
                                        writer.Flush();
 
84
                                        urls2.Remove(0, urls2.Length);
 
85
                                        this.util.Curamount += 100000;
 
86
                                        GC.GetTotalMemory(true);
59
87
                                }
60
 
                                urls2.Append(Environment.NewLine);
61
 
                                writer.Write(urls2);
62
 
                                writer.Flush();
63
 
                                urls2.Remove(0, urls2.Length);
64
 
                                this.util.Curamount += 100000;
65
 
                                GC.GetTotalMemory(true);
 
88
                                this.util.ProgressStop();
 
89
                                writer.Close();
66
90
                        }
67
 
                        this.util.ProgressStop();
68
 
                        writer.Close();
69
 
                        writer.Dispose();
70
91
                        this.util.Message(" done.", true);
71
92
                        this.util.Message("Compressing backup file ... ", false);
72
93
                        using (FileStream file = File.OpenRead(filename))
124
145
                /// </param>
125
146
                public void AddURLs(string filename, bool fast)
126
147
                {
 
148
                        if (this.disposed)
 
149
                        {
 
150
                                throw new ObjectDisposedException(GetType().Name);
 
151
                        }
127
152
                        FileInfo finfo = new FileInfo(filename);
128
153
                        if (finfo.Extension == ".gz")
129
154
                        {
156
181
                        int i = 0;
157
182
                        System.Collections.Generic.List<string> records = new System.Collections.Generic.List<string>();
158
183
                        System.Collections.Generic.Dictionary<string, string[]> urls = new System.Collections.Generic.Dictionary<string, string[]>();
159
 
                        ParseURLs parseurl = new ParseURLs();
160
184
                        int amount = 0;
161
185
                        string record, hash1, hash2, record2 = String.Empty;
162
186
                        using (StreamReader reader = new StreamReader(filename))
163
187
                        {
164
 
                                while (!reader.EndOfStream)
 
188
                                using (ParseURLs parseurl = new ParseURLs())
165
189
                                {
166
 
                                        for (int j = 0; j < 25000; j++)
167
 
                                        {
168
 
                                                record2 = reader.ReadLine();
169
 
                                                if (record2 == null)
170
 
                                                {
171
 
                                                        break;
172
 
                                                }
173
 
                                                records.Add(record2);                                           
174
 
                                        }
175
 
                                        //Remove bad URL's and create hash for each
176
 
                                        foreach (string record3 in records)
177
 
                                        {
178
 
                                                record = ParseURLs.ParseURL(record3);
179
 
                                                i ++;
180
 
                                                if (record.Length == 0)
181
 
                                                {
182
 
                                                        continue;
183
 
                                                }
184
 
                                                hash1 = parseurl.GetHash(record);
185
 
                                                if (!record.StartsWith("www."))
186
 
                                                {
187
 
                                                        hash2 = parseurl.GetHash("www." + record);
 
190
                                        while (!reader.EndOfStream)
 
191
                                        {
 
192
                                                for (int j = 0; j < 25000; j++)
 
193
                                                {
 
194
                                                        record2 = reader.ReadLine();
 
195
                                                        if (record2 == null)
 
196
                                                        {
 
197
                                                                break;
 
198
                                                        }
 
199
                                                        records.Add(record2);                                           
 
200
                                                }
 
201
                                                //Remove bad URL's and create hash for each
 
202
                                                foreach (string record3 in records)
 
203
                                                {
 
204
                                                        record = ParseURLs.ParseURL(record3);
 
205
                                                        i ++;
 
206
                                                        if (record.Length == 0)
 
207
                                                        {
 
208
                                                                continue;
 
209
                                                        }
 
210
                                                        hash1 = parseurl.GetHash(record);
 
211
                                                        if (!record.StartsWith("www."))
 
212
                                                        {
 
213
                                                                hash2 = parseurl.GetHash("www." + record);
 
214
                                                        }
 
215
                                                        else
 
216
                                                        {
 
217
                                                                hash2 = parseurl.GetHash(record.Substring(4));
 
218
                                                        }
 
219
                                                        if (!urls.ContainsKey(record))
 
220
                                                        {
 
221
                                                                urls.Add(record, new string[] {hash1, hash2});
 
222
                                                        }
 
223
                                                        this.util.Curamount += record3.Length;
 
224
                                                }
 
225
                                                if (!fast)
 
226
                                                {
 
227
                                                        amount += this.db.InsertURLs(urls, false);
188
228
                                                }
189
229
                                                else
190
230
                                                {
191
 
                                                        hash2 = parseurl.GetHash(record.Substring(4));
192
 
                                                }
193
 
                                                if (!urls.ContainsKey(record))
194
 
                                                {
195
 
                                                        urls.Add(record, new string[] {hash1, hash2});
196
 
                                                }
197
 
                                                this.util.Curamount += record3.Length;
198
 
                                        }
199
 
                                        if (!fast)
200
 
                                        {
201
 
                                                amount += this.db.InsertURLs(urls, false);
202
 
                                        }
203
 
                                        else
204
 
                                        {
205
 
                                                amount += this.db.InsertURLs(urls, true);
206
 
                                        }
207
 
                                        urls.Clear();
208
 
                                        records.Clear();
209
 
                                        records.TrimExcess();
 
231
                                                        amount += this.db.InsertURLs(urls, true);
 
232
                                                }
 
233
                                                urls.Clear();
 
234
                                                records.Clear();
 
235
                                                records.TrimExcess();
 
236
                                        }
210
237
                                }
211
238
                        }
212
239
                        records = null;
213
240
                        this.util.ProgressStop();
214
241
                        this.util.Message(String.Empty, true);
215
242
                        this.util.Message(amount.ToString() + " URL's was inserted from " + i.ToString() + " URL's.", true);
216
 
                        parseurl.Dispose();
 
243
                }
 
244
                
 
245
                /// <summary>
 
246
                /// Function dispose unmanaged resources.
 
247
                /// </summary>
 
248
                public void Dispose()
 
249
                {
 
250
                        if (this.util != null)
 
251
                        {
 
252
                                this.util.Dispose();
 
253
                        }
 
254
                        this.disposed = true;
217
255
                }
218
256
        }
219
257
}