101
80
public void UpdateDB()
103
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
82
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
105
dbcmd.CommandText = "ALTER TABLE `urlslist` ADD `robot_result1` smallint, ADD `robot_result2` smallint, " +
106
"ADD `robot_result3` smallint;";
107
dbcmd.CommandTimeout = 500;
108
int tmp = dbcmd.ExecuteNonQuery();
85
using (IDbCommand dbcmd = dbcon.CreateCommand())
87
dbcmd.CommandText = "ALTER TABLE `urlslist` ADD `robot_result1` smallint, ADD `robot_result2` smallint, " +
88
"ADD `robot_result3` smallint;";
89
dbcmd.CommandTimeout = 500;
90
int tmp = dbcmd.ExecuteNonQuery();
128
112
public int InstertURL(string urlhash, string url, string secondhash)
131
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
115
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
133
dbcmd.CommandTimeout = 500;
134
dbcmd.CommandText = "SELECT count(`url_hash`) FROM `urlslist` WHERE `url_hash`='" + urlhash + "';";
135
IDataReader reader = dbcmd.ExecuteReader();
137
amount = Convert.ToInt32(reader.GetValue(0));
145
dbcmd.CommandText = "SELECT count(`url_hash`) FROM `urlslist` WHERE `url_hash`='" + secondhash + "';";
146
reader = dbcmd.ExecuteReader();
148
amount = Convert.ToInt32(reader.GetValue(0));
157
dbcmd.CommandText = "INSERT INTO `urlslist` (`url_hash`, `url`, `last_visited`, " +
158
"`http_code1`, `http_code2`, `http_code3`, `robot_check`, " +
159
"`robot_result1`, `robot_result2`, `robot_result3`," +
160
"`user_crawl`, `user_robots`)VALUES (\"" + urlhash + "\", \"" + url +
161
"\", 0, 0, 0, 0, 0, 0, 0, 0, \"\", \"\");";
164
amount = dbcmd.ExecuteNonQuery();
166
catch (MySqlException)
118
using (IDbCommand dbcmd = dbcon.CreateCommand())
120
dbcmd.CommandTimeout = 500;
121
dbcmd.CommandText = "SELECT count(`url_hash`) FROM `urlslist` WHERE `url_hash`='" + urlhash + "';";
122
IDataReader reader = dbcmd.ExecuteReader();
124
amount = Convert.ToInt32(reader.GetValue(0));
132
dbcmd.CommandText = "SELECT count(`url_hash`) FROM `urlslist` WHERE `url_hash`='" + secondhash + "';";
133
reader = dbcmd.ExecuteReader();
135
amount = Convert.ToInt32(reader.GetValue(0));
144
dbcmd.CommandText = "INSERT INTO `urlslist` (`url_hash`, `url`, `last_visited`, " +
145
"`http_code1`, `http_code2`, `http_code3`, `robot_check`, " +
146
"`robot_result1`, `robot_result2`, `robot_result3`," +
147
"`user_crawl`, `user_robots`)VALUES (\"" + urlhash + "\", \"" + url +
148
"\", 0, 0, 0, 0, 0, 0, 0, 0, \"\", \"\");";
151
amount = dbcmd.ExecuteNonQuery();
153
catch (MySqlException)
183
172
public int DeleteURL(string urlhash)
186
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
175
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
188
dbcmd.CommandText = "DELETE FROM `urlslist` WHERE `url_hash`='" + urlhash + "';";
189
dbcmd.CommandTimeout = 500;
192
amount = dbcmd.ExecuteNonQuery();
194
catch (MySqlException)
178
using (IDbCommand dbcmd = dbcon.CreateCommand())
180
dbcmd.CommandText = "DELETE FROM `urlslist` WHERE `url_hash`='" + urlhash + "';";
181
dbcmd.CommandTimeout = 500;
184
amount = dbcmd.ExecuteNonQuery();
186
catch (MySqlException)
212
206
public string SelectURLs(int offset)
214
208
System.Text.StringBuilder returnvalue = new System.Text.StringBuilder();
215
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
209
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
217
dbcmd.CommandTimeout = 500;
218
dbcmd.CommandText = "SET session sort_buffer_size=100000000;";
219
dbcmd.ExecuteNonQuery();
220
dbcmd.CommandText = "SET read_rnd_buffer_size=100000000;";
221
dbcmd.ExecuteNonQuery();
224
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url`, `last_visited` FROM `urlslist` ORDER BY `last_visited` ASC LIMIT 250000;";
228
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url` FROM `urlslist` LIMIT 250000 OFFSET " + offset.ToString() + ";";
230
IDataReader reader = dbcmd.ExecuteReader();
231
while (reader.Read())
233
returnvalue.Append(Convert.ToString(reader.GetValue(0)));
212
using (IDbCommand dbcmd = dbcon.CreateCommand())
214
dbcmd.CommandTimeout = 500;
215
dbcmd.CommandText = "SET session sort_buffer_size=100000000;";
216
dbcmd.ExecuteNonQuery();
217
dbcmd.CommandText = "SET read_rnd_buffer_size=100000000;";
218
dbcmd.ExecuteNonQuery();
234
219
if (offset == -1)
236
returnvalue.Append(' ');
237
returnvalue.Append(Convert.ToString(reader.GetValue(1)));
239
returnvalue.Append("\n");
221
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url`, `last_visited` FROM `urlslist` ORDER BY `last_visited` ASC LIMIT 250000;";
225
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url` FROM `urlslist` LIMIT 250000 OFFSET " + offset.ToString() + ";";
227
IDataReader reader = dbcmd.ExecuteReader();
228
while (reader.Read())
230
returnvalue.Append(Convert.ToString(reader.GetValue(0)));
233
returnvalue.Append(' ');
234
returnvalue.Append(Convert.ToString(reader.GetValue(1)));
236
returnvalue.Append("\n");
240
dbcmd.CommandText = "SET sort_buffer_size=DEFAULT;";
241
dbcmd.ExecuteNonQuery();
242
dbcmd.CommandText = "SET read_rnd_buffer_size=DEFAULT;";
243
dbcmd.ExecuteNonQuery();
243
dbcmd.CommandText = "SET sort_buffer_size=DEFAULT;";
244
dbcmd.ExecuteNonQuery();
245
dbcmd.CommandText = "SET read_rnd_buffer_size=DEFAULT;";
246
dbcmd.ExecuteNonQuery();
248
247
return returnvalue.ToString().TrimEnd(new char[] {' ', '\n'});
263
262
public int CleanURLs(string status, bool solrenabled)
266
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
265
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
268
dbcmd.CommandTimeout = 1000;
269
//Delete this same URL's from Solr too
268
using (IDbCommand dbcmd = dbcon.CreateCommand())
272
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url` FROM `urlslist` WHERE `http_code1`=" + status + " AND `http_code2`=" +
273
status + " AND `http_code3`=" + status + ";";
274
IDataReader reader = dbcmd.ExecuteReader();
275
System.Text.StringBuilder solrcommand = new System.Text.StringBuilder();
276
solrcommand.Append("<delete>");
277
string scommand = String.Empty;
278
while (reader.Read())
270
dbcmd.CommandTimeout = 1000;
271
//Delete this same URL's from Solr too
280
solrcommand.Append("<id>http://");
281
solrcommand.Append(System.Security.SecurityElement.Escape(reader.GetString(0)));
282
solrcommand.Append("</id>");
274
dbcmd.CommandText = "SELECT SQL_NO_CACHE `url` FROM `urlslist` WHERE `http_code1`=" + status + " AND `http_code2`=" +
275
status + " AND `http_code3`=" + status + ";";
276
IDataReader reader = dbcmd.ExecuteReader();
277
System.Text.StringBuilder solrcommand = new System.Text.StringBuilder();
278
solrcommand.Append("<delete>");
279
string scommand = String.Empty;
280
while (reader.Read())
286
solrcommand.Append("</delete>");
287
scommand = System.Text.RegularExpressions.Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
288
MainClass.SendToSolr(scommand);
289
MainClass.SendToSolr("<commit/>");
290
solrcommand.Remove(0, solrcommand.Length);
291
solrcommand.Append("<delete>");
282
solrcommand.Append("<id>http://");
283
solrcommand.Append(System.Security.SecurityElement.Escape(reader.GetString(0)));
284
solrcommand.Append("</id>");
288
solrcommand.Append("</delete>");
289
scommand = System.Text.RegularExpressions.Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
290
MainClass.SendToSolr(scommand);
291
MainClass.SendToSolr("<commit/>");
292
solrcommand.Remove(0, solrcommand.Length);
293
solrcommand.Append("<delete>");
297
solrcommand.Append("</delete>");
298
scommand = System.Text.RegularExpressions.Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
299
MainClass.SendToSolr(scommand);
300
MainClass.SendToSolr("<commit/>");
302
dbcmd.CommandText = "DELETE FROM `urlslist` WHERE `http_code1`=" + status + " AND `http_code2`=" + status +
303
" AND `http_code3`=" + status + ";";
306
amount = dbcmd.ExecuteNonQuery();
308
catch (MySqlException)
299
solrcommand.Append("</delete>");
300
scommand = System.Text.RegularExpressions.Regex.Replace(solrcommand.ToString(), @"[\p{IsC}]", String.Empty);
301
MainClass.SendToSolr(scommand);
302
MainClass.SendToSolr("<commit/>");
304
dbcmd.CommandText = "DELETE FROM `urlslist` WHERE `http_code1`=" + status + " AND `http_code2`=" + status +
305
" AND `http_code3`=" + status + ";";
308
amount = dbcmd.ExecuteNonQuery();
310
catch (MySqlException)
319
323
public void OptimizeDB()
321
using (IDbCommand dbcmd = this.dbcon.CreateCommand())
325
using (IDbConnection dbcon = new MySqlConnection(this.connectionString))
323
dbcmd.CommandText = "OPTIMIZE TABLE `urlslist`, `weekly`, `monthly`, `yearly`;";
324
dbcmd.CommandTimeout = 5000;
328
amount = dbcmd.ExecuteNonQuery();
330
catch (MySqlException)
333
dbcmd.CommandText = "FLUSH TABLES;";
336
amount = dbcmd.ExecuteNonQuery();
338
catch (MySqlException)
328
using (IDbCommand dbcmd = dbcon.CreateCommand())
330
dbcmd.CommandText = "OPTIMIZE TABLE `urlslist`, `weekly`, `monthly`, `yearly`;";
331
dbcmd.CommandTimeout = 5000;
335
amount = dbcmd.ExecuteNonQuery();
337
catch (MySqlException)
340
dbcmd.CommandText = "FLUSH TABLES;";
343
amount = dbcmd.ExecuteNonQuery();
345
catch (MySqlException)
346
/// Function dispose unmanaged resources.
348
public void Dispose()
350
this.dbcon.Dispose();