~mark-vejvoda/ipt-parse/trunk : revision 3

5

#include <dirent.h>

6

#include <time.h>

7

8

//#include <sys/statvfs.h>

9

#include <errno.h>

10

#include <sys/stat.h>

11

8

12

#ifndef __WINDOWS__

9

13

10

14

#include <sys/socket.h>

17

21

18

22

#endif

19

23

20

#define IPT_VERSION "1.3"

24

#define IPT_VERSION "1.3.1"

21

25

//#define DEBUG_CODE

22

26

//#define DEBUG_HOST "192.168.0.3"

23

27

24

28

// Maximum days in date range to store data

25

#define MAX_RANGE 90

29

#define MAX_RANGE 91

26

30

// Maximum # of clients to keep track of

27

31

#define MAX_CLIENTCOUNT 101

28

32

50

54

const int GENERATE_TYPE_SQLITE_HTML = 4;

51

55

const int GENERATE_TYPE_SQLITE_DATACOMPRESS = 5;

52

56

const int GENERATE_TYPE_SETUP_IPTABLE_USING_TOMATO_STATIC_DHCP = 6;

57

const int GENERATE_TYPE_SHRINK_DATABASE = 7;

53

58

54

59

static uint64_t start_date = 0;

55

60

static uint64_t end_date = 0;

65

70

static const char *APP_FLAG_HOSTNAME_ONLY = "flags=hostname_only";

66

71

static const char *APP_FLAG_IMPORT_DAILY_ONLY = "flags=importdailyonly";

67

72

static const char *APP_FLAG_IMPORT_LOGFILEDATA = "flags=import=";

73

static const char *APP_FLAG_PURGE_OLD_RECORDS = "flags=purgeoldrecords=";

68

74

69

75

typedef struct

70

76

{

110

116

} ipt_select_cbd;

111

117

112

118

// =================================================================================

119

int IsParameterEnabled(const char *pParam, int argc, char *argv[])

120

{

121

int iParameterEnabled = 0;

122

123

int iLookupLen = strlen(pParam);

124

125

int iIdx = 0;

126

for(iIdx = 1;

127

iIdx < argc;

128

iIdx ++)

129

{

130

char *pArgParam = argv[iIdx];

131

if(strncasecmp(pArgParam,pParam,iLookupLen) == 0)

132

{

133

iParameterEnabled = 1;

134

break;

135

}

136

}

137

138

return iParameterEnabled;

139

}

140

141

// =================================================================================

113

142

double ConvertMBytesToNumber(double bytes, char cConvertToType)

114

143

{

115

144

switch(cConvertToType)

257

286

if(tNewDate < 0)

258

287

{

259

288

char szBuf[100]="";

260

sprintf(szBuf,"[getDateValue] Invalid date conversion value specified, returned: [%ld]",tNewDate);

289

sprintf(szBuf,"[%s] Invalid date conversion value specified, returned: [%ld]",__FUNCTION__,tNewDate);

261

290

perror(szBuf);

262

291

}

263

292

342

371

343

372

#ifdef USE_SQLITE

344

373

374

static int stub_callback(void *NotUsed, int argc, char **argv, char **azColName){

375

printf("In [%s] \n",__FUNCTION__);

376

fflush(stdout);

377

return 0;

378

}

379

// ==========================================================================

380

345

381

static int callback(void *NotUsed, int argc, char **argv, char **azColName){

346

382

int i=0;

347

383

for(i=0; i<argc; i++)

405

441

printf("\n");

406

442

#endif

407

443

408

//!!!

409

444

char *pLogDateTime = argv[DATE_COLUMN_INDEX];

410

445

if(pLogDateTime != NULL)

411

446

{

554

589

if(tNewDate < 0)

555

590

{

556

591

char szBuf[100]="";

557

sprintf(szBuf,"Invalid date conversion value specified, adding [%d] returned: [%ld]",iAddDays,tNewDate);

592

sprintf(szBuf,"In [%s] Invalid date conversion value specified, adding [%d] returned: [%ld]",__FUNCTION__,iAddDays,tNewDate);

558

593

perror(szBuf);

559

594

}

560

595

1099

1134

if( rc )

1100

1135

{

1101

1136

fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));

1137

fflush(stderr);

1138

1102

1139

sqlite3_close(db);

1103

1140

exit(1);

1104

1141

}

1188

1225

if(iGenerateType == GENERATE_TYPE_SQLITE_CONSOLE || iGenerateType == GENERATE_TYPE_SQLITE_DATACOMPRESS)

1189

1226

{

1190

1227

fprintf(stderr, "Selecting SQL [%s]\n", szSQL);

1228

fflush(stderr);

1191

1229

}

1192

1230

1193

1231

ipt_select_cbd tCBD;

1206

1244

if( rc != SQLITE_OK )

1207

1245

{

1208

1246

fprintf(stderr, "SQL error: %s\n", zErrMsg);

1247

fflush(stderr);

1248

1209

1249

sqlite3_free(zErrMsg);

1210

1250

}

1211

1251

1213

1253

}

1214

1254

// ==========================================================================

1215

1255

1256

int CanShrinkDatabase(char *pDB)

1257

{

1258

int iReturn = 0;

1259

1260

#ifdef __WINDOWS__

1261

1262

iReturn = 1;

1263

1264

#else

1265

1266

char szBuf[1024]="";

1267

sprintf(szBuf,"df %s",pDB);

1268

1269

FILE *fp = popen(szBuf, "r");

1270

if(fp == NULL)

1271

{

1272

fprintf(stderr, "Can't open pipe for command [%s]\n", szBuf);

1273

}

1274

else

1275

{

1276

char szDiscard[1024] = "";

1277

for(;feof(fp) == 0 && strstr(szDiscard,"Mounted on") == NULL;)

1278

{

1279

fgets(szDiscard,1023,fp);

1280

}

1281

1282

if(feof(fp) == 0)

1283

{

1284

char szFileSystem[300]="";

1285

1286

char sz1KBlocks[50]="";

1287

char szUsed[50]="";

1288

char szAvailable[50]="";

1289

char szUsedPercent[300]="";

1290

char szMountedOn[300]="";

1291

1292

fscanf(fp, "%s %s %s %s %s %s",

1293

szFileSystem,

1294

sz1KBlocks,

1295

szUsed,

1296

szAvailable,

1297

szUsedPercent,

1298

szMountedOn);

1299

1300

char *pEnd=0;

1301

double d1KBlocks=strtod(sz1KBlocks,&pEnd);

1302

double dUsed=strtod(szUsed,&pEnd);

1303

double dAvailable=strtod(szAvailable,&pEnd);

1304

1305

double dBytesFree = (dAvailable * d1KBlocks);

1306

1307

struct stat fBuf;

1308

if(stat(pDB, &fBuf) < 0)

1309

{

1310

printf("ERROR: %s\n", strerror(errno));

1311

fflush(stdout);

1312

}

1313

else

1314

{

1315

if(dBytesFree >= fBuf.st_size)

1316

{

1317

iReturn = 1;

1318

}

1319

1320

printf("Filesystem\t1K-blocks\tUsed\tAvailable\tUse%%\tMounted on\tFILESIZE\tBYTES FREE\n");

1321

printf("%s\t%.2f\t\t%.2f\t%.2f\t%s\t%s\t%i\t%.2f\n",

1322

szFileSystem,

1323

d1KBlocks,

1324

dUsed,

1325

dAvailable,

1326

szUsedPercent,

1327

szMountedOn,

1328

fBuf.st_size,

1329

dBytesFree);

1330

fflush(stdout);

1331

}

1332

}

1333

1334

pclose(fp);

1335

}

1336

1337

/*

1338

struct statvfs fiData;

1339

struct statvfs *fpData;

1340

char fnPath[128];

1341

1342

strcpy(fnPath, pDB);

1343

if((statvfs(fnPath,&fiData)) < 0 )

1344

{

1345

printf("Failed to stat %s:\n", fnPath);

1346

fflush(stdout);

1347

}

1348

else

1349

{

1350

printf("Disk %s: \n", fnPath);

1351

printf("\tblock size: %lu\n", fiData.f_bsize);

1352

printf("\ttotal no blocks: %u\n", fiData.f_blocks);

1353

printf("\tfree blocks: %u\n", fiData.f_bfree);

1354

fflush(stdout);

1355

1356

iReturn = 0;

1357

}

1358

*/

1359

#endif

1360

1361

return iReturn;

1362

}

1363

1364

void vacuumDB(char *pDB)

1365

{

1366

sqlite3 *db=0;

1367

char *zErrMsg = 0;

1368

1369

int rc = sqlite3_open(pDB, &db);

1370

if( rc )

1371

{

1372

fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));

1373

fflush(stderr);

1374

1375

sqlite3_close(db);

1376

exit(1);

1377

}

1378

1379

//int bCanShrink = CanShrinkDatabase(pDB);

1380

1381

//fprintf(stderr, "Records Deleted [%d] check to shrink database = [%d]...\n", iTotalRowsDelete,bCanShrink);

1382

//fflush(stderr);

1383

1384

//if(bCanShrink)

1385

{

1386

rc = sqlite3_exec(db, "VACUUM;", stub_callback, 0, &zErrMsg);

1387

if( rc != SQLITE_OK )

1388

{

1389

fprintf(stderr, "VACUUM SQL error: %s\n", zErrMsg);

1390

fflush(stderr);

1391

1392

sqlite3_free(zErrMsg);

1393

}

1394

}

1395

1396

fprintf(stderr, "Closing after shrink database...\n");

1397

fflush(stderr);

1398

1399

sqlite3_close(db);

1400

}

1401

1216

1402

int deleteStatsFromSqlite(char *pDB, uint64_t startDate, uint64_t endDate)

1217

1403

{

1218

1404

int iTotalRowsDelete = 0;

1230

1416

szEndDate);

1231

1417

1232

1418

fprintf(stderr, "Delete SQL [%s]\n", szSQL);

1419

fflush(stderr);

1233

1420

1234

1421

sqlite3 *db=0;

1235

1422

char *zErrMsg = 0;

1238

1425

if( rc )

1239

1426

{

1240

1427

fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));

1428

fflush(stderr);

1429

1241

1430

sqlite3_close(db);

1242

1431

exit(1);

1243

1432

}

1246

1435

if( rc != SQLITE_OK )

1247

1436

{

1248

1437

fprintf(stderr, "DELETE SQL error: %s\n", zErrMsg);

1438

fflush(stderr);

1439

1249

1440

sqlite3_free(zErrMsg);

1250

1441

}

1251

1442

1252

1443

int iChangeCount = sqlite3_changes(db);

1253

1444

iTotalRowsDelete = iChangeCount;

1254

1445

1255

rc = sqlite3_exec(db, "VACUUM;", callback, 0, &zErrMsg);

1256

if( rc != SQLITE_OK )

1446

fprintf(stderr, "Closing database...\n");

1447

fflush(stderr);

1448

1449

sqlite3_close(db);

1450

1451

int bCanShrink = CanShrinkDatabase(pDB);

1452

1453

fprintf(stderr, "Records Deleted [%d] check to shrink database = [%d]...\n", iTotalRowsDelete,bCanShrink);

1454

fflush(stderr);

1455

1456

if(bCanShrink)

1257

1457

{

1258

fprintf(stderr, "VACUUM SQL error: %s\n", zErrMsg);

1259

sqlite3_free(zErrMsg);

1458

vacuumDB(pDB);

1260

1459

}

1261

1460

1262

sqlite3_close(db);

1263

1264

1461

return iTotalRowsDelete;

1265

1462

}

1266

1463

// ==========================================================================

1267

1464

1268

int insertStatsIntoSqlite(char *pDateTimeStamp, sqlite3 *db, data_t tBandwidthUsage[], data_t tBandwidthUsageInitValue[])

1465

int insertStatsIntoSqlite( char *pDateTimeStamp, sqlite3 *db, data_t tBandwidthUsage[], data_t tBandwidthUsageInitValue[],

1466

int iForceDailyInsertOrUpdate)

1269

1467

{

1270

1468

int iTotalRowsInserted = 0;

1271

1469

int iMaxDay_index = GetDayIndex(start_date,end_date);

1365

1563

pBandwidthUsage->ip,

1366

1564

pDateTimeStamp);

1367

1565

}

1566

else if(iForceDailyInsertOrUpdate == 1)

1567

{

1568

sprintf(szSQL,"INSERT INTO bandwidthusage (log_datetime,host,in_mb,out_mb,mac) SELECT '%s','%s',%f,%f,'' WHERE (SELECT COUNT(*) FROM bandwidthusage WHERE host='%s' AND date(log_datetime)=date('%s')) = 0;",

1569

pDateTimeStamp,

1570

pBandwidthUsage->ip,

1571

bytes_in_total,

1572

bytes_out_total,

1573

pBandwidthUsage->ip,

1574

pDateTimeStamp);

1575

}

1368

1576

1369

fprintf(stderr, "Inserting SQL [%s]\n", szSQL);

1577

fprintf(stderr, "Trying to Insert SQL [%s]\n", szSQL);

1578

fflush(stderr);

1579

//fflush(stdout);

1370

1580

1371

1581

int rc = sqlite3_exec(db, szSQL, callback, 0, &zErrMsg);

1372

1582

if( rc != SQLITE_OK )

1373

1583

{

1374

1584

fprintf(stderr, "SQL error: %s\n", zErrMsg);

1585

//fflush(stderr);

1586

//fflush(stdout);

1587

1375

1588

sqlite3_free(zErrMsg);

1376

1589

1377

1590

break;

1378

1591

}

1379

1592

1380

1593

int iChangeCount = sqlite3_changes(db);

1594

1595

if(iChangeCount == 0 && iForceDailyInsertOrUpdate == 1)

1596

{

1597

sprintf(szSQL,"UPDATE bandwidthusage SET in_mb=in_mb+%f, out_mb=out_mb+%f WHERE host='%s' AND date(log_datetime)=date('%s');",

1598

bytes_in_total,

1599

bytes_out_total,

1600

pBandwidthUsage->ip,

1601

pDateTimeStamp);

1602

1603

fprintf(stderr, "Trying to Update SQL [%s]\n", szSQL);

1604

fflush(stderr);

1605

//fflush(stdout);

1606

1607

rc = sqlite3_exec(db, szSQL, callback, 0, &zErrMsg);

1608

if( rc != SQLITE_OK )

1609

{

1610

fprintf(stderr, "SQL error: %s\n", zErrMsg);

1611

//fflush(stderr);

1612

//fflush(stdout);

1613

1614

sqlite3_free(zErrMsg);

1615

1616

break;

1617

}

1618

1619

iChangeCount = sqlite3_changes(db);

1620

}

1621

1381

1622

iTotalRowsInserted += iChangeCount;

1382

1623

}

1383

1624

}

1460

1701

char szNowStamp[40]="";

1461

1702

strftime (szNowStamp, 39, "%Y-%m-%d %H:%M:%S", tm_now);

1462

1703

1463

int iTotalRowsInserted = insertStatsIntoSqlite(szNowStamp, db, tBandwidthUsage, tBandwidthUsageInitValue);

1704

int bImportStatsDailyOnly = IsParameterEnabled(APP_FLAG_IMPORT_DAILY_ONLY, argc, argv);

1705

1706

int iTotalRowsInserted = insertStatsIntoSqlite(szNowStamp, db, tBandwidthUsage, tBandwidthUsageInitValue,bImportStatsDailyOnly);

1464

1707

sqlite3_close(db);

1465

1708

1466

1709

fprintf(stderr, "*NOTE: Total rows inserted = %d\n", iTotalRowsInserted);

1485

1728

exit(1);

1486

1729

}

1487

1730

1488

int iTotalRowsInserted = insertStatsIntoSqlite(NULL, db, tBandwidthUsage, tBandwidthUsageInitValue);

1731

int iTotalRowsInserted = insertStatsIntoSqlite(NULL, db, tBandwidthUsage, tBandwidthUsageInitValue,0);

1489

1732

sqlite3_close(db);

1490

1733

1491

1734

fprintf(stderr, "*NOTE: Total rows inserted = %d\n", iTotalRowsInserted);

1631

1874

}

1632

1875

// ==========================================================================

1633

1876

1634

void ProcessSpecialDateParameter(char *pLogFileFilter)

1877

void ProcessSpecialDateParameter(char *pLogFileFilter, int bNeedSeparators)

1635

1878

{

1636

1879

time_t now = time ( NULL );

1637

1880

struct tm *tm_now = localtime ( &now );

1638

1881

1639

1882

if(strlen(pLogFileFilter) == 0)

1640

1883

{

1641

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1884

if(bNeedSeparators)

1885

{

1886

strftime (pLogFileFilter, 1023, "%Y-%m-%d", tm_now);

1887

}

1888

else

1889

{

1890

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1891

}

1642

1892

}

1643

1893

else if(strstr(pLogFileFilter,TODAYS_DATE_KEYWORD) != NULL)

1644

1894

{

1658

1908

1659

1909

tm_now = localtime ( &tNewDate );

1660

1910

}

1661

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1911

1912

if(bNeedSeparators)

1913

{

1914

strftime (pLogFileFilter, 1023, "%Y-%m-%d", tm_now);

1915

}

1916

else

1917

{

1918

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1919

}

1662

1920

}

1663

1921

else if(strncasecmp(pLogFileFilter,MONTHLY_PERIOD_STARTDATE_KEYWORD,strlen(MONTHLY_PERIOD_STARTDATE_KEYWORD)) == 0)

1664

1922

{

1682

1940

}

1683

1941

1684

1942

tm_now = localtime ( &tNewDate );

1685

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1943

1944

if(bNeedSeparators)

1945

{

1946

strftime (pLogFileFilter, 1023, "%Y-%m-%d", tm_now);

1947

}

1948

else

1949

{

1950

strftime (pLogFileFilter, 1023, "%Y%m%d", tm_now);

1951

}

1686

1952

}

1687

1953

}

1688

1954

// ==========================================================================

1689

1955

1690

int IsParameterEnabled(char *pParam, int argc, char *argv[])

1956

int GetParameterIndex(const char *pParam, int argc, char *argv[])

1691

1957

{

1692

int iParameterEnabled = 0;

1958

int iParameterIndex = -1;

1693

1959

1694

1960

int iLookupLen = strlen(pParam);

1695

1961

1701

1967

char *pArgParam = argv[iIdx];

1702

1968

if(strncasecmp(pArgParam,pParam,iLookupLen) == 0)

1703

1969

{

1704

iParameterEnabled = 1;

1970

iParameterIndex = iIdx;

1705

1971

break;

1706

1972

}

1707

1973

}

1708

1974

1709

return iParameterEnabled;

1975

return iParameterIndex;

1710

1976

}

1711

1977

1712

1978

int AutoSetupBandwidthMonitoring(data_t tBandwidthUsage[], const char *pIPTableName)

1713

1979

{

1714

1980

int iHostsDetected = 0;

1715

//!!!

1716

1981

char szCmd[1024] = "nvram get dhcpd_static";

1717

1982

1718

1983

fprintf(stderr, "Executing get static hosts command [%s]\n", szCmd);

1846

2111

#endif

1847

2112

}

1848

2113

1849

//!!!

1850

2114

int iIdx = 0;

1851

2115

for(iIdx = 0;

1852

2116

iIdx < MIN(iHostsDetected,MAX_CLIENTCOUNT);

1914

2178

data_t tBandwidthUsage[], data_t tBandwidthUsageInitValue[], int iGenerateType)

1915

2179

{

1916

2180

printf("Using filter date-range [%s] - [%s]\n",szLogFileStartFilter,szLogFileEndFilter);

2181

fflush(stdout);

1917

2182

1918

2183

// Step #1 generate daily stats from existing intra-day records for specified date range

1919

2184

mainSelectStatsByCriteria(szDatabase,szLogFileStartFilter,szLogFileEndFilter,tBandwidthUsage,tBandwidthUsageInitValue,iGenerateType);

1920

2185

2186

printf("About to delete records for date-range [%llu] - [%llu]\n",start_date,end_date);

2187

fflush(stdout);

2188

1921

2189

// Step #2 delete old records for specified date range

1922

2190

int iDeleteRows = deleteStatsFromSqlite(szDatabase, start_date, end_date);

1923

2191

2192

printf("About to insert compressed records for date-range [%llu] - [%llu]\n",start_date,end_date);

2193

fflush(stdout);

2194

1924

2195

// Step #3 insert new daily stats into database for specified date range

1925

2196

int iInsertedRows = importLogfileDataIntoSqlite(szDatabase, tBandwidthUsage, tBandwidthUsageInitValue);

1926

2197

1927

2198

fprintf(stderr, "Total old rows deleted: [%d]\n", iDeleteRows);

1928

2199

fprintf(stderr, "Total new rows added: [%d]\n", iInsertedRows);

2200

fflush(stderr);

1929

2201

1930

2202

return iInsertedRows;

1931

2203

}

1954

2226

printf(": \t 3 = console [based on sqlite db], 4 = html [based on sqlite db].\n");

1955

2227

printf(": \t 5 = console [based on sqlite db] and compresses intra-day record data into one record per day.\n");

1956

2228

printf(": \t 6 = Use this option to auto-discover static DHCP clients when running in the Tomato firmware.\n");

2229

printf(": \t 7 = console [based on sqlite db] shrink database by removing empty space.\n");

1957

2230

printf(": <start-date-filter>\t= the optional start date range of logfiles to parse:\n");

1958

2231

printf(": \t example: 20090425 and default is current date. You may also use the keyword today like this: today\n");

1959

2232

printf(": <end-date-filter> \t= the optional end date range of logfiles to parse:\n");

1960

2233

printf(": \t example: 20090426 and default is current date. You may also use the keyword today like this: today-30\n");

1961

printf(": \t *NOTE: for <generate-type> 3 and 4 you may specifc a date and time value using this format: 20090425 19:45\n");

2234

printf(": \t *NOTE: for <generate-type> 3 and 4 you may specify a date and time value using this format: 20090425 19:45\n");

1962

2235

printf(": <logfiles-path> \t= the optional path to search for the logfiles (default value is ./).\n");

1963

2236

printf(": <incoming-data-logfileprefix>\t= the optional log filename prefix used to describe incoming data logs (example: traffic_in_).\n");

1964

2237

printf(": <outgoing-data-logfileprefix>\t= the optional log filename prefix used to describe outgoing data logs (example: traffic_out_).\n");

1965

printf(": <flags=x> \t= the optional value containg special values depending on the <generate-type>, possible values:\n");

2238

printf(": <flags=x> \t= the optional value containing special values depending on the <generate-type>, possible values:\n");

1966

2239

printf(": \t For <generate-type> = 0 or 1 you may import logfile data into sqlite using:\n");

1967

2240

printf(": \t flags=import=BANDWIDTH\n");

1968

2241

printf(": \t where BANDWIDTH = the sqlite database filename to import into (the table must be named bandwidthusage)\n");

1972

2245

printf(": \t %s (will display ONLY the Hostname without references to the IP Address)\n",APP_FLAG_HOSTNAME_ONLY);

1973

2246

printf(": \t For <generate-type> = 2 you may want to keep your database smaller and import statistic summarized by date:\n");

1974

2247

printf(": \t %s (this will allow you to frequently update your database but only store 1 record per Client per day)\n",APP_FLAG_IMPORT_DAILY_ONLY);

2248

printf(": \t %s (this flag will delete record older then x days, example: %s=60 would delete records older than 60 days)\n",APP_FLAG_PURGE_OLD_RECORDS,APP_FLAG_PURGE_OLD_RECORDS);

1975

2249

printf("\n: \t Special notes: There are some assumed limitations in this tool namely:\n");

1976

2250

printf(": \t Maximum supported Clients (IP's) = %d, Maximum supported DateRange length = %d days\n",MAX_CLIENTCOUNT,MAX_RANGE);

1977

2251

printf(": \t These limits are hard coded but may be easily changed and a new binary produced.\n");

2006

2280

printf("calling mainInsertLatestStats...\n");

2007

2281

iResult = mainInsertLatestStats(argc, argv);

2008

2282

2283

char *pDB = argv[2];

2284

2009

2285

int bImportStatsDailyOnly = IsParameterEnabled(APP_FLAG_IMPORT_DAILY_ONLY, argc, argv);

2010

2286

if(bImportStatsDailyOnly == 1)

2011

2287

{

2015

2291

memset(&tBandwidthUsageInitValue[0],0,sizeof(data_t) * MAX_CLIENTCOUNT);

2016

2292

2017

2293

sprintf(szLogFileStartFilter,"today");

2018

ProcessSpecialDateParameter(szLogFileStartFilter);

2294

ProcessSpecialDateParameter(szLogFileStartFilter,0);

2019

2295

sprintf(szLogFileEndFilter,"today");

2020

ProcessSpecialDateParameter(szLogFileEndFilter);

2021

2022

char *pDB = argv[2];

2023

iResult = compressDataIntoDailyStats(szLogFileStartFilter,szLogFileEndFilter,pDB,

2024

tBandwidthUsage, tBandwidthUsageInitValue, GENERATE_TYPE_SQLITE_DATACOMPRESS);

2025

2026

}

2296

ProcessSpecialDateParameter(szLogFileEndFilter,0);

2297

2298

//printf("calling compressDataIntoDailyStats with date range [%s] - [%s]\n",szLogFileStartFilter,szLogFileEndFilter);

2299

//fflush(stdout);

2300

2301

//iResult = compressDataIntoDailyStats(szLogFileStartFilter,szLogFileEndFilter,pDB,

2302

// tBandwidthUsage, tBandwidthUsageInitValue, GENERATE_TYPE_SQLITE_DATACOMPRESS);

2303

}

2304

2305

int iPurgeDBIndex = GetParameterIndex(APP_FLAG_PURGE_OLD_RECORDS, argc, argv);

2306

if(iPurgeDBIndex >= 0)

2307

{

2308

printf("detected PURGE flag\n");

2309

fflush(stdout);

2310

2311

char *szSpecialFlags = argv[iPurgeDBIndex];

2312

char szSpecialFlagsValue[1024]="";

2313

2314

strcpy(szSpecialFlagsValue,&szSpecialFlags[strlen(APP_FLAG_PURGE_OLD_RECORDS)]);

2315

if(strlen(szSpecialFlagsValue) > 0)

2316

{

2317

int iRecordsThisOldAndOlder = atoi(szSpecialFlagsValue);

2318

if(iRecordsThisOldAndOlder > 0)

2319

{

2320

sprintf(szLogFileStartFilter,"1970-01-01");

2321

2322

sprintf(szLogFileEndFilter,"today-%d",iRecordsThisOldAndOlder);

2323

ProcessSpecialDateParameter(szLogFileEndFilter,1);

2324

2325

uint64_t delete_start_date = getDateValue(szLogFileStartFilter);

2326

uint64_t delete_end_date = getDateValue(szLogFileEndFilter);

2327

2328

printf("calling deleteStatsFromSqlite to PURGE old records with date range [%llu] - [%llu]\n",delete_start_date,delete_end_date);

2329

fflush(stdout);

2330

2331

// Step #2 delete old records for specified date range

2332

int iDeleteRows = deleteStatsFromSqlite(pDB, delete_start_date, delete_end_date);

2333

2334

fprintf(stderr, "Total old rows deleted: [%d]\n", iDeleteRows);

2335

fflush(stderr);

2336

}

2337

}

2338

}

2339

2027

2340

#endif

2028

2341

return iResult;

2029

2342

}

2030

else if(iGenerateType == GENERATE_TYPE_TESTLOG_PARSE ||

2031

iGenerateType == GENERATE_TYPE_LOGFILE_CONSOLE ||

2032

iGenerateType == GENERATE_TYPE_SQLITE_CONSOLE ||

2033

iGenerateType == GENERATE_TYPE_SQLITE_DATACOMPRESS)

2343

else if(iGenerateType == GENERATE_TYPE_TESTLOG_PARSE ||

2344

iGenerateType == GENERATE_TYPE_LOGFILE_CONSOLE ||

2345

iGenerateType == GENERATE_TYPE_SQLITE_CONSOLE ||

2346

iGenerateType == GENERATE_TYPE_SQLITE_DATACOMPRESS ||

2347

iGenerateType == GENERATE_TYPE_SHRINK_DATABASE)

2034

2348

{

2035

2349

2036

2350

}

2050

2364

}

2051

2365

2052

2366

sprintf(szLogFileStartFilter,"%s",TODAYS_DATE_KEYWORD);

2053

ProcessSpecialDateParameter(szLogFileStartFilter);

2367

ProcessSpecialDateParameter(szLogFileStartFilter,0);

2054

2368

2055

2369

sprintf(szLogFileEndFilter,"%s",TODAYS_DATE_KEYWORD);

2056

ProcessSpecialDateParameter(szLogFileEndFilter);

2370

ProcessSpecialDateParameter(szLogFileEndFilter,0);

2371

}

2372

else if(iGenerateType == GENERATE_TYPE_SHRINK_DATABASE)

2373

{

2374

char szDatabase[1024]="./";

2375

if (argc >= 3)

2376

{

2377

sprintf(szDatabase,"%s",argv[2]);

2378

}

2379

2380

int iPurgeDBIndex = GetParameterIndex(APP_FLAG_PURGE_OLD_RECORDS, argc, argv);

2381

if(iPurgeDBIndex >= 0)

2382

{

2383

printf("detected PURGE flag\n");

2384

fflush(stdout);

2385

2386

char *szSpecialFlags = argv[iPurgeDBIndex];

2387

char szSpecialFlagsValue[1024]="";

2388

2389

strcpy(szSpecialFlagsValue,&szSpecialFlags[strlen(APP_FLAG_PURGE_OLD_RECORDS)]);

2390

if(strlen(szSpecialFlagsValue) > 0)

2391

{

2392

int iRecordsThisOldAndOlder = atoi(szSpecialFlagsValue);

2393

if(iRecordsThisOldAndOlder > 0)

2394

{

2395

sprintf(szLogFileStartFilter,"1970-01-01");

2396

2397

sprintf(szLogFileEndFilter,"today-%d",iRecordsThisOldAndOlder);

2398

ProcessSpecialDateParameter(szLogFileEndFilter,1);

2399

2400

uint64_t delete_start_date = getDateValue(szLogFileStartFilter);

2401

uint64_t delete_end_date = getDateValue(szLogFileEndFilter);

2402

2403

printf("calling deleteStatsFromSqlite to PURGE old records with date range [%llu] - [%llu]\n",delete_start_date,delete_end_date);

2404

fflush(stdout);

2405

2406

// Step #2 delete old records for specified date range

2407

int iDeleteRows = deleteStatsFromSqlite(szDatabase, delete_start_date, delete_end_date);

2408

2409

fprintf(stderr, "Total old rows deleted: [%d]\n", iDeleteRows);

2410

fflush(stderr);

2411

}

2412

}

2413

}

2414

2415

vacuumDB(szDatabase);

2416

2417

return 0;

2057

2418

}

2058

2419

else

2059

2420

{

2061

2422

{

2062

2423

sprintf(szLogFileStartFilter,"%s",argv[2]);

2063

2424

}

2064

ProcessSpecialDateParameter(szLogFileStartFilter);

2425

ProcessSpecialDateParameter(szLogFileStartFilter,0);

2065

2426

2066

2427

2067

2428

if (argc >= 4)

2068

2429

{

2069

2430

sprintf(szLogFileEndFilter,"%s",argv[3]);

2070

2431

}

2071

ProcessSpecialDateParameter(szLogFileEndFilter);

2432

ProcessSpecialDateParameter(szLogFileEndFilter,0);

2072

2433

2073

2434

if(iGenerateType == GENERATE_TYPE_SQLITE_DATACOMPRESS)

2074

2435

{