« back to all changes in this revision
Viewing changes to storage/myisam/myisampack.c
- browse files at revision 77.3.26
- compare with another revision
- download diff
- download tarball
- view history from revision 77.3.26
- Committer: Monty Taylor
- Date: 2008-07-16 19:10:24 UTC
- mfrom: (51.1.127 remove-dbug)
- mto: This revision was merged to the branch mainline in revision 176.
- Revision ID: monty@inaugust.com-20080716191024-prjgoh7fbri7rx26
Merged from remove-dbug.
- files removed:
- storage/myisam/make-ccc
- files modified:
- storage/myisam/Makefile.am
added
removed
storage/myisam/myisampack.c
170
170
static int mrg_close(PACK_MRG_INFO *mrg);
171
171
static int mrg_rrnd(PACK_MRG_INFO *info,uchar *buf);
172
172
static void mrg_reset(PACK_MRG_INFO *mrg);
173
#if !defined(DBUG_OFF)
174
static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count);
175
static int fakecmp(my_off_t **count1, my_off_t **count2);
176
#endif
177
178
173
179
174
static int error_on_write=0,test_only=0,verbose=0,silent=0,
180
175
write_loop=0,force_pack=0, isamchk_neaded=0;
351
346
verbose++; /* Allow for selecting the level of verbosity. */
352
347
silent= 0;
353
348
break;
354
case '#':
355
DBUG_PUSH(argument ? argument : "d:t:o");
356
break;
357
349
case 'V':
358
350
print_version();
359
351
exit(0);
397
389
{
398
390
MI_INFO *isam_file;
399
391
MYISAM_SHARE *share;
400
DBUG_ENTER("open_isam_file");
401
392
402
393
if (!(isam_file=mi_open(name,mode,
403
394
(opt_wait ? HA_OPEN_WAIT_IF_LOCKED :
404
395
HA_OPEN_ABORT_IF_LOCKED))))
405
396
{
406
397
VOID(fprintf(stderr, "%s gave error %d on open\n", name, my_errno));
407
DBUG_RETURN(0);
398
return(0);
408
399
}
409
400
share=isam_file->s;
410
401
if (share->options & HA_OPTION_COMPRESS_RECORD && !join_table)
413
404
{
414
405
VOID(fprintf(stderr, "%s is already compressed\n", name));
415
406
VOID(mi_close(isam_file));
416
DBUG_RETURN(0);
407
return(0);
417
408
}
418
409
if (verbose)
419
410
puts("Recompressing already compressed table");
425
416
{
426
417
VOID(fprintf(stderr, "%s is too small to compress\n", name));
427
418
VOID(mi_close(isam_file));
428
DBUG_RETURN(0);
419
return(0);
429
420
}
430
421
VOID(mi_lock_database(isam_file,F_WRLCK));
431
DBUG_RETURN(isam_file);
422
return(isam_file);
432
423
}
433
424
434
425
490
481
my_off_t old_length,new_length,tot_elements;
491
482
HUFF_COUNTS *huff_counts;
492
483
HUFF_TREE *huff_trees;
493
DBUG_ENTER("compress");
494
484
495
485
isam_file=mrg->file[0]; /* Take this as an example */
496
486
share=isam_file->s;
542
532
for (i=0 ; i < mrg->count ; i++)
543
533
mrg->records+=mrg->file[i]->s->state.state.records;
544
534
545
DBUG_PRINT("info", ("Compressing %s: (%lu records)",
546
result_table ? new_name : org_name,
547
(ulong) mrg->records));
548
535
if (write_loop || verbose)
549
536
{
550
537
VOID(printf("Compressing %s: (%lu records)\n",
557
544
/*
558
545
Read the whole data file(s) for statistics.
559
546
*/
560
DBUG_PRINT("info", ("- Calculating statistics"));
561
547
if (write_loop || verbose)
562
548
VOID(printf("- Calculating statistics\n"));
563
549
if (get_statistic(mrg,huff_counts))
633
619
/*
634
620
Compress the source file into the target file.
635
621
*/
636
DBUG_PRINT("info", ("- Compressing file"));
637
622
if (write_loop || verbose)
638
623
VOID(printf("- Compressing file\n"));
639
624
error=compress_isam_file(mrg,huff_counts);
657
642
end_file_buffer();
658
643
659
644
/* Display statistics. */
660
DBUG_PRINT("info", ("Min record length: %6d Max length: %6d "
661
"Mean total length: %6ld\n",
662
mrg->min_pack_length, mrg->max_pack_length,
663
(ulong) (mrg->records ? (new_length/mrg->records) : 0)));
664
645
if (verbose && mrg->records)
665
646
VOID(printf("Min record length: %6d Max length: %6d "
666
647
"Mean total length: %6ld\n", mrg->min_pack_length,
729
710
{
730
711
VOID(fprintf(stderr, "Aborting: %s is not compressed\n", org_name));
731
712
VOID(my_delete(new_name,MYF(MY_WME)));
732
DBUG_RETURN(-1);
713
return(-1);
733
714
}
734
715
if (write_loop || verbose)
735
716
{
740
721
else
741
722
puts("Empty file saved in compressed format");
742
723
}
743
DBUG_RETURN(0);
724
return(0);
744
725
745
726
err:
746
727
free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields);
750
731
VOID(my_close(join_isam_file,MYF(0)));
751
732
mrg_close(mrg);
752
733
VOID(fprintf(stderr, "Aborted: %s is not compressed\n", org_name));
753
DBUG_RETURN(-1);
734
return(-1);
754
735
}
755
736
756
737
/* Init a huff_count-struct for each field and init it */
841
822
my_bool static_row_size;
842
823
HUFF_COUNTS *count,*end_count;
843
824
TREE_ELEMENT *element;
844
DBUG_ENTER("get_statistic");
845
825
846
826
reclength=mrg->file[0]->s->base.reclength;
847
827
record=(uchar*) my_alloca(reclength);
1083
1063
VOID(fflush(stdout));
1084
1064
}
1085
1065
1086
/*
1087
If --debug=d,fakebigcodes is set, fake the counts to get big Huffman
1088
codes.
1089
*/
1090
DBUG_EXECUTE_IF("fakebigcodes", fakebigcodes(huff_counts, end_count););
1091
1092
DBUG_PRINT("info", ("Found the following number of incidents "
1093
"of the byte codes:"));
1094
1066
if (verbose >= 2)
1095
1067
VOID(printf("Found the following number of incidents "
1096
1068
"of the byte codes:\n"));
1100
1072
my_off_t total_count;
1101
1073
char llbuf[32];
1102
1074
1103
DBUG_PRINT("info", ("column: %3u", (uint) (count - huff_counts + 1)));
1104
1075
if (verbose >= 2)
1105
1076
VOID(printf("column: %3u\n", (uint) (count - huff_counts + 1)));
1106
1077
if (count->tree_buff)
1107
1078
{
1108
DBUG_PRINT("info", ("number of distinct values: %u",
1109
(uint) ((count->tree_pos - count->tree_buff) /
1110
count->field_length)));
1111
1079
if (verbose >= 2)
1112
1080
VOID(printf("number of distinct values: %u\n",
1113
1081
(uint) ((count->tree_pos - count->tree_buff) /
1119
1087
if (count->counts[idx])
1120
1088
{
1121
1089
total_count+= count->counts[idx];
1122
DBUG_PRINT("info", ("counts[0x%02x]: %12s", idx,
1123
llstr((int64_t) count->counts[idx], llbuf)));
1124
1090
if (verbose >= 2)
1125
1091
VOID(printf("counts[0x%02x]: %12s\n", idx,
1126
1092
llstr((int64_t) count->counts[idx], llbuf)));
1127
1093
}
1128
1094
}
1129
DBUG_PRINT("info", ("total: %12s", llstr((int64_t) total_count,
1130
llbuf)));
1131
1095
if ((verbose >= 2) && total_count)
1132
1096
{
1133
1097
VOID(printf("total: %12s\n",
1138
1102
mrg->records=record_count;
1139
1103
mrg->max_blob_length=max_blob_length;
1140
1104
my_afree((uchar*) record);
1141
DBUG_RETURN(error != HA_ERR_END_OF_FILE);
1105
return(error != HA_ERR_END_OF_FILE);
1142
1106
}
1143
1107
1144
1108
static int compare_huff_elements(void *not_used __attribute__((unused)),
1156
1120
{
1157
1121
uint space_fields,fill_zero_fields,field_count[(int) FIELD_enum_val_count];
1158
1122
my_off_t old_length,new_length,length;
1159
DBUG_ENTER("check_counts");
1160
1123
1161
1124
bzero((uchar*) field_count,sizeof(field_count));
1162
1125
space_fields=fill_zero_fields=0;
1310
1273
{
1311
1274
HUFF_TREE tree;
1312
1275
1313
DBUG_EXECUTE_IF("forceintervall",
1314
huff_counts->bytes_packed= ~ (my_off_t) 0;);
1315
1276
tree.element_buffer=0;
1316
1277
if (!make_huff_tree(&tree,huff_counts) &&
1317
1278
tree.bytes_packed+tree.tree_pack_length < huff_counts->bytes_packed)
1337
1298
fill_zero_fields++;
1338
1299
field_count[huff_counts->field_type]++;
1339
1300
}
1340
DBUG_PRINT("info", ("normal: %3d empty-space: %3d "
1341
"empty-zero: %3d empty-fill: %3d",
1342
field_count[FIELD_NORMAL],space_fields,
1343
field_count[FIELD_SKIP_ZERO],fill_zero_fields));
1344
DBUG_PRINT("info", ("pre-space: %3d end-space: %3d "
1345
"intervall-fields: %3d zero: %3d",
1346
field_count[FIELD_SKIP_PRESPACE],
1347
field_count[FIELD_SKIP_ENDSPACE],
1348
field_count[FIELD_INTERVALL],
1349
field_count[FIELD_ZERO]));
1350
1301
if (verbose)
1351
1302
VOID(printf("\nnormal: %3d empty-space: %3d "
1352
1303
"empty-zero: %3d empty-fill: %3d\n"
1358
1309
field_count[FIELD_SKIP_ENDSPACE],
1359
1310
field_count[FIELD_INTERVALL],
1360
1311
field_count[FIELD_ZERO]));
1361
DBUG_VOID_RETURN;
1312
return;
1362
1313
}
1363
1314
1364
1315
/* Test if we can use space-compression and empty-field-compression */
1435
1386
{
1436
1387
uint tree;
1437
1388
HUFF_TREE *huff_tree;
1438
DBUG_ENTER("make_huff_trees");
1439
1389
1440
1390
if (!(huff_tree=(HUFF_TREE*) my_malloc(trees*sizeof(HUFF_TREE),
1441
1391
MYF(MY_WME | MY_ZEROFILL))))
1442
DBUG_RETURN(0);
1392
return(0);
1443
1393
1444
1394
for (tree=0 ; tree < trees ; tree++)
1445
1395
{
1448
1398
while (tree--)
1449
1399
my_free((uchar*) huff_tree[tree].element_buffer,MYF(0));
1450
1400
my_free((uchar*) huff_tree,MYF(0));
1451
DBUG_RETURN(0);
1401
return(0);
1452
1402
}
1453
1403
}
1454
DBUG_RETURN(huff_tree);
1404
return(huff_tree);
1455
1405
}
1456
1406
1457
1407
/*
1728
1678
uint i,found,bits_packed,first,last;
1729
1679
my_off_t bytes_packed;
1730
1680
HUFF_ELEMENT element_buffer[256];
1731
DBUG_ENTER("calc_packed_length");
1732
1681
1733
1682
/*
1734
1683
WARNING: We use a small hack for efficiency: Instead of placing
1760
1709
}
1761
1710
}
1762
1711
if (!found)
1763
DBUG_RETURN(0); /* Empty tree */
1712
return(0); /* Empty tree */
1764
1713
/*
1765
1714
If there is only a single byte value in this field in all records,
1766
1715
add a second element with zero incidence. This is required to enter
1825
1774
queue.root[1]=(uchar*) new_huff_el;
1826
1775
queue_replaced(&queue);
1827
1776
}
1828
DBUG_RETURN(bytes_packed+(bits_packed+7)/8);
1777
return(bytes_packed+(bits_packed+7)/8);
1829
1778
}
1830
1779
1831
1780
1869
1818
}
1870
1819
}
1871
1820
}
1872
DBUG_PRINT("info", ("Original trees: %d After join: %d",
1873
trees, tree_number));
1874
1821
if (verbose)
1875
1822
VOID(printf("Original trees: %d After join: %d\n", trees, tree_number));
1876
1823
return tree_number; /* Return trees left */
1958
1905
char *ptr= digits;
1959
1906
uint idx= bits;
1960
1907
1961
DBUG_ASSERT(idx < sizeof(digits));
1908
assert(idx < sizeof(digits));
1962
1909
while (idx)
1963
1910
*(ptr++)= '0' + ((char) (value >> (--idx)) & (char) 1);
1964
1911
*ptr= '\0';
1987
1934
char *ptr= digits;
1988
1935
uint idx= 2 * sizeof(value); /* Two hex digits per byte. */
1989
1936
1990
DBUG_ASSERT(idx < sizeof(digits));
1937
assert(idx < sizeof(digits));
1991
1938
while (idx)
1992
1939
{
1993
1940
if ((*(ptr++)= '0' + ((char) (value >> (4 * (--idx))) & (char) 0xf)) > '9')
2031
1978
uint huff_tree_bits;
2032
1979
huff_tree_bits=max_bit(trees ? trees-1 : 0);
2033
1980
2034
DBUG_PRINT("info", (" "));
2035
DBUG_PRINT("info", ("column types:"));
2036
DBUG_PRINT("info", ("FIELD_NORMAL 0"));
2037
DBUG_PRINT("info", ("FIELD_SKIP_ENDSPACE 1"));
2038
DBUG_PRINT("info", ("FIELD_SKIP_PRESPACE 2"));
2039
DBUG_PRINT("info", ("FIELD_SKIP_ZERO 3"));
2040
DBUG_PRINT("info", ("FIELD_BLOB 4"));
2041
DBUG_PRINT("info", ("FIELD_CONSTANT 5"));
2042
DBUG_PRINT("info", ("FIELD_INTERVALL 6"));
2043
DBUG_PRINT("info", ("FIELD_ZERO 7"));
2044
DBUG_PRINT("info", ("FIELD_VARCHAR 8"));
2045
DBUG_PRINT("info", ("FIELD_CHECK 9"));
2046
DBUG_PRINT("info", (" "));
2047
DBUG_PRINT("info", ("pack type as a set of flags:"));
2048
DBUG_PRINT("info", ("PACK_TYPE_SELECTED 1"));
2049
DBUG_PRINT("info", ("PACK_TYPE_SPACE_FIELDS 2"));
2050
DBUG_PRINT("info", ("PACK_TYPE_ZERO_FILL 4"));
2051
DBUG_PRINT("info", (" "));
2052
1981
if (verbose >= 2)
2053
1982
{
2054
1983
VOID(printf("\n"));
2079
2008
else
2080
2009
write_bits(counts->length_bits,5);
2081
2010
write_bits((uint64_t) counts->tree->tree_number - 1, huff_tree_bits);
2082
DBUG_PRINT("info", ("column: %3u type: %2u pack: %2u zero: %4u "
2083
"lbits: %2u tree: %2u length: %4u",
2084
i , counts->field_type, counts->pack_type,
2085
counts->max_zero_fill, counts->length_bits,
2086
counts->tree->tree_number, counts->field_length));
2087
2011
if (verbose >= 2)
2088
2012
VOID(printf("column: %3u type: %2u pack: %2u zero: %4u lbits: %2u "
2089
2013
"tree: %2u length: %4u\n", i , counts->field_type,
2121
2045
return 0;
2122
2046
}
2123
2047
2124
DBUG_PRINT("info", (" "));
2125
2048
if (verbose >= 2)
2126
2049
VOID(printf("\n"));
2127
2050
tree_no= 0;
2132
2055
if (huff_tree->tree_number == 0)
2133
2056
continue; /* Deleted tree */
2134
2057
tree_no++;
2135
DBUG_PRINT("info", (" "));
2136
2058
if (verbose >= 3)
2137
2059
VOID(printf("\n"));
2138
2060
/* Count the total number of elements (byte codes or column values). */
2159
2081
return 0;
2160
2082
}
2161
2083
2162
DBUG_PRINT("info", ("pos: %lu elements: %u tree-elements: %lu "
2163
"char_bits: %u\n",
2164
(ulong) (file_buffer.pos - file_buffer.buffer),
2165
huff_tree->elements, (ulong) (offset - packed_tree),
2166
huff_tree->char_bits));
2167
2084
if (!huff_tree->counts->tree_buff)
2168
2085
{
2169
2086
/* We do a byte compression on this column. Mark with bit 0. */
2186
2103
write_bits(huff_tree->offset_bits,5);
2187
2104
intervall_length+=int_length;
2188
2105
}
2189
DBUG_PRINT("info", ("tree: %2u elements: %4u char_bits: %2u "
2190
"offset_bits: %2u %s: %5u codelen: %2u",
2191
tree_no, huff_tree->elements, huff_tree->char_bits,
2192
huff_tree->offset_bits, huff_tree->counts->tree_buff ?
2193
"bufflen" : "min_chr", huff_tree->counts->tree_buff ?
2194
int_length : huff_tree->min_chr, huff_tree->height));
2195
2106
if (verbose >= 2)
2196
2107
VOID(printf("tree: %2u elements: %4u char_bits: %2u offset_bits: %2u "
2197
2108
"%s: %5u codelen: %2u\n", tree_no, huff_tree->elements,
2217
2128
huff_tree->offset_bits+1);
2218
2129
else
2219
2130
write_bits(packed_tree[i]-huff_tree->min_chr,huff_tree->char_bits+1);
2220
DBUG_PRINT("info", ("tree[0x%04x]: %s0x%04x",
2221
i, (packed_tree[i] & IS_OFFSET) ?
2222
" -> " : "", (packed_tree[i] & IS_OFFSET) ?
2223
packed_tree[i] - IS_OFFSET + i : packed_tree[i]));
2224
2131
if (verbose >= 3)
2225
2132
VOID(printf("tree[0x%04x]: %s0x%04x\n",
2226
2133
i, (packed_tree[i] & IS_OFFSET) ? " -> " : "",
2242
2149
2243
2150
if (! (len= huff_tree->code_len[i]))
2244
2151
continue;
2245
DBUG_PRINT("info", ("code[0x%04x]: 0x%s bits: %2u bin: %s", i,
2246
hexdigits(huff_tree->code[i]), huff_tree->code_len[i],
2247
bindigits(huff_tree->code[i],
2248
huff_tree->code_len[i])));
2249
2152
if (verbose >= 3)
2250
2153
VOID(printf("code[0x%04x]: 0x%s bits: %2u bin: %s\n", i,
2251
2154
hexdigits(huff_tree->code[i]), huff_tree->code_len[i],
2255
2158
code= 0;
2256
2159
bits= 0;
2257
2160
idx= 0;
2258
DBUG_EXECUTE_IF("forcechkerr1", len--;);
2259
DBUG_EXECUTE_IF("forcechkerr2", bits= 8 * sizeof(code););
2260
DBUG_EXECUTE_IF("forcechkerr3", idx= length;);
2261
2161
for (;;)
2262
2162
{
2263
2163
if (! len)
2296
2196
if (errors)
2297
2197
break;
2298
2198
2299
DBUG_EXECUTE_IF("forcechkerr4", packed_tree[idx]++;);
2300
2199
if (packed_tree[idx] != i)
2301
2200
{
2302
2201
VOID(fflush(stdout));
2315
2214
for (i=0 ; i < int_length ; i++)
2316
2215
{
2317
2216
write_bits((uint64_t) (uchar) huff_tree->counts->tree_buff[i], 8);
2318
DBUG_PRINT("info", ("column_values[0x%04x]: 0x%02x",
2319
i, (uchar) huff_tree->counts->tree_buff[i]));
2320
2217
if (verbose >= 3)
2321
2218
VOID(printf("column_values[0x%04x]: 0x%02x\n",
2322
2219
i, (uchar) huff_tree->counts->tree_buff[i]));
2324
2221
}
2325
2222
flush_bits();
2326
2223
}
2327
DBUG_PRINT("info", (" "));
2328
2224
if (verbose >= 2)
2329
2225
VOID(printf("\n"));
2330
2226
my_afree((uchar*) packed_tree);
2413
2309
HUFF_TREE *tree;
2414
2310
MI_INFO *isam_file=mrg->file[0];
2415
2311
uint pack_version= (uint) isam_file->s->pack.version;
2416
DBUG_ENTER("compress_isam_file");
2417
2312
2418
2313
/* Allocate a buffer for the records (excluding blobs). */
2419
2314
if (!(record=(uchar*) my_alloca(isam_file->s->base.reclength)))
2458
2353
calc_pack_length(pack_version, mrg->max_blob_length) : 0;
2459
2354
max_pack_length=pack_ref_length+pack_blob_length;
2460
2355
2461
DBUG_PRINT("fields", ("==="));
2462
2356
mrg_reset(mrg);
2463
2357
while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE)
2464
2358
{
2474
2368
end_pos=start_pos+(field_length=count->field_length);
2475
2369
tree=count->tree;
2476
2370
2477
DBUG_PRINT("fields", ("column: %3lu type: %2u pack: %2u zero: %4u "
2478
"lbits: %2u tree: %2u length: %4u",
2479
(ulong) (count - huff_counts + 1),
2480
count->field_type,
2481
count->pack_type, count->max_zero_fill,
2482
count->length_bits, count->tree->tree_number,
2483
count->field_length));
2484
2485
2371
/* Check if the column contains spaces only. */
2486
2372
if (count->pack_type & PACK_TYPE_SPACE_FIELDS)
2487
2373
{
2488
2374
for (pos=start_pos ; *pos == ' ' && pos < end_pos; pos++) ;
2489
2375
if (pos == end_pos)
2490
2376
{
2491
DBUG_PRINT("fields",
2492
("PACK_TYPE_SPACE_FIELDS spaces only, bits: 1"));
2493
DBUG_PRINT("fields", ("---"));
2494
2377
write_bits(1,1);
2495
2378
start_pos=end_pos;
2496
2379
continue;
2497
2380
}
2498
DBUG_PRINT("fields",
2499
("PACK_TYPE_SPACE_FIELDS not only spaces, bits: 1"));
2500
2381
write_bits(0,1);
2501
2382
}
2502
2383
end_pos-=count->max_zero_fill;
2506
2387
case FIELD_SKIP_ZERO:
2507
2388
if (!memcmp((uchar*) start_pos,zero_string,field_length))
2508
2389
{
2509
DBUG_PRINT("fields", ("FIELD_SKIP_ZERO zeroes only, bits: 1"));
2510
2390
write_bits(1,1);
2511
2391
start_pos=end_pos;
2512
2392
break;
2513
2393
}
2514
DBUG_PRINT("fields", ("FIELD_SKIP_ZERO not only zeroes, bits: 1"));
2515
2394
write_bits(0,1);
2516
2395
/* Fall through */
2517
2396
case FIELD_NORMAL:
2518
DBUG_PRINT("fields", ("FIELD_NORMAL %lu bytes",
2519
(ulong) (end_pos - start_pos)));
2520
2397
for ( ; start_pos < end_pos ; start_pos++)
2521
2398
{
2522
DBUG_PRINT("fields",
2523
("value: 0x%02x code: 0x%s bits: %2u bin: %s",
2524
(uchar) *start_pos,
2525
hexdigits(tree->code[(uchar) *start_pos]),
2526
(uint) tree->code_len[(uchar) *start_pos],
2527
bindigits(tree->code[(uchar) *start_pos],
2528
(uint) tree->code_len[(uchar) *start_pos])));
2529
2399
write_bits(tree->code[(uchar) *start_pos],
2530
2400
(uint) tree->code_len[(uchar) *start_pos]);
2531
2401
}
2537
2407
{
2538
2408
if (length > count->min_space)
2539
2409
{
2540
DBUG_PRINT("fields",
2541
("FIELD_SKIP_ENDSPACE more than min_space, bits: 1"));
2542
DBUG_PRINT("fields",
2543
("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u",
2544
length, field_length, count->length_bits));
2545
2410
write_bits(1,1);
2546
2411
write_bits(length,count->length_bits);
2547
2412
}
2548
2413
else
2549
2414
{
2550
DBUG_PRINT("fields",
2551
("FIELD_SKIP_ENDSPACE not more than min_space, "
2552
"bits: 1"));
2553
2415
write_bits(0,1);
2554
2416
pos=end_pos;
2555
2417
}
2556
2418
}
2557
2419
else
2558
2420
{
2559
DBUG_PRINT("fields",
2560
("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u",
2561
length, field_length, count->length_bits));
2562
2421
write_bits(length,count->length_bits);
2563
2422
}
2564
2423
/* Encode all significant bytes. */
2565
DBUG_PRINT("fields", ("FIELD_SKIP_ENDSPACE %lu bytes",
2566
(ulong) (pos - start_pos)));
2567
2424
for ( ; start_pos < pos ; start_pos++)
2568
2425
{
2569
DBUG_PRINT("fields",
2570
("value: 0x%02x code: 0x%s bits: %2u bin: %s",
2571
(uchar) *start_pos,
2572
hexdigits(tree->code[(uchar) *start_pos]),
2573
(uint) tree->code_len[(uchar) *start_pos],
2574
bindigits(tree->code[(uchar) *start_pos],
2575
(uint) tree->code_len[(uchar) *start_pos])));
2576
2426
write_bits(tree->code[(uchar) *start_pos],
2577
2427
(uint) tree->code_len[(uchar) *start_pos]);
2578
2428
}
2585
2435
{
2586
2436
if (length > count->min_space)
2587
2437
{
2588
DBUG_PRINT("fields",
2589
("FIELD_SKIP_PRESPACE more than min_space, bits: 1"));
2590
DBUG_PRINT("fields",
2591
("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u",
2592
length, field_length, count->length_bits));
2593
2438
write_bits(1,1);
2594
2439
write_bits(length,count->length_bits);
2595
2440
}
2596
2441
else
2597
2442
{
2598
DBUG_PRINT("fields",
2599
("FIELD_SKIP_PRESPACE not more than min_space, "
2600
"bits: 1"));
2601
2443
pos=start_pos;
2602
2444
write_bits(0,1);
2603
2445
}
2604
2446
}
2605
2447
else
2606
2448
{
2607
DBUG_PRINT("fields",
2608
("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u",
2609
length, field_length, count->length_bits));
2610
2449
write_bits(length,count->length_bits);
2611
2450
}
2612
2451
/* Encode all significant bytes. */
2613
DBUG_PRINT("fields", ("FIELD_SKIP_PRESPACE %lu bytes",
2614
(ulong) (end_pos - start_pos)));
2615
2452
for (start_pos=pos ; start_pos < end_pos ; start_pos++)
2616
2453
{
2617
DBUG_PRINT("fields",
2618
("value: 0x%02x code: 0x%s bits: %2u bin: %s",
2619
(uchar) *start_pos,
2620
hexdigits(tree->code[(uchar) *start_pos]),
2621
(uint) tree->code_len[(uchar) *start_pos],
2622
bindigits(tree->code[(uchar) *start_pos],
2623
(uint) tree->code_len[(uchar) *start_pos])));
2624
2454
write_bits(tree->code[(uchar) *start_pos],
2625
2455
(uint) tree->code_len[(uchar) *start_pos]);
2626
2456
}
2628
2458
case FIELD_CONSTANT:
2629
2459
case FIELD_ZERO:
2630
2460
case FIELD_CHECK:
2631
DBUG_PRINT("fields", ("FIELD_CONSTANT/ZERO/CHECK"));
2632
2461
start_pos=end_pos;
2633
2462
break;
2634
2463
case FIELD_INTERVALL:
2636
2465
pos=(uchar*) tree_search(&count->int_tree, start_pos,
2637
2466
count->int_tree.custom_arg);
2638
2467
intervall=(uint) (pos - count->tree_buff)/field_length;
2639
DBUG_PRINT("fields", ("FIELD_INTERVALL"));
2640
DBUG_PRINT("fields", ("index: %4u code: 0x%s bits: %2u",
2641
intervall, hexdigits(tree->code[intervall]),
2642
(uint) tree->code_len[intervall]));
2643
2468
write_bits(tree->code[intervall],(uint) tree->code_len[intervall]);
2644
2469
start_pos=end_pos;
2645
2470
break;
2651
2476
/* Empty blobs are encoded with a single 1 bit. */
2652
2477
if (!blob_length)
2653
2478
{
2654
DBUG_PRINT("fields", ("FIELD_BLOB empty, bits: 1"));
2655
2479
write_bits(1,1);
2656
2480
}
2657
2481
else
2658
2482
{
2659
2483
uchar *blob,*blob_end;
2660
DBUG_PRINT("fields", ("FIELD_BLOB not empty, bits: 1"));
2661
2484
write_bits(0,1);
2662
2485
/* Write the blob length. */
2663
DBUG_PRINT("fields", ("FIELD_BLOB %lu bytes, bits: %2u",
2664
blob_length, count->length_bits));
2665
2486
write_bits(blob_length,count->length_bits);
2666
2487
memcpy_fixed(&blob,end_pos-portable_sizeof_char_ptr,
2667
2488
sizeof(char*));
2669
2490
/* Encode the blob bytes. */
2670
2491
for ( ; blob < blob_end ; blob++)
2671
2492
{
2672
DBUG_PRINT("fields",
2673
("value: 0x%02x code: 0x%s bits: %2u bin: %s",
2674
(uchar) *blob, hexdigits(tree->code[(uchar) *blob]),
2675
(uint) tree->code_len[(uchar) *blob],
2676
bindigits(tree->code[(uchar) *start_pos],
2677
(uint)tree->code_len[(uchar) *start_pos])));
2678
2493
write_bits(tree->code[(uchar) *blob],
2679
2494
(uint) tree->code_len[(uchar) *blob]);
2680
2495
}
2692
2507
/* Empty varchar are encoded with a single 1 bit. */
2693
2508
if (!col_length)
2694
2509
{
2695
DBUG_PRINT("fields", ("FIELD_VARCHAR empty, bits: 1"));
2696
2510
write_bits(1,1); /* Empty varchar */
2697
2511
}
2698
2512
else
2699
2513
{
2700
2514
uchar *end= start_pos + var_pack_length + col_length;
2701
DBUG_PRINT("fields", ("FIELD_VARCHAR not empty, bits: 1"));
2702
2515
write_bits(0,1);
2703
2516
/* Write the varchar length. */
2704
DBUG_PRINT("fields", ("FIELD_VARCHAR %lu bytes, bits: %2u",
2705
col_length, count->length_bits));
2706
2517
write_bits(col_length,count->length_bits);
2707
2518
/* Encode the varchar bytes. */
2708
2519
for (start_pos+= var_pack_length ; start_pos < end ; start_pos++)
2709
2520
{
2710
DBUG_PRINT("fields",
2711
("value: 0x%02x code: 0x%s bits: %2u bin: %s",
2712
(uchar) *start_pos,
2713
hexdigits(tree->code[(uchar) *start_pos]),
2714
(uint) tree->code_len[(uchar) *start_pos],
2715
bindigits(tree->code[(uchar) *start_pos],
2716
(uint)tree->code_len[(uchar) *start_pos])));
2717
2521
write_bits(tree->code[(uchar) *start_pos],
2718
2522
(uint) tree->code_len[(uchar) *start_pos]);
2719
2523
}
2726
2530
abort(); /* Impossible */
2727
2531
}
2728
2532
start_pos+=count->max_zero_fill;
2729
DBUG_PRINT("fields", ("---"));
2730
2533
}
2731
2534
flush_bits();
2732
2535
length=(ulong) ((uchar*) file_buffer.pos - record_pos) - max_pack_length;
2734
2537
if (pack_blob_length)
2735
2538
pack_length+= save_pack_length(pack_version, record_pos + pack_length,
2736
2539
tot_blob_length);
2737
DBUG_PRINT("fields", ("record: %lu length: %lu blob-length: %lu "
2738
"length-bytes: %lu", (ulong) record_count, length,
2739
tot_blob_length, pack_length));
2740
DBUG_PRINT("fields", ("==="));
2741
2742
2540
/* Correct file buffer if the header was smaller */
2743
2541
if (pack_length != max_pack_length)
2744
2542
{
2773
2571
mrg->ref_length=max_pack_length;
2774
2572
mrg->min_pack_length=max_record_length ? min_record_length : 0;
2775
2573
mrg->max_pack_length=max_record_length;
2776
DBUG_RETURN(error || error_on_write || flush_buffer(~(ulong) 0));
2574
return(error || error_on_write || flush_buffer(~(ulong) 0));
2777
2575
}
2778
2576
2779
2577
2870
2668
2871
2669
static void write_bits(register uint64_t value, register uint bits)
2872
2670
{
2873
DBUG_ASSERT(((bits < 8 * sizeof(value)) && ! (value >> bits)) ||
2671
assert(((bits < 8 * sizeof(value)) && ! (value >> bits)) ||
2874
2672
(bits == 8 * sizeof(value)));
2875
2673
2876
2674
if ((file_buffer.bits-= (int) bits) >= 0)
2936
2734
MYISAM_SHARE *share=isam_file->s;
2937
2735
uint options=mi_uint2korr(share->state.header.options);
2938
2736
uint key;
2939
DBUG_ENTER("save_state");
2940
2737
2941
2738
options|= HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA;
2942
2739
mi_int2store(share->state.header.options,options);
2973
2770
(void)ftruncate(share->kfile, share->base.keystart);
2974
2771
if (share->base.keys)
2975
2772
isamchk_neaded=1;
2976
DBUG_RETURN(mi_state_info_write(share->kfile,&share->state,1+2));
2773
return(mi_state_info_write(share->kfile,&share->state,1+2));
2977
2774
}
2978
2775
2979
2776
2983
2780
MI_STATE_INFO state;
2984
2781
MI_INFO *isam_file=mrg->file[0];
2985
2782
uint options;
2986
DBUG_ENTER("save_state_mrg");
2987
2783
2988
2784
state= isam_file->s->state;
2989
2785
options= (mi_uint2korr(state.header.options) | HA_OPTION_COMPRESS_RECORD |
3006
2802
if (isam_file->s->base.keys)
3007
2803
isamchk_neaded=1;
3008
2804
state.changed=STATE_CHANGED | STATE_NOT_ANALYZED; /* Force check of table */
3009
DBUG_RETURN (mi_state_info_write(file,&state,1+2));
2805
return (mi_state_info_write(file,&state,1+2));
3010
2806
}
3011
2807
3012
2808
3070
2866
my_free((uchar*) mrg->file,MYF(0));
3071
2867
return error;
3072
2868
}
3073
3074
3075
#if !defined(DBUG_OFF)
3076
/*
3077
Fake the counts to get big Huffman codes.
3078
3079
SYNOPSIS
3080
fakebigcodes()
3081
huff_counts A pointer to the counts array.
3082
end_count A pointer past the counts array.
3083
3084
DESCRIPTION
3085
3086
Huffman coding works by removing the two least frequent values from
3087
the list of values and add a new value with the sum of their
3088
incidences in a loop until only one value is left. Every time a
3089
value is reused for a new value, it gets one more bit for its
3090
encoding. Hence, the least frequent values get the longest codes.
3091
3092
To get a maximum code length for a value, two of the values must
3093
have an incidence of 1. As their sum is 2, the next infrequent value
3094
must have at least an incidence of 2, then 4, 8, 16 and so on. This
3095
means that one needs 2**n bytes (values) for a code length of n
3096
bits. However, using more distinct values forces the use of longer
3097
codes, or reaching the code length with less total bytes (values).
3098
3099
To get 64(32)-bit codes, I sort the counts by decreasing incidence.
3100
I assign counts of 1 to the two most frequent values, a count of 2
3101
for the next one, then 4, 8, and so on until 2**64-1(2**30-1). All
3102
the remaining values get 1. That way every possible byte has an
3103
assigned code, though not all codes are used if not all byte values
3104
are present in the column.
3105
3106
This strategy would work with distinct column values too, but
3107
requires that at least 64(32) values are present. To make things
3108
easier here, I cancel all distinct column values and force byte
3109
compression for all columns.
3110
3111
RETURN
3112
void
3113
*/
3114
3115
static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count)
3116
{
3117
HUFF_COUNTS *count;
3118
my_off_t *cur_count_p;
3119
my_off_t *end_count_p;
3120
my_off_t **cur_sort_p;
3121
my_off_t **end_sort_p;
3122
my_off_t *sort_counts[256];
3123
my_off_t total;
3124
DBUG_ENTER("fakebigcodes");
3125
3126
for (count= huff_counts; count < end_count; count++)
3127
{
3128
/*
3129
Remove distinct column values.
3130
*/
3131
if (huff_counts->tree_buff)
3132
{
3133
my_free((uchar*) huff_counts->tree_buff, MYF(0));
3134
delete_tree(&huff_counts->int_tree);
3135
huff_counts->tree_buff= NULL;
3136
DBUG_PRINT("fakebigcodes", ("freed distinct column values"));
3137
}
3138
3139
/*
3140
Sort counts by decreasing incidence.
3141
*/
3142
cur_count_p= count->counts;
3143
end_count_p= cur_count_p + 256;
3144
cur_sort_p= sort_counts;
3145
while (cur_count_p < end_count_p)
3146
*(cur_sort_p++)= cur_count_p++;
3147
(void) my_qsort(sort_counts, 256, sizeof(my_off_t*), (qsort_cmp) fakecmp);
3148
3149
/*
3150
Assign faked counts.
3151
*/
3152
cur_sort_p= sort_counts;
3153
#if SIZEOF_LONG_LONG > 4
3154
end_sort_p= sort_counts + 8 * sizeof(uint64_t) - 1;
3155
#else
3156
end_sort_p= sort_counts + 8 * sizeof(uint64_t) - 2;
3157
#endif
3158
/* Most frequent value gets a faked count of 1. */
3159
**(cur_sort_p++)= 1;
3160
total= 1;
3161
while (cur_sort_p < end_sort_p)
3162
{
3163
**(cur_sort_p++)= total;
3164
total<<= 1;
3165
}
3166
/* Set the last value. */
3167
**(cur_sort_p++)= --total;
3168
/*
3169
Set the remaining counts.
3170
*/
3171
end_sort_p= sort_counts + 256;
3172
while (cur_sort_p < end_sort_p)
3173
**(cur_sort_p++)= 1;
3174
}
3175
DBUG_VOID_RETURN;
3176
}
3177
3178
3179
/*
3180
Compare two counts for reverse sorting.
3181
3182
SYNOPSIS
3183
fakecmp()
3184
count1 One count.
3185
count2 Another count.
3186
3187
RETURN
3188
1 count1 < count2
3189
0 count1 == count2
3190
-1 count1 > count2
3191
*/
3192
3193
static int fakecmp(my_off_t **count1, my_off_t **count2)
3194
{
3195
return ((**count1 < **count2) ? 1 :
3196
(**count1 > **count2) ? -1 : 0);
3197
}
3198
#endif
Loggerhead is a web-based interface for Breezy
Version: 2.0.1