2
* Copyright 2009 The Apache Software Foundation
4
* Licensed to the Apache Software Foundation (ASF) under one
5
* or more contributor license agreements. See the NOTICE file
6
* distributed with this work for additional information
7
* regarding copyright ownership. The ASF licenses this file
8
* to you under the Apache License, Version 2.0 (the
9
* "License"); you may not use this file except in compliance
10
* with the License. You may obtain a copy of the License at
12
* http://www.apache.org/licenses/LICENSE-2.0
14
* Unless required by applicable law or agreed to in writing, software
15
* distributed under the License is distributed on an "AS IS" BASIS,
16
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
* See the License for the specific language governing permissions and
18
* limitations under the License.
20
package org.apache.hadoop.hbase.regionserver;
22
import java.io.FileNotFoundException;
23
import java.io.IOException;
24
import java.lang.management.ManagementFactory;
25
import java.lang.management.MemoryUsage;
27
import java.util.Random;
28
import java.util.concurrent.atomic.AtomicBoolean;
29
import java.util.regex.Matcher;
30
import java.util.regex.Pattern;
32
import org.apache.commons.logging.Log;
33
import org.apache.commons.logging.LogFactory;
34
import org.apache.hadoop.fs.FileSystem;
35
import org.apache.hadoop.fs.Path;
36
import org.apache.hadoop.hbase.HBaseConfiguration;
37
import org.apache.hadoop.hbase.HConstants;
38
import org.apache.hadoop.hbase.KeyValue;
39
import org.apache.hadoop.hbase.io.HalfHFileReader;
40
import org.apache.hadoop.hbase.io.Reference;
41
import org.apache.hadoop.hbase.io.hfile.BlockCache;
42
import org.apache.hadoop.hbase.io.hfile.Compression;
43
import org.apache.hadoop.hbase.io.hfile.HFile;
44
import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
45
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
46
import org.apache.hadoop.hbase.util.Bytes;
47
import org.apache.hadoop.util.StringUtils;
50
* A Store data file. Stores usually have one or more of these files. They
51
* are produced by flushing the memstore to disk. To
52
* create, call {@link #getWriter(FileSystem, Path)} and append data. Be
53
* sure to add any metadata before calling close on the Writer
54
* (Use the appendMetadata convenience methods). On close, a StoreFile is
55
* sitting in the Filesystem. To refer to it, create a StoreFile instance
56
* passing filesystem and path. To read, call {@link #getReader()}.
57
* <p>StoreFiles may also reference store files in another Store.
59
public class StoreFile implements HConstants {
60
static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
62
private static final String HFILE_CACHE_SIZE_KEY = "hfile.block.cache.size";
64
private static BlockCache hfileBlockCache = null;
66
// Make default block size for StoreFiles 8k while testing. TODO: FIX!
67
// Need to make it 8k for testing.
68
private static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
70
private final FileSystem fs;
72
private final Path path;
73
// If this storefile references another, this is the reference instance.
74
private Reference reference;
75
// If this StoreFile references another, this is the other files path.
76
private Path referencePath;
77
// Should the block cache be used or not.
78
private boolean blockcache;
79
// Is this from an in-memory store
80
private boolean inMemory;
82
// Keys for metadata stored in backing HFile.
83
private static final byte [] MAX_SEQ_ID_KEY = Bytes.toBytes("MAX_SEQ_ID_KEY");
84
// Set when we obtain a Reader.
85
private long sequenceid = -1;
87
private static final byte [] MAJOR_COMPACTION_KEY =
88
Bytes.toBytes("MAJOR_COMPACTION_KEY");
89
// If true, this file was product of a major compaction. Its then set
90
// whenever you get a Reader.
91
private AtomicBoolean majorCompaction = null;
94
* Regex that will work for straight filenames and for reference names.
95
* If reference, then the regex has more than just one group. Group 1 is
96
* this files id. Group 2 the referenced region name, etc.
98
private static final Pattern REF_NAME_PARSER =
99
Pattern.compile("^(\\d+)(?:\\.(.+))?$");
101
private volatile HFile.Reader reader;
103
// Used making file ids.
104
private final static Random rand = new Random();
105
private final HBaseConfiguration conf;
108
* Constructor, loads a reader and it's indices, etc. May allocate a
109
* substantial amount of ram depending on the underlying files (10-20MB?).
111
* @param fs The current file system to use.
112
* @param p The path of the file.
113
* @param blockcache <code>true</code> if the block cache is enabled.
114
* @param conf The current configuration.
115
* @throws IOException When opening the reader fails.
117
StoreFile(final FileSystem fs, final Path p, final boolean blockcache,
118
final HBaseConfiguration conf, final boolean inMemory)
123
this.blockcache = blockcache;
124
this.inMemory = inMemory;
125
if (isReference(p)) {
126
this.reference = Reference.read(fs, p);
127
this.referencePath = getReferredToFile(this.path);
129
this.reader = open();
133
* @return Path or null if this StoreFile was made with a Stream.
140
* @return The Store/ColumnFamily this file belongs to.
142
byte [] getFamily() {
143
return Bytes.toBytes(this.path.getParent().getName());
147
* @return True if this is a StoreFile Reference; call after {@link #open()}
148
* else may get wrong answer.
150
boolean isReference() {
151
return this.reference != null;
155
* @param p Path to check.
156
* @return True if the path has format of a HStoreFile reference.
158
public static boolean isReference(final Path p) {
159
return isReference(p, REF_NAME_PARSER.matcher(p.getName()));
163
* @param p Path to check.
164
* @param m Matcher to use.
165
* @return True if the path has format of a HStoreFile reference.
167
public static boolean isReference(final Path p, final Matcher m) {
168
if (m == null || !m.matches()) {
169
LOG.warn("Failed match of store file name " + p.toString());
170
throw new RuntimeException("Failed match of store file name " +
173
return m.groupCount() > 1 && m.group(2) != null;
177
* Return path to the file referred to by a Reference. Presumes a directory
178
* hierarchy of <code>${hbase.rootdir}/tablename/regionname/familyname</code>.
179
* @param p Path to a Reference file.
180
* @return Calculated path to parent region file.
181
* @throws IOException
183
static Path getReferredToFile(final Path p) {
184
Matcher m = REF_NAME_PARSER.matcher(p.getName());
185
if (m == null || !m.matches()) {
186
LOG.warn("Failed match of store file name " + p.toString());
187
throw new RuntimeException("Failed match of store file name " +
190
// Other region name is suffix on the passed Reference file name
191
String otherRegion = m.group(2);
192
// Tabledir is up two directories from where Reference was written.
193
Path tableDir = p.getParent().getParent().getParent();
194
String nameStrippedOfSuffix = m.group(1);
195
// Build up new path with the referenced region in place of our current
196
// region in the reference path. Also strip regionname suffix from name.
197
return new Path(new Path(new Path(tableDir, otherRegion),
198
p.getParent().getName()), nameStrippedOfSuffix);
202
* @return True if this file was made by a major compaction.
204
boolean isMajorCompaction() {
205
if (this.majorCompaction == null) {
206
throw new NullPointerException("This has not been set yet");
208
return this.majorCompaction.get();
212
* @return This files maximum edit sequence id.
214
public long getMaxSequenceId() {
215
if (this.sequenceid == -1) {
216
throw new IllegalAccessError("Has not been initialized");
218
return this.sequenceid;
222
* Returns the block cache or <code>null</code> in case none should be used.
224
* @param conf The current configuration.
225
* @return The block cache or <code>null</code>.
227
public static synchronized BlockCache getBlockCache(HBaseConfiguration conf) {
228
if (hfileBlockCache != null) return hfileBlockCache;
230
float cachePercentage = conf.getFloat(HFILE_CACHE_SIZE_KEY, 0.0f);
231
// There should be a better way to optimize this. But oh well.
232
if (cachePercentage == 0L) return null;
233
if (cachePercentage > 1.0) {
234
throw new IllegalArgumentException(HFILE_CACHE_SIZE_KEY +
235
" must be between 0.0 and 1.0, not > 1.0");
238
// Calculate the amount of heap to give the heap.
239
MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
240
long cacheSize = (long)(mu.getMax() * cachePercentage);
241
LOG.info("Allocating LruBlockCache with maximum size " +
242
StringUtils.humanReadableInt(cacheSize));
243
hfileBlockCache = new LruBlockCache(cacheSize, DEFAULT_BLOCKSIZE_SMALL);
244
return hfileBlockCache;
248
* @return the blockcache
250
public BlockCache getBlockCache() {
251
return blockcache ? getBlockCache(conf) : null;
255
* Opens reader on this store file. Called by Constructor.
256
* @return Reader for the store file.
257
* @throws IOException
260
protected HFile.Reader open()
262
if (this.reader != null) {
263
throw new IllegalAccessError("Already open");
266
this.reader = new HalfHFileReader(this.fs, this.referencePath,
267
getBlockCache(), this.reference);
269
this.reader = new Reader(this.fs, this.path, getBlockCache(),
272
// Load up indices and fileinfo.
273
Map<byte [], byte []> map = this.reader.loadFileInfo();
274
// Read in our metadata.
275
byte [] b = map.get(MAX_SEQ_ID_KEY);
277
// By convention, if halfhfile, top half has a sequence number > bottom
278
// half. Thats why we add one in below. Its done for case the two halves
279
// are ever merged back together --rare. Without it, on open of store,
280
// since store files are distingushed by sequence id, the one half would
281
// subsume the other.
282
this.sequenceid = Bytes.toLong(b);
284
if (Reference.isTopFileRegion(this.reference.getFileRegion())) {
285
this.sequenceid += 1;
290
b = map.get(MAJOR_COMPACTION_KEY);
292
boolean mc = Bytes.toBoolean(b);
293
if (this.majorCompaction == null) {
294
this.majorCompaction = new AtomicBoolean(mc);
296
this.majorCompaction.set(mc);
300
// TODO read in bloom filter here, ignore if the column family config says
301
// "no bloom filter" even if there is one in the hfile.
306
* @return Current reader. Must call open first else returns null.
308
public HFile.Reader getReader() {
313
* @throws IOException
315
public synchronized void close() throws IOException {
316
if (this.reader != null) {
323
public String toString() {
324
return this.path.toString() +
325
(isReference()? "-" + this.referencePath + "-" + reference.toString(): "");
330
* @throws IOException
332
public void delete() throws IOException {
334
this.fs.delete(getPath(), true);
338
* Utility to help with rename.
342
* @return True if succeeded.
343
* @throws IOException
345
public static Path rename(final FileSystem fs, final Path src,
348
if (!fs.exists(src)) {
349
throw new FileNotFoundException(src.toString());
351
if (!fs.rename(src, tgt)) {
352
throw new IOException("Failed rename of " + src + " to " + tgt);
358
* Get a store file writer. Client is responsible for closing file when done.
359
* If metadata, add BEFORE closing using
360
* {@link #appendMetadata(org.apache.hadoop.hbase.io.hfile.HFile.Writer, long)}.
362
* @param dir Path to family directory. Makes the directory if doesn't exist.
363
* Creates a file with a unique name in this directory.
364
* @return HFile.Writer
365
* @throws IOException
367
public static HFile.Writer getWriter(final FileSystem fs, final Path dir)
369
return getWriter(fs, dir, DEFAULT_BLOCKSIZE_SMALL, null, null);
373
* Get a store file writer. Client is responsible for closing file when done.
374
* If metadata, add BEFORE closing using
375
* {@link #appendMetadata(org.apache.hadoop.hbase.io.hfile.HFile.Writer, long)}.
377
* @param dir Path to family directory. Makes the directory if doesn't exist.
378
* Creates a file with a unique name in this directory.
380
* @param algorithm Pass null to get default.
381
* @param c Pass null to get default.
382
* @return HFile.Writer
383
* @throws IOException
385
public static HFile.Writer getWriter(final FileSystem fs, final Path dir,
386
final int blocksize, final Compression.Algorithm algorithm,
387
final KeyValue.KeyComparator c)
389
if (!fs.exists(dir)) {
392
Path path = getUniqueFile(fs, dir);
393
return new HFile.Writer(fs, path, blocksize,
394
algorithm == null? HFile.DEFAULT_COMPRESSION_ALGORITHM: algorithm,
395
c == null? KeyValue.KEY_COMPARATOR: c);
400
* @param dir Directory to create file in.
401
* @return random filename inside passed <code>dir</code>
403
public static Path getUniqueFile(final FileSystem fs, final Path dir)
405
if (!fs.getFileStatus(dir).isDir()) {
406
throw new IOException("Expecting " + dir.toString() +
407
" to be a directory");
409
return fs.getFileStatus(dir).isDir()? getRandomFilename(fs, dir): dir;
416
* @return Path to a file that doesn't exist at time of this invocation.
417
* @throws IOException
419
static Path getRandomFilename(final FileSystem fs, final Path dir)
421
return getRandomFilename(fs, dir, null);
429
* @return Path to a file that doesn't exist at time of this invocation.
430
* @throws IOException
432
static Path getRandomFilename(final FileSystem fs, final Path dir,
438
id = Math.abs(rand.nextLong());
439
p = new Path(dir, Long.toString(id) +
440
((suffix == null || suffix.length() <= 0)? "": suffix));
441
} while(fs.exists(p));
446
* Write file metadata.
447
* Call before you call close on the passed <code>w</code> since its written
448
* as metadata to that file.
450
* @param w hfile writer
451
* @param maxSequenceId Maximum sequence id.
452
* @throws IOException
454
static void appendMetadata(final HFile.Writer w, final long maxSequenceId)
456
appendMetadata(w, maxSequenceId, false);
461
* Call before you call close on the passed <code>w</code> since its written
462
* as metadata to that file.
463
* @param maxSequenceId Maximum sequence id.
464
* @param mc True if this file is product of a major compaction
465
* @throws IOException
467
public static void appendMetadata(final HFile.Writer w, final long maxSequenceId,
470
w.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId));
471
w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(mc));
475
* Write out a split reference.
477
* @param splitDir Presumes path format is actually
478
* <code>SOME_DIRECTORY/REGIONNAME/FAMILY</code>.
479
* @param f File to split.
482
* @return Path to created reference.
483
* @throws IOException
485
static Path split(final FileSystem fs, final Path splitDir,
486
final StoreFile f, final byte [] splitRow, final Reference.Range range)
488
// A reference to the bottom half of the hsf store file.
489
Reference r = new Reference(splitRow, range);
490
// Add the referred-to regions name as a dot separated suffix.
491
// See REF_NAME_PARSER regex above. The referred-to regions name is
492
// up in the path of the passed in <code>f</code> -- parentdir is family,
493
// then the directory above is the region name.
494
String parentRegionName = f.getPath().getParent().getParent().getName();
495
// Write reference with same file id only with the other region name as
496
// suffix and into the new region location (under same family).
497
Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
498
return r.write(fs, p);
added
removed
src/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
