2
# Copyright (c) 2014-2015 The Bitcoin Core developers
2
# Copyright (c) 2014-2016 The Bitcoin Core developers
3
3
# Distributed under the MIT software license, see the accompanying
4
4
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
15
15
from test_framework.util import *
17
17
def calc_usage(blockdir):
18
return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f))/(1024*1024)
18
return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f)) / (1024. * 1024.)
20
20
class PruneTest(BitcoinTestFramework):
22
22
def __init__(self):
24
self.setup_clean_chain = True
24
28
self.address = ["",""]
25
29
self.txouts = gen_return_txouts()
27
def setup_chain(self):
28
print("Initializing test directory "+self.options.tmpdir)
29
initialize_chain_clean(self.options.tmpdir, 3)
31
31
def setup_network(self):
33
33
self.is_network_split = False
56
56
self.nodes[1].generate(200)
57
57
sync_blocks(self.nodes[0:2])
58
58
self.nodes[0].generate(150)
59
# Then mine enough full blocks to create more than 550MB of data
59
# Then mine enough full blocks to create more than 550MiB of data
61
61
self.mine_full_block(self.nodes[0], self.address[0])
63
63
sync_blocks(self.nodes[0:3])
65
65
def test_height_min(self):
66
66
if not os.path.isfile(self.prunedir+"blk00000.dat"):
67
67
raise AssertionError("blk00000.dat is missing, pruning too early")
69
print "Though we're already using more than 550MB, current usage:", calc_usage(self.prunedir)
70
print "Mining 25 more blocks should cause the first block file to be pruned"
69
print("Though we're already using more than 550MiB, current usage:", calc_usage(self.prunedir))
70
print("Mining 25 more blocks should cause the first block file to be pruned")
71
71
# Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
73
73
self.mine_full_block(self.nodes[0],self.address[0])
75
75
waitstart = time.time()
76
76
while os.path.isfile(self.prunedir+"blk00000.dat"):
78
if time.time() - waitstart > 10:
78
if time.time() - waitstart > 30:
79
79
raise AssertionError("blk00000.dat not pruned when it should be")
82
82
usage = calc_usage(self.prunedir)
83
print "Usage should be below target:", usage
83
print("Usage should be below target:", usage)
85
85
raise AssertionError("Pruning target not being met")
87
87
def create_chain_with_staleblocks(self):
88
88
# Create stale blocks in manageable sized chunks
89
print "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds"
89
print("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
92
92
# Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
93
93
# Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
94
94
# Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
96
96
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)
97
97
# Mine 24 blocks in node 1
98
98
self.utxo = self.nodes[1].listunspent()
101
101
self.mine_full_block(self.nodes[1],self.address[1])
105
105
# Reorg back with 25 block chain from node 0
106
106
self.utxo = self.nodes[0].listunspent()
108
108
self.mine_full_block(self.nodes[0],self.address[0])
110
110
# Create connections in the order so both nodes can see the reorg at the same time
112
112
connect_nodes(self.nodes[2], 0)
113
113
sync_blocks(self.nodes[0:3])
115
print "Usage can be over target because of high stale rate:", calc_usage(self.prunedir)
115
print("Usage can be over target because of high stale rate:", calc_usage(self.prunedir))
117
117
def reorg_test(self):
118
118
# Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
123
123
self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
125
125
height = self.nodes[1].getblockcount()
126
print "Current block height:", height
126
print("Current block height:", height)
128
128
invalidheight = height-287
129
129
badhash = self.nodes[1].getblockhash(invalidheight)
130
print "Invalidating block at height:",invalidheight,badhash
130
print("Invalidating block at height:",invalidheight,badhash)
131
131
self.nodes[1].invalidateblock(badhash)
133
133
# We've now switched to our previously mined-24 block fork on node 1, but thats not what we want
139
139
curhash = self.nodes[1].getblockhash(invalidheight - 1)
141
141
assert(self.nodes[1].getblockcount() == invalidheight - 1)
142
print "New best height", self.nodes[1].getblockcount()
142
print("New best height", self.nodes[1].getblockcount())
144
144
# Reboot node1 to clear those giant tx's from mempool
145
145
stop_node(self.nodes[1],1)
146
146
self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
148
print "Generating new longer chain of 300 more blocks"
148
print("Generating new longer chain of 300 more blocks")
149
149
self.nodes[1].generate(300)
151
print "Reconnect nodes"
151
print("Reconnect nodes")
152
152
connect_nodes(self.nodes[0], 1)
153
153
connect_nodes(self.nodes[2], 1)
154
sync_blocks(self.nodes[0:3])
156
print "Verify height on node 2:",self.nodes[2].getblockcount()
157
print "Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir)
159
print "Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)"
154
sync_blocks(self.nodes[0:3], timeout=120)
156
print("Verify height on node 2:",self.nodes[2].getblockcount())
157
print("Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir))
159
print("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
160
160
self.nodes[0].generate(220) #node 0 has many large tx's in its mempool from the disconnects
161
sync_blocks(self.nodes[0:3])
161
sync_blocks(self.nodes[0:3], timeout=300)
163
163
usage = calc_usage(self.prunedir)
164
print "Usage should be below target:", usage
164
print("Usage should be below target:", usage)
165
165
if (usage > 550):
166
166
raise AssertionError("Pruning target not being met")
173
173
self.nodes[2].getblock(self.forkhash)
174
174
raise AssertionError("Old block wasn't pruned so can't test redownload")
175
175
except JSONRPCException as e:
176
print "Will need to redownload block",self.forkheight
176
print("Will need to redownload block",self.forkheight)
178
178
# Verify that we have enough history to reorg back to the fork point
179
179
# Although this is more than 288 blocks, because this chain was written more recently
197
197
# At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
198
198
if self.nodes[2].getblockcount() < self.mainchainheight:
199
199
blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
200
print "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine
200
print("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine)
201
201
self.nodes[0].invalidateblock(curchainhash)
202
202
assert(self.nodes[0].getblockcount() == self.mainchainheight)
203
203
assert(self.nodes[0].getbestblockhash() == self.mainchainhash2)
204
204
goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
205
205
goalbestheight = first_reorg_height + 1
207
print "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload"
207
print("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
208
208
waitstart = time.time()
209
209
while self.nodes[2].getblockcount() < goalbestheight:
217
217
def mine_full_block(self, node, address):
218
218
# Want to create a full block
219
219
# We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit
221
221
if len(self.utxo) < 14:
222
222
self.utxo = node.listunspent()
243
243
def run_test(self):
244
print "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)"
245
print "Mining a big blockchain of 995 blocks"
244
print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
245
print("Mining a big blockchain of 995 blocks")
246
246
self.create_big_chain()
247
247
# Chain diagram key:
248
248
# * blocks on main chain
253
253
# Start by mining a simple chain that all nodes have
254
254
# N0=N1=N2 **...*(995)
256
print "Check that we haven't started pruning yet because we're below PruneAfterHeight"
256
print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
257
257
self.test_height_min()
258
258
# Extend this chain past the PruneAfterHeight
259
259
# N0=N1=N2 **...*(1020)
261
print "Check that we'll exceed disk space target if we have a very high stale block rate"
261
print("Check that we'll exceed disk space target if we have a very high stale block rate")
262
262
self.create_chain_with_staleblocks()
264
264
# And mine a 24 block chain on N1 and a separate 25 block chain on N0
282
282
self.mainchainheight = self.nodes[2].getblockcount() #1320
283
283
self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
285
print "Check that we can survive a 288 block reorg still"
285
print("Check that we can survive a 288 block reorg still")
286
286
(self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
287
287
# Now create a 288 block reorg by mining a longer chain on N1
288
288
# First disconnect N1
312
312
# ++...++(1044) ..
314
# N0 ********************(1032) @@...@@@(1552)
314
# N0 ********************(1032) @@...@@@(1552)
318
print "Test that we can rerequest a block we previously pruned if needed for a reorg"
318
print("Test that we can rerequest a block we previously pruned if needed for a reorg")
319
319
self.reorg_back()
320
320
# Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
321
321
# Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
added
removed
qa/rpc-tests/pruning.py
