« back to all changes in this revision
Viewing changes to src/CacheDisk.cpp
- browse files at revision 545
- compare with another revision
- download diff
- download tarball
- view history from revision 545
- Committer: Jonathan Thomas
- Date: 2016-09-07 05:40:01 UTC
- Revision ID: jonathan@openshot.org-20160907054001-v2tbe8uy8a7lk4qw
Added new CacheDisk class, which caches frames to the hard drive, dramatically speeding up preview speeds, at the expense of IO operations. New unittests for caching framework. Fixed a few bugs with Frame constructor, which was causing invalid # width & height. Integrated JSON into the cache framework, to quickly share the state of the cache (including ranges of cached frame numbers). Fixed a bug where some Timeline frames could have no audio samples.
- files added:
- include/CacheDisk.h
- files modified:
- include/CacheBase.h
added
removed
src/CacheDisk.cpp
1
/**
2
* @file
3
* @brief Source file for CacheDisk class
4
* @author Jonathan Thomas <jonathan@openshot.org>
5
*
6
* @section LICENSE
7
*
8
* Copyright (c) 2008-2014 OpenShot Studios, LLC
9
* <http://www.openshotstudios.com/>. This file is part of
10
* OpenShot Library (libopenshot), an open-source project dedicated to
11
* delivering high quality video editing and animation solutions to the
12
* world. For more information visit <http://www.openshot.org/>.
13
*
14
* OpenShot Library (libopenshot) is free software: you can redistribute it
15
* and/or modify it under the terms of the GNU Lesser General Public License
16
* as published by the Free Software Foundation, either version 3 of the
17
* License, or (at your option) any later version.
18
*
19
* OpenShot Library (libopenshot) is distributed in the hope that it will be
20
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22
* GNU Lesser General Public License for more details.
23
*
24
* You should have received a copy of the GNU Lesser General Public License
25
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26
*/
27
28
#include "../include/CacheDisk.h"
29
30
using namespace std;
31
using namespace openshot;
32
33
// Default constructor, no max bytes
34
CacheDisk::CacheDisk(string cache_path, string format, float quality, float scale) : CacheBase(0) {
35
// Set cache type name
36
cache_type = "CacheDisk";
37
range_version = 0;
38
needs_range_processing = false;
39
frame_size_bytes = 0;
40
image_format = format;
41
image_quality = quality;
42
image_scale = scale;
43
44
// Init path directory
45
InitPath(cache_path);
46
};
47
48
// Constructor that sets the max bytes to cache
49
CacheDisk::CacheDisk(string cache_path, string format, float quality, float scale, long long int max_bytes) : CacheBase(max_bytes) {
50
// Set cache type name
51
cache_type = "CacheDisk";
52
range_version = 0;
53
needs_range_processing = false;
54
frame_size_bytes = 0;
55
image_format = format;
56
image_quality = quality;
57
image_scale = scale;
58
59
// Init path directory
60
InitPath(cache_path);
61
};
62
63
// Initialize cache directory
64
void CacheDisk::InitPath(string cache_path) {
65
QString qpath;
66
67
if (!cache_path.empty()) {
68
// Init QDir with cache directory
69
qpath = QString(cache_path.c_str());
70
71
} else {
72
// Init QDir with user's temp directory
73
qpath = QDir::tempPath() + QString("/preview-cache/");
74
}
75
76
// Init QDir with cache directory
77
path = QDir(qpath);
78
79
// Check if cache directory exists
80
if (!path.exists())
81
// Create
82
path.mkpath(qpath);
83
}
84
85
// Calculate ranges of frames
86
void CacheDisk::CalculateRanges() {
87
// Only calculate when something has changed
88
if (needs_range_processing) {
89
90
// Create a scoped lock, to protect the cache from multiple threads
91
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
92
93
// Sort ordered frame #s, and calculate JSON ranges
94
std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
95
96
// Clear existing JSON variable
97
ranges.clear();
98
ranges = Json::Value(Json::arrayValue);
99
100
// Increment range version
101
range_version++;
102
103
vector<long int>::iterator itr_ordered;
104
long int starting_frame = *ordered_frame_numbers.begin();
105
long int ending_frame = *ordered_frame_numbers.begin();
106
107
// Loop through all known frames (in sequential order)
108
for (itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end(); ++itr_ordered) {
109
long int frame_number = *itr_ordered;
110
if (frame_number - ending_frame > 1) {
111
// End of range detected
112
Json::Value range;
113
114
// Add JSON object with start/end attributes
115
// Use strings, since long ints are supported in JSON
116
stringstream start_str;
117
start_str << starting_frame;
118
stringstream end_str;
119
end_str << ending_frame;
120
range["start"] = start_str.str();
121
range["end"] = end_str.str();
122
ranges.append(range);
123
124
// Set new starting range
125
starting_frame = frame_number;
126
}
127
128
// Set current frame as end of range, and keep looping
129
ending_frame = frame_number;
130
}
131
132
// APPEND FINAL VALUE
133
Json::Value range;
134
135
// Add JSON object with start/end attributes
136
// Use strings, since long ints are supported in JSON
137
stringstream start_str;
138
start_str << starting_frame;
139
stringstream end_str;
140
end_str << ending_frame;
141
range["start"] = start_str.str();
142
range["end"] = end_str.str();
143
ranges.append(range);
144
145
// Reset needs_range_processing
146
needs_range_processing = false;
147
}
148
}
149
150
// Default destructor
151
CacheDisk::~CacheDisk()
152
{
153
frames.clear();
154
frame_numbers.clear();
155
ordered_frame_numbers.clear();
156
157
// remove critical section
158
delete cacheCriticalSection;
159
cacheCriticalSection = NULL;
160
}
161
162
// Add a Frame to the cache
163
void CacheDisk::Add(tr1::shared_ptr<Frame> frame)
164
{
165
// Create a scoped lock, to protect the cache from multiple threads
166
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
167
long int frame_number = frame->number;
168
169
// Freshen frame if it already exists
170
if (frames.count(frame_number))
171
// Move frame to front of queue
172
MoveToFront(frame_number);
173
174
else
175
{
176
// Add frame to queue and map
177
frames[frame_number] = frame_number;
178
frame_numbers.push_front(frame_number);
179
ordered_frame_numbers.push_back(frame_number);
180
needs_range_processing = true;
181
182
// Save image to disk (if needed)
183
QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
184
frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
185
if (frame_size_bytes == 0) {
186
// Get compressed size of frame image (to correctly apply max size against)
187
QFile image_file(frame_path);
188
frame_size_bytes = image_file.size();
189
}
190
191
// Save audio data (if needed)
192
if (frame->has_audio_data) {
193
QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
194
QFile audio_file(audio_path);
195
196
if (audio_file.open(QIODevice::WriteOnly)) {
197
QTextStream audio_stream(&audio_file);
198
audio_stream << frame->SampleRate() << endl;
199
audio_stream << frame->GetAudioChannelsCount() << endl;
200
audio_stream << frame->GetAudioSamplesCount() << endl;
201
audio_stream << frame->ChannelsLayout() << endl;
202
203
// Loop through all samples
204
for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
205
{
206
// Get audio for this channel
207
float *samples = frame->GetAudioSamples(channel);
208
for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
209
audio_stream << samples[sample] << endl;
210
}
211
212
}
213
214
}
215
216
// Clean up old frames
217
CleanUp();
218
}
219
}
220
221
// Get a frame from the cache (or NULL shared_ptr if no frame is found)
222
tr1::shared_ptr<Frame> CacheDisk::GetFrame(long int frame_number)
223
{
224
// Create a scoped lock, to protect the cache from multiple threads
225
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
226
227
// Does frame exists in cache?
228
if (frames.count(frame_number)) {
229
// Does frame exist on disk
230
QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
231
if (path.exists(frame_path)) {
232
233
// Load image file
234
tr1::shared_ptr<QImage> image = tr1::shared_ptr<QImage>(new QImage());
235
bool success = image->load(QString::fromStdString(frame_path.toStdString()));
236
237
// Set pixel formatimage->
238
image = tr1::shared_ptr<QImage>(new QImage(image->convertToFormat(QImage::Format_RGBA8888)));
239
240
// Create frame object
241
tr1::shared_ptr<Frame> frame(new Frame());
242
frame->number = frame_number;
243
frame->AddImage(image);
244
245
// Get audio data (if found)
246
QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
247
QFile audio_file(audio_path);
248
if (audio_file.exists()) {
249
// Open audio file
250
QTextStream in(&audio_file);
251
if (audio_file.open(QIODevice::ReadOnly)) {
252
int sample_rate = in.readLine().toInt();
253
int channels = in.readLine().toInt();
254
int sample_count = in.readLine().toInt();
255
int channel_layout = in.readLine().toInt();
256
257
// Set basic audio properties
258
frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
259
260
// Loop through audio samples and add to frame
261
int current_channel = 0;
262
int current_sample = 0;
263
float *channel_samples = new float[sample_count];
264
while (!in.atEnd()) {
265
// Add sample to channel array
266
channel_samples[current_sample] = in.readLine().toFloat();
267
current_sample++;
268
269
if (current_sample == sample_count) {
270
// Add audio to frame
271
frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
272
273
// Increment channel, and reset sample position
274
current_channel++;
275
current_sample = 0;
276
}
277
278
}
279
}
280
}
281
282
// return the Frame object
283
return frame;
284
}
285
}
286
287
// no Frame found
288
return tr1::shared_ptr<Frame>();
289
}
290
291
// Get the smallest frame number (or NULL shared_ptr if no frame is found)
292
tr1::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
293
{
294
// Create a scoped lock, to protect the cache from multiple threads
295
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
296
tr1::shared_ptr<openshot::Frame> f;
297
298
// Loop through frame numbers
299
deque<long int>::iterator itr;
300
long int smallest_frame = -1;
301
for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
302
{
303
if (*itr < smallest_frame || smallest_frame == -1)
304
smallest_frame = *itr;
305
}
306
307
// Return frame
308
f = GetFrame(smallest_frame);
309
310
return f;
311
}
312
313
// Gets the maximum bytes value
314
long long int CacheDisk::GetBytes()
315
{
316
// Create a scoped lock, to protect the cache from multiple threads
317
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
318
319
long long int total_bytes = 0;
320
321
// Loop through frames, and calculate total bytes
322
deque<long int>::reverse_iterator itr;
323
for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
324
total_bytes += frame_size_bytes;
325
326
return total_bytes;
327
}
328
329
// Remove a specific frame
330
void CacheDisk::Remove(long int frame_number)
331
{
332
Remove(frame_number, frame_number);
333
}
334
335
// Remove range of frames
336
void CacheDisk::Remove(long int start_frame_number, long int end_frame_number)
337
{
338
// Create a scoped lock, to protect the cache from multiple threads
339
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
340
341
// Loop through frame numbers
342
deque<long int>::iterator itr = frame_numbers.begin();
343
while (itr != frame_numbers.end())
344
{
345
//deque<long int>::iterator current = itr++;
346
if (*itr >= start_frame_number && *itr <= end_frame_number)
347
{
348
// erase frame number
349
itr = frame_numbers.erase(itr++);
350
} else
351
++itr;
352
}
353
354
// Loop through ordered frame numbers
355
vector<long int>::iterator itr_ordered = ordered_frame_numbers.begin();
356
while (itr_ordered != ordered_frame_numbers.end())
357
{
358
if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
359
{
360
// erase frame number
361
frames.erase(*itr_ordered);
362
363
// Remove the image file (if it exists)
364
QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
365
QFile image_file(frame_path);
366
if (image_file.exists())
367
image_file.remove();
368
369
// Remove audio file (if it exists)
370
QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
371
QFile audio_file(audio_path);
372
if (audio_file.exists())
373
audio_file.remove();
374
375
itr_ordered = ordered_frame_numbers.erase(itr_ordered++);
376
} else
377
++itr_ordered;
378
}
379
380
// Needs range processing (since cache has changed)
381
needs_range_processing = true;
382
}
383
384
// Move frame to front of queue (so it lasts longer)
385
void CacheDisk::MoveToFront(long int frame_number)
386
{
387
// Create a scoped lock, to protect the cache from multiple threads
388
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
389
390
// Does frame exists in cache?
391
/* FIXME if the frame number isn't present, the loop will do nothing, so why protect it?
392
* Is it to save time by avoiding a loop?
393
* Do we really need to optmize the case where we've been given a nonexisting frame_number? */
394
if (frames.count(frame_number))
395
{
396
// Loop through frame numbers
397
deque<long int>::iterator itr;
398
for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
399
{
400
if (*itr == frame_number)
401
{
402
// erase frame number
403
frame_numbers.erase(itr);
404
405
// add frame number to 'front' of queue
406
frame_numbers.push_front(frame_number);
407
break;
408
}
409
}
410
}
411
}
412
413
// Clear the cache of all frames
414
void CacheDisk::Clear()
415
{
416
// Create a scoped lock, to protect the cache from multiple threads
417
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
418
419
// Clear all containers
420
frames.clear();
421
frame_numbers.clear();
422
ordered_frame_numbers.clear();
423
needs_range_processing = true;
424
frame_size_bytes = 0;
425
426
// Delete cache directory, and recreate it
427
QString current_path = path.path();
428
path.removeRecursively();
429
430
// Re-init folder
431
InitPath(current_path.toStdString());
432
}
433
434
// Count the frames in the queue
435
long int CacheDisk::Count()
436
{
437
// Create a scoped lock, to protect the cache from multiple threads
438
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
439
440
// Return the number of frames in the cache
441
return frames.size();
442
}
443
444
// Clean up cached frames that exceed the number in our max_bytes variable
445
void CacheDisk::CleanUp()
446
{
447
// Create a scoped lock, to protect the cache from multiple threads
448
const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
449
450
// Do we auto clean up?
451
if (max_bytes > 0)
452
{
453
while (GetBytes() > max_bytes && frame_numbers.size() > 20)
454
{
455
// Get the oldest frame number.
456
long int frame_to_remove = frame_numbers.back();
457
458
// Remove frame_number and frame
459
Remove(frame_to_remove);
460
}
461
}
462
}
463
464
// Generate JSON string of this object
465
string CacheDisk::Json() {
466
467
// Return formatted string
468
return JsonValue().toStyledString();
469
}
470
471
// Generate Json::JsonValue for this object
472
Json::Value CacheDisk::JsonValue() {
473
474
// Proccess range data (if anything has changed)
475
CalculateRanges();
476
477
// Create root json object
478
Json::Value root = CacheBase::JsonValue(); // get parent properties
479
root["type"] = cache_type;
480
root["path"] = path.path().toStdString();
481
root["ranges"] = ranges;
482
483
Json::Value version;
484
stringstream range_version_str;
485
range_version_str << range_version;
486
root["version"] = range_version_str.str();
487
488
// return JsonValue
489
return root;
490
}
491
492
// Load JSON string into this object
493
void CacheDisk::SetJson(string value) throw(InvalidJSON) {
494
495
// Parse JSON string into JSON objects
496
Json::Value root;
497
Json::Reader reader;
498
bool success = reader.parse( value, root );
499
if (!success)
500
// Raise exception
501
throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
502
503
try
504
{
505
// Set all values that match
506
SetJsonValue(root);
507
}
508
catch (exception e)
509
{
510
// Error parsing JSON (or missing keys)
511
throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
512
}
513
}
514
515
// Load Json::JsonValue into this object
516
void CacheDisk::SetJsonValue(Json::Value root) throw(InvalidFile, ReaderClosed) {
517
518
// Close timeline before we do anything (this also removes all open and closing clips)
519
Clear();
520
521
// Set parent data
522
CacheBase::SetJsonValue(root);
523
524
if (!root["type"].isNull())
525
cache_type = root["type"].asString();
526
if (!root["path"].isNull())
527
// Update duration of timeline
528
InitPath(root["path"].asString());
529
}
b'\\ No newline at end of file'
Loggerhead is a web-based interface for Breezy
Version: 2.0.1