1
// Protocol Buffers - Google's data interchange format
2
// Copyright 2008 Google Inc. All rights reserved.
3
// http://code.google.com/p/protobuf/
5
// Redistribution and use in source and binary forms, with or without
6
// modification, are permitted provided that the following conditions are
9
// * Redistributions of source code must retain the above copyright
10
// notice, this list of conditions and the following disclaimer.
11
// * Redistributions in binary form must reproduce the above
12
// copyright notice, this list of conditions and the following disclaimer
13
// in the documentation and/or other materials provided with the
15
// * Neither the name of Google Inc. nor the names of its
16
// contributors may be used to endorse or promote products derived from
17
// this software without specific prior written permission.
19
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
// Author: kenton@google.com (Kenton Varda)
32
// Based on original Protocol Buffers design by
33
// Sanjay Ghemawat, Jeff Dean, and others.
35
// Testing strategy: For each type of I/O (array, string, file, etc.) we
36
// create an output stream and write some data to it, then create a
37
// corresponding input stream to read the same data back and expect it to
38
// match. When the data is written, it is written in several small chunks
39
// of varying sizes, with a BackUp() after each chunk. It is read back
40
// similarly, but with chunks separated at different points. The whole
41
// process is run with a variety of block sizes for both the input and
44
// TODO(kenton): Rewrite this test to bring it up to the standards of all
45
// the other proto2 tests. May want to wait for gTest to implement
46
// "parametized tests" so that one set of tests can be used on all the
57
#include <sys/types.h>
63
#include <google/protobuf/io/zero_copy_stream_impl.h>
66
#include <google/protobuf/io/gzip_stream.h>
69
#include <google/protobuf/stubs/common.h>
70
#include <google/protobuf/testing/googletest.h>
71
#include <google/protobuf/testing/file.h>
72
#include <gtest/gtest.h>
80
#define pipe(fds) _pipe(fds, 4096, O_BINARY)
85
#define O_BINARY _O_BINARY
87
#define O_BINARY 0 // If this isn't defined, the platform doesn't need it.
91
class IoTest : public testing::Test {
95
// Helper to write an array of data to an output stream.
96
bool WriteToOutput(ZeroCopyOutputStream* output, const void* data, int size);
97
// Helper to read a fixed-length array of data from an input stream.
98
int ReadFromInput(ZeroCopyInputStream* input, void* data, int size);
99
// Write a string to the output stream.
100
void WriteString(ZeroCopyOutputStream* output, const string& str);
101
// Read a number of bytes equal to the size of the given string and checks
102
// that it matches the string.
103
void ReadString(ZeroCopyInputStream* input, const string& str);
104
// Writes some text to the output stream in a particular order. Returns
105
// the number of bytes written, incase the caller needs that to set up an
107
int WriteStuff(ZeroCopyOutputStream* output);
108
// Reads text from an input stream and expects it to match what
109
// WriteStuff() writes.
110
void ReadStuff(ZeroCopyInputStream* input);
112
// Similar to WriteStuff, but performs more sophisticated testing.
113
int WriteStuffLarge(ZeroCopyOutputStream* output);
114
// Reads and tests a stream that should have been written to
115
// via WriteStuffLarge().
116
void ReadStuffLarge(ZeroCopyInputStream* input);
119
string Compress(const string& data, const GzipOutputStream::Options& options);
120
string Uncompress(const string& data);
123
static const int kBlockSizes[];
124
static const int kBlockSizeCount;
127
const int IoTest::kBlockSizes[] = {-1, 1, 2, 5, 7, 10, 23, 64};
128
const int IoTest::kBlockSizeCount = GOOGLE_ARRAYSIZE(IoTest::kBlockSizes);
130
bool IoTest::WriteToOutput(ZeroCopyOutputStream* output,
131
const void* data, int size) {
132
const uint8* in = reinterpret_cast<const uint8*>(data);
139
if (!output->Next(&out, &out_size)) {
142
EXPECT_GT(out_size, 0);
144
if (in_size <= out_size) {
145
memcpy(out, in, in_size);
146
output->BackUp(out_size - in_size);
150
memcpy(out, in, out_size);
156
#define MAX_REPEATED_ZEROS 100
158
int IoTest::ReadFromInput(ZeroCopyInputStream* input, void* data, int size) {
159
uint8* out = reinterpret_cast<uint8*>(data);
165
int repeated_zeros = 0;
168
if (!input->Next(&in, &in_size)) {
169
return size - out_size;
171
EXPECT_GT(in_size, -1);
177
EXPECT_LT(repeated_zeros, MAX_REPEATED_ZEROS);
179
if (out_size <= in_size) {
180
memcpy(out, in, out_size);
181
if (in_size > out_size) {
182
input->BackUp(in_size - out_size);
184
return size; // Copied all of it.
187
memcpy(out, in, in_size);
193
void IoTest::WriteString(ZeroCopyOutputStream* output, const string& str) {
194
EXPECT_TRUE(WriteToOutput(output, str.c_str(), str.size()));
197
void IoTest::ReadString(ZeroCopyInputStream* input, const string& str) {
198
scoped_array<char> buffer(new char[str.size() + 1]);
199
buffer[str.size()] = '\0';
200
EXPECT_EQ(ReadFromInput(input, buffer.get(), str.size()), str.size());
201
EXPECT_STREQ(str.c_str(), buffer.get());
204
int IoTest::WriteStuff(ZeroCopyOutputStream* output) {
205
WriteString(output, "Hello world!\n");
206
WriteString(output, "Some te");
207
WriteString(output, "xt. Blah blah.");
208
WriteString(output, "abcdefg");
209
WriteString(output, "01234567890123456789");
210
WriteString(output, "foobar");
212
EXPECT_EQ(output->ByteCount(), 68);
214
int result = output->ByteCount();
218
// Reads text from an input stream and expects it to match what WriteStuff()
220
void IoTest::ReadStuff(ZeroCopyInputStream* input) {
221
ReadString(input, "Hello world!\n");
222
ReadString(input, "Some text. ");
223
ReadString(input, "Blah ");
224
ReadString(input, "blah.");
225
ReadString(input, "abcdefg");
226
EXPECT_TRUE(input->Skip(20));
227
ReadString(input, "foo");
228
ReadString(input, "bar");
230
EXPECT_EQ(input->ByteCount(), 68);
233
EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
236
int IoTest::WriteStuffLarge(ZeroCopyOutputStream* output) {
237
WriteString(output, "Hello world!\n");
238
WriteString(output, "Some te");
239
WriteString(output, "xt. Blah blah.");
240
WriteString(output, string(100000, 'x')); // A very long string
241
WriteString(output, string(100000, 'y')); // A very long string
242
WriteString(output, "01234567890123456789");
244
EXPECT_EQ(output->ByteCount(), 200055);
246
int result = output->ByteCount();
250
// Reads text from an input stream and expects it to match what WriteStuff()
252
void IoTest::ReadStuffLarge(ZeroCopyInputStream* input) {
253
ReadString(input, "Hello world!\nSome text. ");
254
EXPECT_TRUE(input->Skip(5));
255
ReadString(input, "blah.");
256
EXPECT_TRUE(input->Skip(100000 - 10));
257
ReadString(input, string(10, 'x') + string(100000 - 20000, 'y'));
258
EXPECT_TRUE(input->Skip(20000 - 10));
259
ReadString(input, "yyyyyyyyyy01234567890123456789");
261
EXPECT_EQ(input->ByteCount(), 200055);
264
EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
267
// ===================================================================
269
TEST_F(IoTest, ArrayIo) {
270
const int kBufferSize = 256;
271
uint8 buffer[kBufferSize];
273
for (int i = 0; i < kBlockSizeCount; i++) {
274
for (int j = 0; j < kBlockSizeCount; j++) {
277
ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
278
size = WriteStuff(&output);
281
ArrayInputStream input(buffer, size, kBlockSizes[j]);
289
TEST_F(IoTest, GzipIo) {
290
const int kBufferSize = 2*1024;
291
uint8* buffer = new uint8[kBufferSize];
292
for (int i = 0; i < kBlockSizeCount; i++) {
293
for (int j = 0; j < kBlockSizeCount; j++) {
294
for (int z = 0; z < kBlockSizeCount; z++) {
295
int gzip_buffer_size = kBlockSizes[z];
298
ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
299
GzipOutputStream gzout(
300
&output, GzipOutputStream::GZIP, gzip_buffer_size);
303
size = output.ByteCount();
306
ArrayInputStream input(buffer, size, kBlockSizes[j]);
307
GzipInputStream gzin(
308
&input, GzipInputStream::GZIP, gzip_buffer_size);
317
TEST_F(IoTest, ZlibIo) {
318
const int kBufferSize = 2*1024;
319
uint8* buffer = new uint8[kBufferSize];
320
for (int i = 0; i < kBlockSizeCount; i++) {
321
for (int j = 0; j < kBlockSizeCount; j++) {
322
for (int z = 0; z < kBlockSizeCount; z++) {
323
int gzip_buffer_size = kBlockSizes[z];
326
ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
327
GzipOutputStream gzout(
328
&output, GzipOutputStream::ZLIB, gzip_buffer_size);
331
size = output.ByteCount();
334
ArrayInputStream input(buffer, size, kBlockSizes[j]);
335
GzipInputStream gzin(
336
&input, GzipInputStream::ZLIB, gzip_buffer_size);
345
TEST_F(IoTest, ZlibIoInputAutodetect) {
346
const int kBufferSize = 2*1024;
347
uint8* buffer = new uint8[kBufferSize];
350
ArrayOutputStream output(buffer, kBufferSize);
351
GzipOutputStream gzout(&output, GzipOutputStream::ZLIB);
354
size = output.ByteCount();
357
ArrayInputStream input(buffer, size);
358
GzipInputStream gzin(&input, GzipInputStream::AUTO);
362
ArrayOutputStream output(buffer, kBufferSize);
363
GzipOutputStream gzout(&output, GzipOutputStream::GZIP);
366
size = output.ByteCount();
369
ArrayInputStream input(buffer, size);
370
GzipInputStream gzin(&input, GzipInputStream::AUTO);
376
string IoTest::Compress(const string& data,
377
const GzipOutputStream::Options& options) {
380
StringOutputStream output(&result);
381
GzipOutputStream gzout(&output, options);
382
WriteToOutput(&gzout, data.data(), data.size());
387
string IoTest::Uncompress(const string& data) {
390
ArrayInputStream input(data.data(), data.size());
391
GzipInputStream gzin(&input);
394
while (gzin.Next(&buffer, &size)) {
395
result.append(reinterpret_cast<const char*>(buffer), size);
401
TEST_F(IoTest, CompressionOptions) {
402
// Some ad-hoc testing of compression options.
405
File::ReadFileToStringOrDie(
406
TestSourceDir() + "/google/protobuf/testdata/golden_message",
409
GzipOutputStream::Options options;
410
string gzip_compressed = Compress(golden, options);
412
options.compression_level = 0;
413
string not_compressed = Compress(golden, options);
415
// Try zlib compression for fun.
416
options = GzipOutputStream::Options();
417
options.format = GzipOutputStream::ZLIB;
418
string zlib_compressed = Compress(golden, options);
420
// Uncompressed should be bigger than the original since it should have some
422
EXPECT_GT(not_compressed.size(), golden.size());
424
// Higher compression levels should result in smaller sizes.
425
EXPECT_LT(zlib_compressed.size(), not_compressed.size());
427
// ZLIB format should differ from GZIP format.
428
EXPECT_TRUE(zlib_compressed != gzip_compressed);
430
// Everything should decompress correctly.
431
EXPECT_TRUE(Uncompress(not_compressed) == golden);
432
EXPECT_TRUE(Uncompress(gzip_compressed) == golden);
433
EXPECT_TRUE(Uncompress(zlib_compressed) == golden);
437
// There is no string input, only string output. Also, it doesn't support
438
// explicit block sizes. So, we'll only run one test and we'll use
439
// ArrayInput to read back the results.
440
TEST_F(IoTest, StringIo) {
443
StringOutputStream output(&str);
447
ArrayInputStream input(str.data(), str.size());
453
// To test files, we create a temporary file, write, read, truncate, repeat.
454
TEST_F(IoTest, FileIo) {
455
string filename = TestTempDir() + "/zero_copy_stream_test_file";
457
for (int i = 0; i < kBlockSizeCount; i++) {
458
for (int j = 0; j < kBlockSizeCount; j++) {
459
// Make a temporary file.
461
open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
465
FileOutputStream output(file, kBlockSizes[i]);
467
EXPECT_EQ(0, output.GetErrno());
471
ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
474
FileInputStream input(file, kBlockSizes[j]);
476
EXPECT_EQ(0, input.GetErrno());
485
TEST_F(IoTest, GzipFileIo) {
486
string filename = TestTempDir() + "/zero_copy_stream_test_file";
488
for (int i = 0; i < kBlockSizeCount; i++) {
489
for (int j = 0; j < kBlockSizeCount; j++) {
490
// Make a temporary file.
492
open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
495
FileOutputStream output(file, kBlockSizes[i]);
496
GzipOutputStream gzout(&output);
497
WriteStuffLarge(&gzout);
500
EXPECT_EQ(0, output.GetErrno());
504
ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
507
FileInputStream input(file, kBlockSizes[j]);
508
GzipInputStream gzin(&input);
509
ReadStuffLarge(&gzin);
510
EXPECT_EQ(0, input.GetErrno());
519
// MSVC raises various debugging exceptions if we try to use a file
520
// descriptor of -1, defeating our tests below. This class will disable
521
// these debug assertions while in scope.
522
class MsvcDebugDisabler {
524
#if defined(_MSC_VER) && _MSC_VER >= 1400
525
MsvcDebugDisabler() {
526
old_handler_ = _set_invalid_parameter_handler(MyHandler);
527
old_mode_ = _CrtSetReportMode(_CRT_ASSERT, 0);
529
~MsvcDebugDisabler() {
530
old_handler_ = _set_invalid_parameter_handler(old_handler_);
531
old_mode_ = _CrtSetReportMode(_CRT_ASSERT, old_mode_);
534
static void MyHandler(const wchar_t *expr,
538
uintptr_t pReserved) {
542
_invalid_parameter_handler old_handler_;
545
// Dummy constructor and destructor to ensure that GCC doesn't complain
546
// that debug_disabler is an unused variable.
547
MsvcDebugDisabler() {}
548
~MsvcDebugDisabler() {}
552
// Test that FileInputStreams report errors correctly.
553
TEST_F(IoTest, FileReadError) {
554
MsvcDebugDisabler debug_disabler;
556
// -1 = invalid file descriptor.
557
FileInputStream input(-1);
561
EXPECT_FALSE(input.Next(&buffer, &size));
562
EXPECT_EQ(EBADF, input.GetErrno());
565
// Test that FileOutputStreams report errors correctly.
566
TEST_F(IoTest, FileWriteError) {
567
MsvcDebugDisabler debug_disabler;
569
// -1 = invalid file descriptor.
570
FileOutputStream input(-1);
575
// The first call to Next() succeeds because it doesn't have anything to
577
EXPECT_TRUE(input.Next(&buffer, &size));
579
// Second call fails.
580
EXPECT_FALSE(input.Next(&buffer, &size));
582
EXPECT_EQ(EBADF, input.GetErrno());
585
// Pipes are not seekable, so File{Input,Output}Stream ends up doing some
586
// different things to handle them. We'll test by writing to a pipe and
587
// reading back from it.
588
TEST_F(IoTest, PipeIo) {
591
for (int i = 0; i < kBlockSizeCount; i++) {
592
for (int j = 0; j < kBlockSizeCount; j++) {
593
// Need to create a new pipe each time because ReadStuff() expects
594
// to see EOF at the end.
595
ASSERT_EQ(pipe(files), 0);
598
FileOutputStream output(files[1], kBlockSizes[i]);
600
EXPECT_EQ(0, output.GetErrno());
602
close(files[1]); // Send EOF.
605
FileInputStream input(files[0], kBlockSizes[j]);
607
EXPECT_EQ(0, input.GetErrno());
614
// Test using C++ iostreams.
615
TEST_F(IoTest, IostreamIo) {
616
for (int i = 0; i < kBlockSizeCount; i++) {
617
for (int j = 0; j < kBlockSizeCount; j++) {
622
OstreamOutputStream output(&stream, kBlockSizes[i]);
624
EXPECT_FALSE(stream.fail());
628
IstreamInputStream input(&stream, kBlockSizes[j]);
630
EXPECT_TRUE(stream.eof());
638
OstreamOutputStream output(&stream, kBlockSizes[i]);
639
WriteStuffLarge(&output);
640
EXPECT_FALSE(stream.fail());
644
IstreamInputStream input(&stream, kBlockSizes[j]);
645
ReadStuffLarge(&input);
646
EXPECT_TRUE(stream.eof());
653
// To test ConcatenatingInputStream, we create several ArrayInputStreams
654
// covering a buffer and then concatenate them.
655
TEST_F(IoTest, ConcatenatingInputStream) {
656
const int kBufferSize = 256;
657
uint8 buffer[kBufferSize];
660
ArrayOutputStream output(buffer, kBufferSize);
663
// Now split it up into multiple streams of varying sizes.
664
ASSERT_EQ(68, output.ByteCount()); // Test depends on this.
665
ArrayInputStream input1(buffer , 12);
666
ArrayInputStream input2(buffer + 12, 7);
667
ArrayInputStream input3(buffer + 19, 6);
668
ArrayInputStream input4(buffer + 25, 15);
669
ArrayInputStream input5(buffer + 40, 0);
670
// Note: We want to make sure we have a stream boundary somewhere between
671
// bytes 42 and 62, which is the range that it Skip()ed by ReadStuff(). This
672
// tests that a bug that existed in the original code for Skip() is fixed.
673
ArrayInputStream input6(buffer + 40, 10);
674
ArrayInputStream input7(buffer + 50, 18); // Total = 68 bytes.
676
ZeroCopyInputStream* streams[] =
677
{&input1, &input2, &input3, &input4, &input5, &input6, &input7};
679
// Create the concatenating stream and read.
680
ConcatenatingInputStream input(streams, GOOGLE_ARRAYSIZE(streams));
684
// To test LimitingInputStream, we write our golden text to a buffer, then
685
// create an ArrayInputStream that contains the whole buffer (not just the
686
// bytes written), then use a LimitingInputStream to limit it just to the
688
TEST_F(IoTest, LimitingInputStream) {
689
const int kBufferSize = 256;
690
uint8 buffer[kBufferSize];
693
ArrayOutputStream output(buffer, kBufferSize);
697
ArrayInputStream array_input(buffer, kBufferSize);
698
LimitingInputStream input(&array_input, output.ByteCount());
703
// Check that a zero-size array doesn't confuse the code.
704
TEST(ZeroSizeArray, Input) {
705
ArrayInputStream input(NULL, 0);
708
EXPECT_FALSE(input.Next(&data, &size));
711
TEST(ZeroSizeArray, Output) {
712
ArrayOutputStream output(NULL, 0);
715
EXPECT_FALSE(output.Next(&data, &size));
720
} // namespace protobuf
721
} // namespace google