2
* Licensed to the Apache Software Foundation (ASF) under one or more
3
* contributor license agreements. See the NOTICE file distributed with
4
* this work for additional information regarding copyright ownership.
5
* The ASF licenses this file to you under the Apache License, Version 2.0
6
* (the "License"); you may not use this file except in compliance with
7
* the License. You may obtain a copy of the License at
9
* http://www.apache.org/licenses/LICENSE-2.0
11
* Unless required by applicable law or agreed to in writing, software
12
* distributed under the License is distributed on an "AS IS" BASIS,
13
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14
* implied. See the License for the specific language governing
15
* permissions and limitations under the License.
23
/* Test code for JIRA Issue AVRO-984.
25
* AVRO-984: Avro-C schema resolution fails on nested array
27
* This program tests schema resolution for nested arrays. For the
28
* purposes of this test, there are two schemas "old" and "new" which
29
* are created by reading the same JSON schema.
31
* The test creates and populates a nested array, and serializes it to
32
* memory. The raw memory is written to a file, primarily to decouple
33
* writing and reading. Note that the schema is not written to the
34
* file. The nested array is also printed to the screen.
36
* The binary file is then read using two separate readers -- the
37
* matched reader and the resolved reader.
39
* In the matched reader case, the "old" and "new" schemas are known
40
* to match, and therefore no schema resolution is done. The binary
41
* buffer is deserialized into an avro value and the nested array
42
* encoded in the avro value is printed to the screen.
44
* In the resolved reader case, the "old" and "new" schemas are not
45
* known to match, and therefore schema resolution is performed. (Note
46
* that the schemas *do* match, but we perform schema resolution
47
* anyway, to test the resolution process). The schema resolution
48
* appears to succeed. However, once the code tries to perform an
49
* "avro_value_read()" the code fails to read the nested array into
52
* Additionally valgrind indicates that conditional jumps are being
53
* performed based on uninitialized values.
55
* AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib
56
* The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static
58
* This file was compiled under Linux using:
59
* gcc -g avro-984-test.c -o avro984 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro
61
* The code was tested with valgrind using the command:
62
* valgrind -v --leak-check=full --track-origins=yes ./avro984
67
// Encode the following json string in NESTED_ARRAY
68
// {"type":"array", "items": {"type": "array", "items": "long"}}
70
#define NESTED_ARRAY \
71
"{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}"
73
avro_schema_t schema_old = NULL;
74
avro_schema_t schema_new = NULL;
76
/* Parse schema into a schema data structure */
77
void init_schema(void)
79
avro_schema_error_t error;
80
if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY),
81
&schema_old, &error)) {
82
printf( "Unable to parse old schema\n");
86
if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY),
87
&schema_new, &error)) {
88
printf( "Unable to parse new schema\n");
93
#define try(call, msg) \
96
printf( msg ":\n %s\n", avro_strerror()); \
97
exit (EXIT_FAILURE); \
102
/* The input avro_value_t p_array should contain a nested array.
103
* Print the fields of this nested array to the screen.
105
int print_array_fields ( avro_value_t *p_array )
109
avro_type_t val_type;
111
val_type = avro_value_get_type( p_array );
112
printf( "Main array type = %d\n", val_type );
114
try( avro_value_get_size( p_array, &length ),
115
"Couldn't get array size" );
116
printf( "Main array length = %d\n", (int) length );
118
for ( idx = 0; idx < length; idx ++ )
120
avro_value_t subarray;
125
try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ),
126
"Couldn't get subarray" );
128
val_type = avro_value_get_type( &subarray );
129
printf( "Subarray type = %d\n", val_type );
131
try( avro_value_get_size( &subarray, &sublength ),
132
"Couldn't get subarray size" );
133
printf( "Subarray length = %d\n", (int) sublength );
135
for ( jdx = 0; jdx < sublength; jdx++ )
137
avro_value_t element;
140
try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ),
141
"Couldn't get subarray element" );
143
val_type = avro_value_get_type( &element );
145
try ( avro_value_get_long( &element, &val ),
146
"Couldn't get subarray element value" );
148
printf( "nested_array[%d][%d]: type = %d value = %lld\n",
149
(int) idx, (int) jdx, (int) val_type, (long long) val );
158
/* The input avro_value_t p_subarray should contain an array of long
159
* integers. Add "elements" number of long integers to this array. Set
160
* the values to be distinct based on the iteration parameter.
162
int add_subarray( avro_value_t *p_subarray,
166
avro_value_t element;
170
for ( idx = 0; idx < (size_t) elements; idx ++ )
172
// Append avro array element to subarray
173
try ( avro_value_append( p_subarray, &element, &index ),
174
"Error appending element in subarray" );
176
try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ),
177
"Error setting subarray element" );
184
/* Create a nested array using the schema NESTED_ARRAY. Populate its
185
* elements with unique values. Serialize the nested array to the
186
* memory buffer in avro_writer_t. The number of elements in the first
187
* dimension of the nested array is "elements". The number of elements
188
* in the second dimension of the nested array is hardcoded to 2.
190
int add_array( avro_writer_t writer,
193
avro_schema_t chosen_schema;
194
avro_value_iface_t *nested_array_class;
198
// Select (hardcode) schema to use
199
chosen_schema = schema_old;
201
// Create avro class and value
202
nested_array_class = avro_generic_class_from_schema( chosen_schema );
203
try ( avro_generic_value_new( nested_array_class, &nested ),
204
"Error creating instance of record" );
206
for ( idx = 0; idx < elements; idx ++ )
208
avro_value_t subarray;
211
// Append avro array element for top level array
212
try ( avro_value_append( &nested, &subarray, &index ),
213
"Error appending subarray" );
215
// Populate array element with subarray of length 2
216
#define SUBARRAY_LENGTH (2)
217
try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ),
218
"Error populating subarray" );
221
// Write the value to memory
222
try ( avro_value_write( writer, &nested ),
223
"Unable to write nested into memory" );
225
print_array_fields( &nested );
227
// Release the record
228
avro_value_decref( &nested );
229
avro_value_iface_decref( nested_array_class );
234
/* Create a raw binary file containing a serialized version of a
235
* nested array. This file will later be read by
236
* read_nested_array_file().
238
int write_nested_array_file ( int64_t buf_len, const char *raw_binary_file_name )
241
avro_writer_t nested_writer;
244
fprintf( stdout, "Create %s\n", raw_binary_file_name );
247
buf = (char *) malloc( buf_len * sizeof( char ) );
250
printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name);
254
/* Create a new memory writer */
255
nested_writer = avro_writer_memory( buf, buf_len );
256
if ( nested_writer == NULL )
258
printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name);
262
/* Add an array containing 4 subarrays */
263
printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) );
264
#define ARRAY_LENGTH (4)
265
add_array( nested_writer, ARRAY_LENGTH );
266
printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) );
268
/* Serialize the nested array */
269
printf( "Serialize the data to a file\n");
271
/* Delete the nested array if it exists, and create a new one */
272
remove(raw_binary_file_name);
273
fid = fopen( raw_binary_file_name, "w+");
276
printf( "There was an error creating the file %s.\n", raw_binary_file_name);
279
fwrite( buf, 1, avro_writer_tell( nested_writer ), fid );
281
avro_writer_free( nested_writer );
287
/* Read the raw binary file containing a serialized version of a
288
* nested array, written by write_nested_array_file()
290
int read_nested_array_file ( int64_t buf_len,
291
const char *raw_binary_file_name,
292
avro_schema_t writer_schema,
293
avro_schema_t reader_schema,
294
int use_resolving_reader
300
avro_reader_t nested_reader;
303
// For Matched Reader and Resolving Reader
304
avro_value_iface_t *reader_class;
307
// For Resolving Reader
308
avro_value_iface_t *resolver;
309
avro_value_t resolved_value;
311
fprintf( stdout, "Use %s reader\n", use_resolving_reader ? "Resolving":"Matched" );
314
buf = (char *) calloc( buf_len, sizeof( char ) );
317
printf( "There was an error creating the buffer for reading %s.\n", raw_binary_file_name);
320
// Start with a garbage buffer
321
memset(buf, 0xff, buf_len );
323
// Read the file into the buffer
324
fid = fopen( raw_binary_file_name, "r" );
327
printf( "There was an error reading the file %s.\n", raw_binary_file_name);
330
file_len = fread( buf, 1, buf_len, fid );
331
printf( "Read %d bytes\n", (int) file_len );
334
if ( use_resolving_reader )
338
/* First resolve the writer and reader schemas */
339
resolver = avro_resolved_writer_new( writer_schema, reader_schema );
342
printf( "Could not create resolver\n");
347
/* Create a value that the resolver can write into. This is just
348
* an interface value, that is not directly read from.
350
if ( avro_resolved_writer_new_value( resolver, &resolved_value ) )
352
avro_value_iface_decref( resolver );
357
/* Then create the value with the reader schema, that we are going
358
* to use to read from.
360
reader_class = avro_generic_class_from_schema(reader_schema);
361
try ( avro_generic_value_new( reader_class, &nested ),
362
"Error creating instance of nested array" );
364
// When we read the memory using the resolved writer, we want to
365
// populate the instance of the value with the reader schema. This
366
// is done by set_dest.
367
avro_resolved_writer_set_dest(&resolved_value, &nested);
369
// Create a memory reader
370
nested_reader = avro_reader_memory( buf, buf_len );
372
if ( avro_value_read( nested_reader, &resolved_value ) )
374
printf( "Avro value read failed\n" );
376
avro_value_decref( &nested );
377
avro_value_iface_decref( reader_class );
378
avro_value_iface_decref( resolver );
379
avro_value_decref( &resolved_value );
387
reader_class = avro_generic_class_from_schema(reader_schema);
389
try ( avro_generic_value_new( reader_class, &nested ),
390
"Error creating instance of nested array" );
392
// Send the memory in the buffer into the reader
393
nested_reader = avro_reader_memory( buf, buf_len );
395
try ( avro_value_read( nested_reader, &nested ),
396
"Could not read value from memory" );
400
/* Now the resolved record has been read into "nested" which is
401
* a value of type reader_class
403
print_array_fields( &nested );
405
if ( use_resolving_reader )
408
avro_value_decref( &nested );
409
avro_value_iface_decref( reader_class );
410
avro_value_iface_decref( resolver );
411
avro_value_decref( &resolved_value );
416
avro_value_decref( &nested );
417
avro_value_iface_decref( reader_class );
420
fprintf( stdout, "Done.\n\n");
421
avro_reader_free( nested_reader );
427
/* Top level function to impelement a test for the JIRA issue
428
* AVRO-984. See detailed documentation at the top of this file.
432
const char *raw_binary_file_name = "nested_array.bin";
433
int64_t buf_len = 2048;
434
int use_resolving_reader;
436
/* Initialize the schema structure from JSON */
439
printf( "Write the serialized nested array to %s\n", raw_binary_file_name );
441
write_nested_array_file( buf_len, raw_binary_file_name );
443
printf("\nNow read all the array back out\n\n");
445
for ( use_resolving_reader = 0; use_resolving_reader < 2; use_resolving_reader++ )
447
read_nested_array_file( buf_len,
448
raw_binary_file_name,
456
avro_schema_decref(schema_old);
457
avro_schema_decref(schema_new);
459
// Remove the binary file
460
remove(raw_binary_file_name);
added
removed
tests/test_avro_984.c
