1
/* $Id: er_paf.c,v 1.17 2010/08/11 08:34:54 cgarcia Exp $
3
* This file is part of the ESO Common Pipeline Library
4
* Copyright (C) 2001-2004 European Southern Observatory
6
* This program is free software; you can redistribute it and/or modify
7
* it under the terms of the GNU General Public License as published by
8
* the Free Software Foundation; either version 2 of the License, or
9
* (at your option) any later version.
11
* This program is distributed in the hope that it will be useful,
12
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
* GNU General Public License for more details.
16
* You should have received a copy of the GNU General Public License
17
* along with this program; if not, write to the Free Software
18
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23
* $Date: 2010/08/11 08:34:54 $
25
* $Name: esorex-3_9_0 $
37
#include "cxstrutils.h"
41
#include "er_stringarray.h"
42
#include "er_fileutils.h"
45
#define ER_PAF_MANDATORY_KEYS_MAIN_EXT "PRO CATG|INSTRUME|MJD-OBS|ESO TPL ID|DATE-OBS|ESO OBS ID"
46
#define ER_PAF_MANDATORY_KEYS_EXT "EXTNAME"
47
#define ER_PAF_QC_KEYS "ESO QC"
50
* @defgroup er_paf PAF files handling
52
* This module provides a collection of functions that deal
53
* with the PAF files creation
59
/**********************************************************************/
61
* @brief Creates the PAF files
63
* @param frames The frames to create the PAF from
64
* @param paf_config The configuration file for PAF creation
66
* @returns 0 if successfull, !=0 otherwise
68
* Create a PAF file using the important keywords from the header
69
* of the products. The important keywords are specified in the configuration
73
/**********************************************************************/
75
int er_create_recipe_pafs
76
(const cpl_frameset * frames,
77
const char * recipe_name,
78
const char * paf_config_filename)
80
const cpl_frame * frame = NULL;
81
struct er_paf_config * paf_configuration = NULL;
82
int paf_err = CPL_ERROR_NONE;
83
char * arcfile = NULL;
86
cpl_msg_info(er_func, "Creating PAF files");
88
/* Read the configuration file */
89
paf_configuration = er_paf_read_config_file(paf_config_filename);
90
if(paf_configuration == NULL)
92
cpl_msg_error(er_func, "Could not parse the PAF configuration file");
93
return CPL_ERROR_BAD_FILE_FORMAT;
96
/* Read the ARCFILE keyword from the first raw file */
97
arcfile = er_paf_get_arcfile(frames);
100
cpl_msg_error(er_func, "Could not read ARCFILE from raw frames");
101
er_paf_config_delete(paf_configuration);
102
return CPL_ERROR_DATA_NOT_FOUND;
105
/* Loop into all the products */
106
cpl_msg_debug(__func__,"Looping into all the products");
107
frame = cpl_frameset_get_first_const(frames);
108
while (frame != NULL)
110
const char * filename;
111
filename = cpl_frame_get_filename(frame);
112
if (cpl_frame_get_group(frame) == CPL_FRAME_GROUP_PRODUCT &&
113
er_fileutils_file_is_fits(filename))
115
struct er_paf_config_item * config_item = NULL;
116
const char * this_pro_catg = NULL;
117
cpl_propertylist * procatg_proplist = NULL;
119
/* Get PRO CATG of this product */
120
procatg_proplist = cpl_propertylist_load(filename, 0);
121
this_pro_catg = cpl_propertylist_get_string
122
(procatg_proplist, CPL_DFS_PRO_CATG);
123
if(this_pro_catg == NULL)
125
cpl_msg_error(er_func,"Could not get PRO CATG in file %s",
127
er_paf_config_delete(paf_configuration);
129
paf_err = cpl_error_get_code();
133
cpl_msg_debug(er_func, "Product with PRO CATG=%s", this_pro_catg);
134
cpl_msg_indent_more();
136
/* Get the corresponding matching in the config file (if there is) */
137
config_item = er_paf_get_matching_item
138
(paf_configuration, this_pro_catg);
139
if(config_item != NULL)
143
cpl_msg_debug(er_func, "Matched with regexp =%s",
144
config_item->pro_catg_regexp);
146
/* Decide what to do depending on number of extensions */
147
next = cpl_frame_get_nextensions(frame);
150
paf_err = er_paf_single_hdu_fill_write
151
(frame, config_item, idx_paf, recipe_name, arcfile);
155
paf_err = er_paf_multi_hdu_fill_write
156
(frame, config_item, idx_paf, recipe_name, arcfile);
158
if(paf_err != CPL_ERROR_NONE)
160
cpl_propertylist_delete(procatg_proplist);
161
er_paf_config_delete(paf_configuration);
166
cpl_msg_indent_less();
167
cpl_propertylist_delete(procatg_proplist);
170
cpl_msg_debug(__func__,"File is not a FITS product %s",
172
frame = cpl_frameset_get_next_const(frames);
177
er_paf_config_delete(paf_configuration);
183
/**********************************************************************/
185
* @brief Creates a list of keywords and saves the corresponding PAF file
186
* for single HDU products.
188
* @param frame The frame product where to extract the keywords from
189
* @param config_item The configuration contains the keywords requested
190
* @param idx_paf The index number for this PAF file (it will be added to
192
* @param recipe_name The recipe name (this is saved in the PAF file)
193
* @param arcfile This is added as the ARCFILE keyword
194
* @returns CPL_ERROR_NONE if success or the error code otherwise
197
/**********************************************************************/
199
int er_paf_single_hdu_fill_write
200
(const cpl_frame * frame,
201
struct er_paf_config_item * config_item,
203
const char * recipe_name,
206
cpl_propertylist * paf_proplist = NULL;
209
/* Get the keywords to write */
211
er_paf_fill_paf_proplist_single_hdu(frame, config_item, arcfile);
212
if(paf_proplist == NULL)
214
cpl_msg_debug(__func__,"No keywords found for file %s. "
215
"Skipping writing of paf", cpl_frame_get_filename(frame));
216
return CPL_ERROR_NONE;
219
/* Get the PAF filename */
220
paf_filename = cpl_sprintf("qc_%d.paf",idx_paf);
222
/* Writing PAF file */
223
cpl_dfs_save_paf(cpl_propertylist_get_string
224
(paf_proplist,"INSTRUME"),
230
cpl_free(paf_filename);
231
cpl_propertylist_delete(paf_proplist);
233
return CPL_ERROR_NONE;
236
/**********************************************************************/
238
* @brief Creates a list of keywords and saves the corresponding PAF file
239
* for multiple HDU products.
241
* @param frame The frame product where to extract the keywords from
242
* @param config_item The configuration contains the keywords requested
243
* @param idx_paf The index number for these PAF files (it will be added
245
* @param recipe_name The recipe name (this is saved in the PAF file)
246
* @param arcfile This is added as the ARCFILE keyword
247
* @returns CPL_ERROR_NONE if success or the error code otherwise
250
/**********************************************************************/
252
int er_paf_multi_hdu_fill_write
253
(const cpl_frame * frame,
254
struct er_paf_config_item * config_item,
256
const char * recipe_name,
262
next = cpl_frame_get_nextensions(frame);
263
for(iext = 0 ; iext <= next ; ++iext)
265
cpl_propertylist * paf_proplist;
268
/* Get the keywords to write */
270
er_paf_fill_paf_proplist_multi_hdu(frame,config_item, iext, arcfile);
271
if(paf_proplist == NULL)
273
cpl_msg_warning(__func__,"Problem reading headers in file %s, extension %d",
274
cpl_frame_get_filename(frame), iext);
275
return CPL_ERROR_ILLEGAL_INPUT;
277
if(cpl_propertylist_get_size(paf_proplist) == 0)
279
cpl_msg_debug(__func__,"Skipping writing of paf for file %s, "
280
"extension %d (starting in 0).",
281
cpl_frame_get_filename(frame), iext);
282
cpl_propertylist_delete(paf_proplist);
286
/* Get the PAF filename */
287
paf_filename = cpl_sprintf("qc_%d_%d.paf",idx_paf, iext);
289
/* Writing PAF file */
290
cpl_dfs_save_paf(cpl_propertylist_get_string
291
(paf_proplist,"INSTRUME"),
296
cpl_free(paf_filename);
297
cpl_propertylist_delete(paf_proplist);
300
return CPL_ERROR_NONE;
304
/**********************************************************************/
306
* @brief Get the matching configuration section
308
* @param paf_configuration The whole configuration
309
* @param this_pro_catg The procatg to match
310
* @returns a pointer to a er_paf_config_item structure if there is
311
* a match, NULL otherwise
313
* This function loops through the config pro catg regular expressions and
314
* return the first match
316
/**********************************************************************/
317
struct er_paf_config_item * er_paf_get_matching_item
318
(struct er_paf_config * paf_configuration, const char * this_pro_catg)
320
struct er_paf_config_item * config_item = NULL;
323
cpl_msg_indent_more();
324
/* Loop through the config pro catg regular expressions */
325
for(item = 0; item < paf_configuration->nitems; ++item)
330
cpl_msg_debug(er_func, "Comparing PRO CATG with %s",
331
paf_configuration->items[item].pro_catg_regexp);
333
regstatus = regcomp(&pattern,
334
paf_configuration->items[item].pro_catg_regexp,
335
REG_EXTENDED|REG_NOSUB);
338
cpl_msg_error(er_func, "Bad formatted PRO CATG regular expresion");
339
cpl_error_set(er_func, CPL_ERROR_ILLEGAL_INPUT);
340
cpl_msg_indent_less();
345
regexec(&pattern, this_pro_catg, (size_t)0, NULL, 0);
346
if (regstatus != REG_NOMATCH)
348
config_item = paf_configuration->items+item;
349
cpl_msg_indent_less();
355
cpl_msg_indent_less();
362
/**********************************************************************/
364
* @brief Reads the configuration to write PAF files
366
* @param paf_config_filename Path to the configuration file
367
* @returns a pointer to a er_paf_config structure if successfull,
370
* This function reads the configuration file for PAF writing functionality.
371
* It returns an allocated er_paf_config object, which must be deallocated
372
* with er_paf_config_delete.
374
/**********************************************************************/
376
struct er_paf_config * er_paf_read_config_file
377
(const char * paf_config_filename)
379
struct er_paf_config * paf_configuration;
380
er_stringarray_t * file_buffer;
381
er_stringarray_t * file_stripped;
384
cpl_msg_debug(__func__,"Reading configuration file %s",paf_config_filename);
386
/* Allocate and init the structure */
387
paf_configuration = cpl_malloc(sizeof(struct er_paf_config));
388
paf_configuration->items = NULL;
389
paf_configuration->nitems = 0;
391
/* Read the whole file */
392
file_buffer = er_paf_read_whole_file(paf_config_filename);
393
if(file_buffer == NULL)
396
/* Strip the comments and blank lines */
397
cpl_msg_debug(__func__,"Stripping comments and blank lines");
398
file_stripped = er_stringarray_new();
399
for (iline = 0; iline < er_stringarray_size(file_buffer); ++iline)
402
line = er_stringarray_get(file_buffer, iline);
403
if (strcmp(line, "") != 0 && line[0] != '#' )
404
er_stringarray_append(file_stripped, line);
407
/* Parse the configuration lines and store it in a convenient structure */
408
er_paf_parse_config(file_stripped, paf_configuration);
410
/* clean and return */
411
er_stringarray_delete(file_buffer);
412
er_stringarray_delete(file_stripped);
413
return paf_configuration;
416
/**********************************************************************/
418
* @brief Creates a propertylist with all keywords to be saved for single
421
* @param frame The frame product where to extract the keywords from
422
* @param config_item The configuration contains the keywords requested
423
* @param arcfile This is added as the ARCFILE keyword
424
* @returns a pointer to the propertylist if sucess,
427
* This function will read all the keywords that are going to be
428
* saved in the PAF file. It includes the mandatory keywords, the QC
429
* keywords and the specified keywords in the config file and the ARCFILE
430
* keyword which comes from a external frame (the first raw frame from the
432
* This function should be used only for single extension products
434
/**********************************************************************/
436
cpl_propertylist* er_paf_fill_paf_proplist_single_hdu
437
(const cpl_frame * frame,
438
struct er_paf_config_item * config_item,
441
cpl_propertylist * paf_proplist;
442
cpl_propertylist * all_keywords;
443
const char * filename;
447
/* Get All keywords from FITS header */
448
filename = cpl_frame_get_filename(frame);
449
all_keywords = cpl_propertylist_load(filename, 0);
450
paf_proplist = cpl_propertylist_new();
451
if(all_keywords == NULL)
453
cpl_msg_error(er_func,"Could not read the main header of %s", filename);
457
/* Retrieve mandatory keywords */
458
cpl_propertylist_copy_property_regexp
459
(paf_proplist, all_keywords, ER_PAF_MANDATORY_KEYS_MAIN_EXT, 0);
460
cpl_propertylist_append_string(paf_proplist, "ARCFILE", arcfile);
462
/* Retrieve requested keywords in configuration */
463
nkeys = er_stringarray_size(config_item->requested_keywords);
464
for(ikey = 0 ; ikey < nkeys; ++ikey)
466
char * key = er_stringarray_get(config_item->requested_keywords, ikey);
467
cpl_propertylist * matched_keys;
469
matched_keys = cpl_propertylist_new();
470
cpl_propertylist_copy_property_regexp(matched_keys, all_keywords,key,0);
471
if(cpl_propertylist_get_size(matched_keys) == 0)
472
cpl_msg_debug(er_func, "Cannot find keywords matching %s", key);
473
cpl_propertylist_copy_property_regexp(paf_proplist, matched_keys,"",0);
474
cpl_propertylist_delete(matched_keys);
477
/* Retrieve QC keywords */
478
cpl_propertylist_copy_property_regexp
479
(paf_proplist, all_keywords, ER_PAF_QC_KEYS, 0);
481
/* Bad return if there are no keywords at all */
482
if(cpl_propertylist_get_size(paf_proplist) == 0)
484
cpl_msg_error(er_func, "Could not get any keyword from file %s",
486
cpl_propertylist_delete(paf_proplist);
487
cpl_propertylist_delete(all_keywords);
491
/* Clean and return */
492
cpl_propertylist_delete(all_keywords);
496
/**********************************************************************/
498
* @brief Creates a propertylist with all keywords to be saved for multi
501
* @param frame The frame product where to extract the keywords from
502
* @param config_item The configuration contains the keywords requested
503
* @param iext The number of the extension to get the keywords from
504
* @param arcfile This is added as the ARCFILE keyword
505
* @returns a pointer to the propertylist if success,
508
* This function will read all the keywords that are going to be
509
* saved in the PAF file. It includes the mandatory keywords, the QC
510
* keywords and the specified keywords in the config file and the ARCFILE
511
* keyword which comes from a external frame (the first raw frame from the
512
* frameset). All the keywords are retrieved from both the main extension
513
* and the requested extension. If there are duplicated keywords in the
514
* main header, the extension ones will take precedence
516
* This function should be used only for multi extension products
518
/**********************************************************************/
520
cpl_propertylist* er_paf_fill_paf_proplist_multi_hdu
521
(const cpl_frame * frame,
522
struct er_paf_config_item * config_item,
526
cpl_propertylist * paf_proplist;
527
cpl_propertylist * all_keywords_hdu_main;
528
cpl_propertylist * all_keywords_hdu_ext;
529
cpl_propertylist * qc_keywords;
530
const char * filename;
534
/* Read All keywords from FITS headers */
535
paf_proplist = cpl_propertylist_new();
536
filename = cpl_frame_get_filename(frame);
537
all_keywords_hdu_main = cpl_propertylist_load(filename, 0);
538
if(all_keywords_hdu_main == NULL)
540
cpl_msg_error(er_func,"Could not read the main header of %s", filename);
543
all_keywords_hdu_ext = cpl_propertylist_load(filename, iext);
544
if(all_keywords_hdu_ext == NULL)
546
cpl_msg_error(er_func,"Could not read header of extension %d in %s",
551
/* Retrieve mandatory keywords from main header */
552
cpl_propertylist_copy_property_regexp
553
(paf_proplist, all_keywords_hdu_main,
554
ER_PAF_MANDATORY_KEYS_MAIN_EXT, 0);
555
cpl_propertylist_append_string(paf_proplist, "ARCFILE", arcfile);
557
/* Retrieve mandatory keywords from extension */
558
cpl_propertylist_copy_property_regexp
559
(paf_proplist, all_keywords_hdu_ext, ER_PAF_MANDATORY_KEYS_EXT, 0);
561
/* Retrieve requested keywords in configuration */
562
nkeys = er_stringarray_size(config_item->requested_keywords);
563
for(ikey = 0 ; ikey < nkeys; ++ikey)
565
char * key = er_stringarray_get(config_item->requested_keywords, ikey);
566
cpl_propertylist * matched_keys;
568
matched_keys = cpl_propertylist_new();
569
cpl_propertylist_copy_property_regexp(matched_keys,
570
all_keywords_hdu_main,key,0);
571
cpl_propertylist_copy_property_regexp(matched_keys,
572
all_keywords_hdu_ext,key,0);
573
if(cpl_propertylist_get_size(matched_keys) == 0)
574
cpl_msg_debug(er_func, "Cannot find keywords matching %s", key);
575
cpl_propertylist_copy_property_regexp(paf_proplist, matched_keys,"",0);
576
cpl_propertylist_delete(matched_keys);
579
/* Retrieve QC keywords */
580
qc_keywords = cpl_propertylist_new();
581
cpl_propertylist_copy_property_regexp
582
(qc_keywords, all_keywords_hdu_ext, ER_PAF_QC_KEYS, 0);
583
if(cpl_propertylist_get_size(qc_keywords) == 0)
585
cpl_msg_debug(er_func, "No QC keywords found in file %s, extension %d",
587
cpl_propertylist_delete(paf_proplist);
588
cpl_propertylist_delete(all_keywords_hdu_main);
589
cpl_propertylist_delete(all_keywords_hdu_ext);
592
cpl_propertylist_copy_property_regexp(paf_proplist, qc_keywords,"",0);
593
cpl_propertylist_delete(qc_keywords);
595
/* Bad return if there are no keywords at all */
596
if(cpl_propertylist_get_size(paf_proplist) == 0)
598
cpl_msg_warning(er_func, "Could not get any keyword from file %s",
600
cpl_propertylist_delete(all_keywords_hdu_main);
601
cpl_propertylist_delete(all_keywords_hdu_ext);
605
/* Clean and return */
606
cpl_propertylist_delete(all_keywords_hdu_main);
607
cpl_propertylist_delete(all_keywords_hdu_ext);
611
char * er_paf_get_arcfile(const cpl_frameset * frames)
613
const cpl_frame * frame;
614
char * arcfile = NULL;
615
cpl_propertylist * all_keywords;
616
const char * filename;
618
/* Loop into all the products */
619
frame = cpl_frameset_get_first_const(frames);
620
while (frame != NULL)
622
if (cpl_frame_get_group(frame) == CPL_FRAME_GROUP_RAW)
624
filename = cpl_frame_get_filename(frame);
625
all_keywords = cpl_propertylist_load(filename, 0);
626
if(all_keywords == NULL)
628
cpl_msg_error(er_func,"Could not read the main header of %s",
633
(cpl_propertylist_get_string(all_keywords, "ARCFILE"));
634
cpl_propertylist_delete(all_keywords);
637
frame = cpl_frameset_get_next_const(frames);;
639
cpl_msg_error(er_func,"There are no RAW frames");
644
/**********************************************************************/
646
* @brief Reads all the file in a string buffer
648
* @param paf_config_filename Path to the configuration file
649
* @returns a pointer to the lines buffer if successfull, NULL otherwise
652
/**********************************************************************/
653
er_stringarray_t * er_paf_read_whole_file(const char * paf_config_filename)
655
FILE * file_descriptor;
656
char line[MAXSTRLENCONF];
657
er_stringarray_t * file_buffer;
659
/* Allocate string array */
660
file_buffer = er_stringarray_new();
662
/* Go through all the file */
663
file_descriptor = fopen(paf_config_filename, "r");
664
if (file_descriptor == NULL)
666
memset(line, 0, MAXSTRLENCONF);
668
while (fgets(line, MAXSTRLENCONF -1, file_descriptor) != NULL)
670
/* Fill the array with this new line */
671
er_stringarray_append(file_buffer, line);
672
/* Reinitialize buffer */
673
memset(line, '\0', MAXSTRLENCONF);
676
fclose(file_descriptor);
680
/**********************************************************************/
682
* @brief Actually parses the configuration into items, one per PRO CATG regexp
684
* @param paf_configuration The configuration
685
* @returns 0 if successfull, !=0 otherwise
688
/**********************************************************************/
689
int er_paf_parse_config
690
(er_stringarray_t * config_lines,
691
struct er_paf_config * paf_configuration)
693
unsigned int iline = 0;
695
unsigned int nitem = 0;
696
struct er_paf_config_item * items;
698
cpl_msg_debug(__func__,"Parsing the paf configuration file");
700
/* Alias for the items structure */
701
items = paf_configuration->items;
703
/* Parse all the lines */
704
nlines = er_stringarray_size(config_lines);
705
cpl_msg_debug(__func__,"Number of lines in pkd file: %d", nlines);
706
while(iline < nlines)
708
char * current_line = er_stringarray_get(config_lines, (int)iline);
709
if(strncmp(current_line, "PRO CATG",8) == 0)
711
struct er_paf_config_item* new_item;
712
char * pro_catg_regexp = NULL;
714
/* Allocate space for the new config item */
716
items = cpl_realloc(items,
717
nitem*sizeof(struct er_paf_config_item));
718
new_item = items + nitem -1;
719
new_item->pro_catg_regexp = cpl_malloc(MAXSTRLENCONF*sizeof(char));
720
new_item->requested_keywords = er_stringarray_new();
721
paf_configuration->nitems = nitem;
723
/* Get the regexp for PRO CATG */
724
pro_catg_regexp = strchr(current_line, '=');
725
if(pro_catg_regexp == NULL || strlen(pro_catg_regexp) < 2)
727
cpl_msg_error(er_func,"Parse error in PAF config. "
731
strncpy(new_item->pro_catg_regexp,
732
pro_catg_regexp + 1, MAXSTRLENCONF);
733
/* Remove leading and trailing characters */
734
cx_strstrip(new_item->pro_catg_regexp);
737
while(iline < nlines)
739
char * keyword = er_stringarray_get(config_lines, (int)iline);
740
if(strncmp(keyword, "PRO CATG",8) == 0)
742
/* Remove leading and trailing characters */
743
cx_strstrip(keyword);
745
/* If it is a keyword containing only spaces, do not count it */
746
if(strcmp(keyword,"") != 0)
748
/* Add this keyword to the requested keywords */
749
er_stringarray_append(new_item->requested_keywords, keyword);
756
cpl_msg_error(er_func, "Error parsing PAF configuration file");
757
return CPL_ERROR_BAD_FILE_FORMAT;
760
cpl_msg_debug(__func__,"Number of pro catg definitions found in file: %d",
761
paf_configuration->nitems);
763
/* Get the alias back */
764
paf_configuration->items = items;
769
/**********************************************************************/
771
* @brief Deallocates the configuration to write PAF files
773
* @param paf_configuration The configuration
774
* @returns 0 if successfull, !=0 otherwise
776
* This function deallocates the configuration of a PAF writing process,
777
* that is, the er_paf_config structure.
779
/**********************************************************************/
781
int er_paf_config_delete(struct er_paf_config* paf_configuration)
785
/* Deallocate the items */
786
for (item = 0; item < paf_configuration->nitems; ++item)
788
er_stringarray_delete
789
(paf_configuration->items[item].requested_keywords);
790
cpl_free(paf_configuration->items[item].pro_catg_regexp);
792
cpl_free(paf_configuration->items);
794
/* Deallocate the configuration itself */
795
cpl_free(paf_configuration);
797
return CPL_ERROR_NONE;