← Back to branch summary

~mortenoh/+junk/dhis2-detailed-import-export

« back to all changes in this revision

Viewing changes to local/in/dhis-web-integration/src/main/java/org/hisp/dhis/integration/rims/action/RIMSImportResultAction.java

  • Committer: larshelge at gmail
  • Date: 2009-03-03 16:46:36 UTC
  • Revision ID: larshelge@gmail.com-20090303164636-2sjlrquo7ib1gf7r
Initial check-in

Show diffs side-by-side

added added

removed removed

Lines of Context:
local/in/dhis-web-integration/src/main/java/org/hisp/dhis/integration/rims/action/RIMSImportResultAction.java
 
1
package org.hisp.dhis.integration.rims.action;
 
2
 
 
3
import java.sql.SQLException;
 
4
import java.util.ArrayList;
 
5
import java.util.Calendar;
 
6
import java.util.Collection;
 
7
import java.util.Date;
 
8
import java.util.Iterator;
 
9
import java.util.List;
 
10
import java.util.regex.Pattern;
 
11
 
 
12
import org.hibernate.HibernateException;
 
13
import org.hisp.dhis.aggregation.batch.statement.StatementManager;
 
14
import org.hisp.dhis.dataelement.DataElement;
 
15
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
 
16
import org.hisp.dhis.dataelement.DataElementCategoryOptionComboService;
 
17
import org.hisp.dhis.dataelement.DataElementService;
 
18
import org.hisp.dhis.dataset.DataSet;
 
19
import org.hisp.dhis.dataset.DataSetService;
 
20
import org.hisp.dhis.datavalue.DataValue;
 
21
import org.hisp.dhis.datavalue.DataValueService;
 
22
import org.hisp.dhis.i18n.I18nFormat;
 
23
import org.hisp.dhis.integration.rims.api.RIMSService;
 
24
import org.hisp.dhis.integration.rims.api.RIMSTable;
 
25
import org.hisp.dhis.integration.rims.api.RIMS_Mapping_DataElement;
 
26
import org.hisp.dhis.integration.rims.api.RIMS_Mapping_Orgunit;
 
27
import org.hisp.dhis.integration.rims.api.tables.DistrictTable;
 
28
import org.hisp.dhis.integration.rims.api.tables.PHCTable;
 
29
import org.hisp.dhis.integration.rims.api.tables.RIMSDistrict;
 
30
import org.hisp.dhis.integration.rims.api.tables.RIMSOrgUnit;
 
31
import org.hisp.dhis.integration.rims.api.tables.RIMS_PHC;
 
32
import org.hisp.dhis.integration.rims.util.Configuration;
 
33
import org.hisp.dhis.integration.rims.util.ImportExportError;
 
34
import org.hisp.dhis.organisationunit.OrganisationUnit;
 
35
import org.hisp.dhis.organisationunit.OrganisationUnitService;
 
36
import org.hisp.dhis.period.MonthlyPeriodType;
 
37
import org.hisp.dhis.period.Period;
 
38
import org.hisp.dhis.period.PeriodService;
 
39
import org.hisp.dhis.period.PeriodType;
 
40
import org.hisp.dhis.period.YearlyPeriodType;
 
41
 
 
42
import com.opensymphony.xwork.ActionSupport;
 
43
 
 
44
import static java.util.Calendar.YEAR;
 
45
import static java.util.Calendar.MONTH;
 
46
import static java.util.Calendar.DAY_OF_MONTH;
 
47
import static java.util.Calendar.DAY_OF_YEAR;
 
48
 
 
49
public class RIMSImportResultAction
 
50
    extends ActionSupport
 
51
{
 
52
    // -------------------------------------------------------------------------
 
53
    // Dependencies
 
54
    // -------------------------------------------------------------------------
 
55
    private StatementManager statementManager;
 
56
 
 
57
    public void setStatementManager( StatementManager statementManager )
 
58
    {
 
59
        this.statementManager = statementManager;
 
60
    }
 
61
 
 
62
    private RIMSService rimsService;
 
63
 
 
64
    public void setRimsService( RIMSService rimsService )
 
65
    {
 
66
        this.rimsService = rimsService;
 
67
    }
 
68
 
 
69
    private DataSetService dataSetService;
 
70
 
 
71
    public void setDataSetService( DataSetService dataSetService )
 
72
    {
 
73
        this.dataSetService = dataSetService;
 
74
    }
 
75
 
 
76
    private PeriodService periodService;
 
77
 
 
78
    public void setPeriodService( PeriodService periodService )
 
79
    {
 
80
        this.periodService = periodService;
 
81
    }
 
82
 
 
83
    private OrganisationUnitService organisationUnitService;
 
84
 
 
85
    public void setOrganisationUnitService( OrganisationUnitService organisationUnitService )
 
86
    {
 
87
        this.organisationUnitService = organisationUnitService;
 
88
    }
 
89
 
 
90
    private DataElementService dataElementService;
 
91
 
 
92
    public void setDataElementService( DataElementService dataElementService )
 
93
    {
 
94
        this.dataElementService = dataElementService;
 
95
    }
 
96
 
 
97
    private DataValueService dataValueService;
 
98
 
 
99
    public void setDataValueService( DataValueService dataValueService )
 
100
    {
 
101
        this.dataValueService = dataValueService;
 
102
    }
 
103
 
 
104
    private DataElementCategoryOptionComboService dataElementCategoryOptionComboService;
 
105
 
 
106
    public void setDataElementCategoryOptionComboService(
 
107
        DataElementCategoryOptionComboService dataElementCategoryOptionComboService )
 
108
    {
 
109
        this.dataElementCategoryOptionComboService = dataElementCategoryOptionComboService;
 
110
    }
 
111
 
 
112
    private I18nFormat format;
 
113
 
 
114
    public void setFormat( I18nFormat format )
 
115
    {
 
116
        this.format = format;
 
117
    }
 
118
 
 
119
    // --------------------------------------------------------------------------
 
120
    // Parameters
 
121
    // --------------------------------------------------------------------------
 
122
 
 
123
    private Date sDate;
 
124
 
 
125
    private Date eDate;
 
126
 
 
127
    private List<String> selectedOrgunits;
 
128
 
 
129
    public void setSelectedOrgunits( List<String> selectedOrgunits )
 
130
    {
 
131
        this.selectedOrgunits = selectedOrgunits;
 
132
    }
 
133
 
 
134
    private List<String> rimsDEGroups;
 
135
 
 
136
    public void setRimsDEGroups( List<String> rimsDEGroups )
 
137
    {
 
138
        this.rimsDEGroups = rimsDEGroups;
 
139
    }
 
140
 
 
141
    private String startDate;
 
142
 
 
143
    public void setStartDate( String startDate )
 
144
    {
 
145
        this.startDate = startDate;
 
146
    }
 
147
 
 
148
    private String endDate;
 
149
 
 
150
    public void setEndDate( String endDate )
 
151
    {
 
152
        this.endDate = endDate;
 
153
    }
 
154
    
 
155
    private String rimsDistricts;
 
156
    
 
157
    private String includeDistrict;
 
158
    
 
159
    private String connection;
 
160
 
 
161
    private int totalRecCount;
 
162
 
 
163
    private Collection<ImportExportError> skipped = new ArrayList<ImportExportError>();
 
164
 
 
165
    private boolean abort;
 
166
 
 
167
    public int getTotalRecCount()
 
168
    {
 
169
        return totalRecCount;
 
170
    }
 
171
 
 
172
    // --------------------------------------------------------------------------
 
173
    // Action Implementation
 
174
    // --------------------------------------------------------------------------
 
175
    public String execute()
 
176
        throws Exception
 
177
    {
 
178
        // TODO Useful for testing, turn off for production use
 
179
        boolean includeTotals = false;
 
180
        
 
181
        Configuration config = new Configuration( connection );
 
182
        statementManager.initialise();
 
183
 
 
184
        // ---------------------------------------------------------------------
 
185
        // OrganisationUnit Info
 
186
        // ---------------------------------------------------------------------
 
187
        List<Object> rimsSelectedOus = new ArrayList<Object>();
 
188
        
 
189
        if ( includeDistrict != null )
 
190
        {
 
191
            // Get selected district and add it to the list of selected OUs
 
192
            rimsSelectedOus.add( rimsService.getDistrict( rimsDistricts, connection ) );
 
193
        }
 
194
        else if ( selectedOrgunits == null )
 
195
        {
 
196
            throw new IllegalArgumentException( "No orgunits selected" );
 
197
        }
 
198
 
 
199
        if ( selectedOrgunits != null )
 
200
        {
 
201
            for ( String rimsOU: selectedOrgunits )
 
202
            {
 
203
                RIMS_PHC rimsOUObj = rimsService.getPHC( rimsOU, connection );
 
204
                rimsSelectedOus.add( rimsOUObj );
 
205
            }
 
206
        }
 
207
 
 
208
        // DataElement Info
 
209
        List<RIMS_Mapping_DataElement> rimsSelectedMappingDEs = new ArrayList<RIMS_Mapping_DataElement>(
 
210
            getSelectedMappingDEs() );
 
211
 
 
212
        // Period Info
 
213
        sDate = format.parseDate( startDate );
 
214
        eDate = format.parseDate( endDate );
 
215
        if ( eDate == null )
 
216
        {
 
217
            throw new IllegalArgumentException( endDate +" is not a valid date" );
 
218
        }
 
219
        // Add periods if necessary
 
220
        checkAllPeriods( sDate, eDate );
 
221
        List<Period> selectedPeriods = new ArrayList<Period>( periodService.getIntersectingPeriods( sDate, eDate ) );
 
222
    
 
223
        int orgUnitI = 0;
 
224
all:
 
225
        // TODO Figure out this generics stuff
 
226
        for ( Object o: rimsSelectedOus)
 
227
        {
 
228
            RIMSOrgUnit rimsOrgUnit = (RIMSOrgUnit) o;
 
229
            int percentage = (int) (orgUnitI++ / (double) rimsSelectedOus.size() * 100);
 
230
            System.out.print( "("+ percentage +" %) Importing "+ rimsOrgUnit.getName() );
 
231
            
 
232
            RIMS_Mapping_Orgunit mappingOrgUnit = rimsService
 
233
            .getMappingOrgUnit( rimsOrgUnit.getCode() ); 
 
234
            
 
235
            for ( RIMS_Mapping_DataElement mappingDataElement: rimsSelectedMappingDEs)
 
236
            {
 
237
                System.out.print( '.' );
 
238
                if ( mappingDataElement.getDhisExpression().equalsIgnoreCase( "NA" )
 
239
                    || mappingDataElement.getDhisExpression().equalsIgnoreCase( "NO_SUCH_DE" ) )
 
240
                {
 
241
                    System.out.println( "Skipped dataelement \""+ mappingDataElement.getDeName() +"\": marked as "+ mappingDataElement.getDhisExpression() );
 
242
                    continue;
 
243
                }
 
244
                
 
245
                if ( mappingDataElement.isTotal() && !includeTotals )
 
246
                {
 
247
                    continue;
 
248
                }
 
249
 
 
250
                for ( Period period: selectedPeriods )
 
251
                {
 
252
                    // Flip this using the debugger to abort.
 
253
                    // Eventually the progress indicator will have an abort button.                    
 
254
                    if ( abort )
 
255
                    {
 
256
                        break all;
 
257
                    }
 
258
                    try
 
259
                    {
 
260
 
 
261
                        if ( period.getPeriodType().getName().equalsIgnoreCase(
 
262
                            "monthly" ) )
 
263
                        {
 
264
                            if ( !config.getMonthlyFields().contains(
 
265
                                mappingDataElement.getRimsColumn() ) )
 
266
                            {
 
267
                                continue;
 
268
                            }
 
269
                        }
 
270
                        else if ( period.getPeriodType().getName()
 
271
                            .equalsIgnoreCase( "yearly" ) )
 
272
                        {
 
273
                            if ( !config.getYearlyFields().contains(
 
274
                                mappingDataElement.getRimsColumn() ) )
 
275
                            {
 
276
                                continue;
 
277
                            }
 
278
                        }
 
279
                        else
 
280
                        {
 
281
                            continue;
 
282
                        }
 
283
 
 
284
                        // -----------------------------------------------------
 
285
                        // Skip if mixing district/PHC data
 
286
                        // -----------------------------------------------------
 
287
                        RIMSTable table = config.getTable( mappingDataElement.getTableName() );
 
288
                        
 
289
                        // Prevent superfluous "dataelement not found" errors
 
290
                        if ( ( table instanceof DistrictTable
 
291
                            && rimsOrgUnit instanceof RIMS_PHC ) 
 
292
                          || ( table instanceof PHCTable
 
293
                            && rimsOrgUnit instanceof RIMSDistrict ) )
 
294
                        {
 
295
                            continue;
 
296
                        }
 
297
                        
 
298
                        // -----------------------------------------------------
 
299
                        // Get data value
 
300
                        // -----------------------------------------------------
 
301
    
 
302
                        OrganisationUnit dhisOrgUnit = organisationUnitService
 
303
                            .getOrganisationUnit( Integer.parseInt( 
 
304
                                mappingOrgUnit.getDhisid() ) );
 
305
                        if ( !dhisFormulaIsPlain( mappingDataElement
 
306
                            .getDhisExpression() ) )
 
307
                        {
 
308
                            skipped.add( new ImportExportError( mappingDataElement, mappingOrgUnit,
 
309
                                    "Only plain formulas (no operators) are supported for import" ) );
 
310
                            continue;
 
311
                        }
 
312
                        // Get value from RIMS database
 
313
                        String rimsValue = getRIMSDataValue(
 
314
                            mappingDataElement, period, rimsOrgUnit, config );
 
315
 
 
316
                        if ( rimsValue == null )
 
317
                        {
 
318
                            // Log and skip
 
319
                            DataElement dataElement = dataElementService
 
320
                                .getDataElement( mappingDataElement
 
321
                                    .getDhisDataElementId() );
 
322
                            skipped.add( new ImportExportError( dataElement, mappingOrgUnit, period,
 
323
                                "Data value not found in RIMS" ) );
 
324
                            continue;
 
325
                        }
 
326
 
 
327
                        // ---------------------------------------------------------
 
328
                        // Actual import
 
329
                        // ---------------------------------------------------------
 
330
 
 
331
                        // Data element
 
332
                        DataElement dataElement = dataElementService
 
333
                            .getDataElement( mappingDataElement
 
334
                                .getDhisDataElementId() );
 
335
                        if ( dataElement == null )
 
336
                        {
 
337
                            skipped.add( new ImportExportError( dataElement, mappingOrgUnit,
 
338
                                "Data element not found in DHIS" ) );
 
339
                            continue;
 
340
                        }
 
341
 
 
342
                        // Option combo
 
343
                        DataElementCategoryOptionCombo optionCombo = dataElementCategoryOptionComboService
 
344
                            .getDataElementCategoryOptionCombo( mappingDataElement
 
345
                                .getDhisOptionCombo() );
 
346
 
 
347
                        // If data value exists, replace it, otherwise create
 
348
                        // it.
 
349
                        DataValue dataValue = dataValueService.getDataValue(
 
350
                            dhisOrgUnit, dataElement, period, optionCombo );
 
351
 
 
352
                        if ( dataValue == null )
 
353
                        {
 
354
                            dataValue = new DataValue( dataElement, period,
 
355
                                dhisOrgUnit, rimsValue, config.getStoredBy(),
 
356
                                new Date(), null, optionCombo );
 
357
                            dataValueService.addDataValue( dataValue );
 
358
                        }
 
359
                        else
 
360
                        {
 
361
                            dataValue.setValue( rimsValue );
 
362
                            dataValue.setTimestamp( new Date() );
 
363
                            dataValue.setStoredBy( config.getStoredBy() );
 
364
 
 
365
                            dataValueService.updateDataValue( dataValue );
 
366
                        }
 
367
                        totalRecCount++;
 
368
                    }
 
369
                    catch ( HibernateException hbe )
 
370
                    {
 
371
                        throw new Exception( "Database-access error", hbe );
 
372
                    }
 
373
                    catch ( Exception e )
 
374
                    {
 
375
                        // Log dataelement and reason
 
376
                        try 
 
377
                        {
 
378
                            DataElement dataElement = dataElementService
 
379
                                .getDataElement( mappingDataElement
 
380
                                    .getDhisDataElementId() );
 
381
                            skipped.add( new ImportExportError( dataElement,
 
382
                                mappingOrgUnit, period, e ) );
 
383
                            // TODO DEBUG: Beep added
 
384
                            System.err.println( e.getClass().getSimpleName() +": "+ e.getMessage() );                            
 
385
                        }
 
386
                        catch ( NumberFormatException f )
 
387
                        {
 
388
                            skipped.add( new ImportExportError( mappingDataElement, 
 
389
                                mappingOrgUnit, period, e ) );
 
390
                        }
 
391
                        continue;
 
392
                    }
 
393
                }
 
394
            }
 
395
            System.out.println( "done" );
 
396
        }
 
397
 
 
398
        statementManager.destroy();
 
399
        return SUCCESS;
 
400
    }
 
401
 
 
402
    // TODO A candidate for the DHIS core.
 
403
    /**
 
404
     * Check that all required periods are present, and add them if necessary.
 
405
     *  
 
406
     * @param start check periods after this.
 
407
     * @param end check periods before this.
 
408
     */
 
409
    private void checkAllPeriods( Date start, Date end )
 
410
    {
 
411
        PeriodType monthly = new MonthlyPeriodType();
 
412
        PeriodType yearly = new YearlyPeriodType();
 
413
        
 
414
        // ---------------------------------------------------------------------
 
415
        // Monthly
 
416
        // ---------------------------------------------------------------------
 
417
        
 
418
        // These calendars keep track of the first and last of each month as we iterate.
 
419
        Calendar firstOfMonth = Calendar.getInstance();
 
420
        Calendar lastOfMonth = Calendar.getInstance();
 
421
        
 
422
        // Start with the month which startDate is in. 
 
423
        firstOfMonth.setTime( start );
 
424
        firstOfMonth.set( DAY_OF_MONTH, 1 );
 
425
        
 
426
        lastOfMonth.setTime( start );
 
427
        lastOfMonth.set( DAY_OF_MONTH, lastOfMonth.getActualMaximum( DAY_OF_MONTH ) );
 
428
        
 
429
        // Stop when firstOfMonth has passed endDate.
 
430
        while( firstOfMonth.getTime().compareTo( end ) < 0 )
 
431
        {
 
432
            // Is the period present?
 
433
            Period checkPeriod = periodService.getPeriod( 
 
434
                firstOfMonth.getTime(), lastOfMonth.getTime(), monthly );
 
435
            if ( checkPeriod == null )
 
436
            {
 
437
                // Add it.
 
438
                periodService.addPeriod( new Period( 
 
439
                    monthly, firstOfMonth.getTime(), lastOfMonth.getTime() ) );
 
440
            }
 
441
 
 
442
            // Increment by one month.
 
443
            firstOfMonth.add( MONTH, 1 );
 
444
            lastOfMonth.add( MONTH, 1 );
 
445
            lastOfMonth.set( DAY_OF_MONTH, lastOfMonth.getActualMaximum( DAY_OF_MONTH ) );
 
446
        }
 
447
        
 
448
        // ---------------------------------------------------------------------
 
449
        // Yearly
 
450
        // ---------------------------------------------------------------------
 
451
        // These calendars keep track of the first and last of each month as we iterate.
 
452
        Calendar firstOfYear = Calendar.getInstance();
 
453
        Calendar lastOfYear = Calendar.getInstance();
 
454
        
 
455
        // Start with the month which startDate is in. 
 
456
        firstOfYear.setTime( start );
 
457
        firstOfYear.set( DAY_OF_YEAR, 1 );
 
458
        
 
459
        lastOfYear.setTime( start );
 
460
        lastOfYear.set( DAY_OF_YEAR, lastOfYear.getActualMaximum( DAY_OF_YEAR ) );
 
461
        
 
462
        // Stop when firstOfYear has passed endDate.
 
463
        while( firstOfYear.getTime().compareTo( end ) < 0 )
 
464
        {
 
465
            // Is the period present?
 
466
            Period checkPeriod = periodService.getPeriod( 
 
467
                firstOfYear.getTime(), lastOfYear.getTime(), yearly);
 
468
            if ( checkPeriod == null )
 
469
            {
 
470
                // Add it.
 
471
                periodService.addPeriod( new Period( 
 
472
                    yearly, firstOfYear.getTime(), lastOfYear.getTime() ) );
 
473
            }
 
474
 
 
475
            // Increment by one year.
 
476
            firstOfYear.add( YEAR, 1 );
 
477
            lastOfYear.add( YEAR, 1 );
 
478
            lastOfYear.set( DAY_OF_YEAR, lastOfYear.getActualMaximum( DAY_OF_YEAR ) );
 
479
        }
 
480
    }
 
481
 
 
482
    private String getRIMSDataValue(
 
483
        RIMS_Mapping_DataElement mappingDataElement, Period period,
 
484
        RIMSOrgUnit orgUnit, Configuration config ) throws SQLException
 
485
    {
 
486
        Calendar cal = Calendar.getInstance();
 
487
        cal.setTime( period.getStartDate() );
 
488
        int year = cal.get( Calendar.YEAR );
 
489
        int month = cal.get( Calendar.MONTH );
 
490
        month = month + 1;
 
491
 
 
492
        RIMSTable table = config.getTable( mappingDataElement.getTableName() );
 
493
        return table.getDataValue( mappingDataElement, orgUnit, month, year );
 
494
    }
 
495
 
 
496
    /**
 
497
     * Retrieve all selected dataelements for mapping from each dataelement group.
 
498
     *  
 
499
     * @return a list of all dataelements from each group.
 
500
     */
 
501
    public List<RIMS_Mapping_DataElement> getSelectedMappingDEs() throws Exception
 
502
    {
 
503
        List<RIMS_Mapping_DataElement> rimsSelectedMappingsDes = new ArrayList<RIMS_Mapping_DataElement>();
 
504
 
 
505
        for ( String sectionName: rimsDEGroups )
 
506
        {
 
507
            rimsSelectedMappingsDes.addAll( rimsService.getRIMSDataElementsByDEGroup( sectionName ) );
 
508
        }
 
509
        return rimsSelectedMappingsDes;
 
510
    }
 
511
 
 
512
    /*
 
513
     * Returns the PeriodType Object for selected DataElement, If no PeriodType
 
514
     * is found then by default returns Monthly Period type
 
515
     */
 
516
    public PeriodType getDataElementPeriodType( DataElement de )
 
517
    {
 
518
        List<DataSet> dataSetList = new ArrayList<DataSet>( dataSetService.getAllDataSets() );
 
519
        Iterator<DataSet> it = dataSetList.iterator();
 
520
        while ( it.hasNext() )
 
521
        {
 
522
            DataSet ds = (DataSet) it.next();
 
523
            List<DataElement> dataElementList = new ArrayList<DataElement>( ds.getDataElements() );
 
524
            if ( dataElementList.contains( de ) )
 
525
            {
 
526
                return ds.getPeriodType();
 
527
            }
 
528
        }
 
529
 
 
530
        return periodService.getPeriodTypeByName( "Monthly" );
 
531
    } // getDataElementPeriodType end
 
532
 
 
533
    private boolean dhisFormulaIsPlain( String formula )
 
534
    {
 
535
        // First check for the plain syntax (no operators)
 
536
        String plain = "^\\[\\d+\\.\\d+\\]$";
 
537
        if ( Pattern.matches( plain, formula ) )
 
538
        {
 
539
            return true;
 
540
        }
 
541
        // Then check that the formula has at least one data-element/option combo
 
542
        String legal = "\\[\\d+\\.\\d+\\]";
 
543
        if ( Pattern.matches( legal, formula ) )
 
544
        {
 
545
            return false;
 
546
        }
 
547
        throw new IllegalArgumentException( formula +" is not a valid dataelement expression" );
 
548
    }
 
549
 
 
550
    public Collection<ImportExportError> getSkipped()
 
551
    {
 
552
        return skipped;
 
553
    }
 
554
 
 
555
    public void setIncludeDistrict( String includeDistrict )
 
556
    {
 
557
        this.includeDistrict = includeDistrict;
 
558
    }
 
559
 
 
560
    public void setRimsDistricts( String rimsDistricts )
 
561
    {
 
562
        this.rimsDistricts = rimsDistricts;
 
563
    }
 
564
 
 
565
    public void setConnection( String connection )
 
566
    {
 
567
        this.connection = connection;
 
568
    }
 
569
 
 
570
}