← Back to branch summary

~mortenoh/+junk/dhis2-detailed-import-export

« back to all changes in this revision

Viewing changes to dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/file/importer/DefaultDhis14FileImportService.java

  • Committer: larshelge at gmail
  • Date: 2009-03-03 16:46:36 UTC
  • Revision ID: larshelge@gmail.com-20090303164636-2sjlrquo7ib1gf7r
Initial check-in

Show diffs side-by-side

added added

removed removed

Lines of Context:
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/file/importer/DefaultDhis14FileImportService.java
 
1
package org.hisp.dhis.importexport.dhis14.file.importer;
 
2
 
 
3
/*
 
4
 * Copyright (c) 2004-2007, University of Oslo
 
5
 * All rights reserved.
 
6
 *
 
7
 * Redistribution and use in source and binary forms, with or without
 
8
 * modification, are permitted provided that the following conditions are met:
 
9
 * * Redistributions of source code must retain the above copyright notice, this
 
10
 *   list of conditions and the following disclaimer.
 
11
 * * Redistributions in binary form must reproduce the above copyright notice,
 
12
 *   this list of conditions and the following disclaimer in the documentation
 
13
 *   and/or other materials provided with the distribution.
 
14
 * * Neither the name of the HISP project nor the names of its contributors may
 
15
 *   be used to endorse or promote products derived from this software without
 
16
 *   specific prior written permission.
 
17
 *
 
18
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 
20
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 
21
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 
22
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 
23
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 
24
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 
25
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 
26
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 
27
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
28
 */
 
29
 
 
30
import static org.hisp.dhis.expression.Expression.SEPARATOR;
 
31
 
 
32
import java.io.InputStream;
 
33
import java.util.HashMap;
 
34
import java.util.HashSet;
 
35
import java.util.List;
 
36
import java.util.Map;
 
37
import java.util.Set;
 
38
 
 
39
import org.apache.commons.logging.Log;
 
40
import org.apache.commons.logging.LogFactory;
 
41
import org.hisp.dhis.cache.HibernateCacheManager;
 
42
import org.hisp.dhis.dataelement.DataElementCategoryCombo;
 
43
import org.hisp.dhis.dataelement.DataElementCategoryComboService;
 
44
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
 
45
import org.hisp.dhis.dataelement.DataElementCategoryOptionComboService;
 
46
import org.hisp.dhis.dataelement.DataElementService;
 
47
import org.hisp.dhis.dataset.DataSetService;
 
48
import org.hisp.dhis.datavalue.DataValueService;
 
49
import org.hisp.dhis.expression.ExpressionService;
 
50
import org.hisp.dhis.importexport.ImportInternalProcess;
 
51
import org.hisp.dhis.importexport.ImportObjectService;
 
52
import org.hisp.dhis.importexport.ImportParams;
 
53
import org.hisp.dhis.importexport.analysis.ImportAnalyser;
 
54
import org.hisp.dhis.importexport.dhis14.file.query.QueryManager;
 
55
import org.hisp.dhis.importexport.dhis14.file.rowhandler.CalculatedDataElementRowHandler;
 
56
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataElementGroupMemberRowHandler;
 
57
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataElementGroupRowHandler;
 
58
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataElementRowHandler;
 
59
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataSetMemberRowHandler;
 
60
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataSetOrganisationUnitAssociationRowHandler;
 
61
import org.hisp.dhis.importexport.dhis14.file.rowhandler.DataSetRowHandler;
 
62
import org.hisp.dhis.importexport.dhis14.file.rowhandler.GroupSetMemberRowHandler;
 
63
import org.hisp.dhis.importexport.dhis14.file.rowhandler.GroupSetRowHandler;
 
64
import org.hisp.dhis.importexport.dhis14.file.rowhandler.IndicatorGroupMemberRowHandler;
 
65
import org.hisp.dhis.importexport.dhis14.file.rowhandler.IndicatorGroupRowHandler;
 
66
import org.hisp.dhis.importexport.dhis14.file.rowhandler.IndicatorRowHandler;
 
67
import org.hisp.dhis.importexport.dhis14.file.rowhandler.IndicatorTypeRowHandler;
 
68
import org.hisp.dhis.importexport.dhis14.file.rowhandler.OnChangePeriodRowHandler;
 
69
import org.hisp.dhis.importexport.dhis14.file.rowhandler.OrganisationUnitGroupMemberRowHandler;
 
70
import org.hisp.dhis.importexport.dhis14.file.rowhandler.OrganisationUnitGroupRowHandler;
 
71
import org.hisp.dhis.importexport.dhis14.file.rowhandler.OrganisationUnitRelationshipRowHandler;
 
72
import org.hisp.dhis.importexport.dhis14.file.rowhandler.OrganisationUnitRowHandler;
 
73
import org.hisp.dhis.importexport.dhis14.file.rowhandler.PeriodRowHandler;
 
74
import org.hisp.dhis.importexport.dhis14.file.rowhandler.RoutineDataValueRowHandler;
 
75
import org.hisp.dhis.importexport.dhis14.file.rowhandler.SemiPermanentDataValueRowHandler;
 
76
import org.hisp.dhis.importexport.dhis14.object.Dhis14CalculatedDataElementEntry;
 
77
import org.hisp.dhis.importexport.dhis14.util.Dhis14PeriodUtil;
 
78
import org.hisp.dhis.importexport.mapping.NameMappingUtil;
 
79
import org.hisp.dhis.importexport.mapping.ObjectMappingGenerator;
 
80
import org.hisp.dhis.indicator.IndicatorService;
 
81
import org.hisp.dhis.jdbc.BatchHandler;
 
82
import org.hisp.dhis.jdbc.BatchHandlerFactory;
 
83
import org.hisp.dhis.jdbc.batchhandler.DataElementBatchHandler;
 
84
import org.hisp.dhis.jdbc.batchhandler.DataElementGroupBatchHandler;
 
85
import org.hisp.dhis.jdbc.batchhandler.DataElementGroupMemberBatchHandler;
 
86
import org.hisp.dhis.jdbc.batchhandler.DataSetBatchHandler;
 
87
import org.hisp.dhis.jdbc.batchhandler.DataSetMemberBatchHandler;
 
88
import org.hisp.dhis.jdbc.batchhandler.DataSetSourceAssociationBatchHandler;
 
89
import org.hisp.dhis.jdbc.batchhandler.DataValueBatchHandler;
 
90
import org.hisp.dhis.jdbc.batchhandler.GroupSetBatchHandler;
 
91
import org.hisp.dhis.jdbc.batchhandler.GroupSetMemberBatchHandler;
 
92
import org.hisp.dhis.jdbc.batchhandler.ImportDataValueBatchHandler;
 
93
import org.hisp.dhis.jdbc.batchhandler.IndicatorBatchHandler;
 
94
import org.hisp.dhis.jdbc.batchhandler.IndicatorGroupBatchHandler;
 
95
import org.hisp.dhis.jdbc.batchhandler.IndicatorGroupMemberBatchHandler;
 
96
import org.hisp.dhis.jdbc.batchhandler.IndicatorTypeBatchHandler;
 
97
import org.hisp.dhis.jdbc.batchhandler.OrganisationUnitBatchHandler;
 
98
import org.hisp.dhis.jdbc.batchhandler.OrganisationUnitGroupBatchHandler;
 
99
import org.hisp.dhis.jdbc.batchhandler.OrganisationUnitGroupMemberBatchHandler;
 
100
import org.hisp.dhis.jdbc.batchhandler.PeriodBatchHandler;
 
101
import org.hisp.dhis.jdbc.batchhandler.SourceBatchHandler;
 
102
import org.hisp.dhis.organisationunit.OrganisationUnitGroupService;
 
103
import org.hisp.dhis.organisationunit.OrganisationUnitService;
 
104
import org.hisp.dhis.period.PeriodService;
 
105
import org.hisp.dhis.system.util.AppendingHashMap;
 
106
import org.hisp.dhis.system.util.DateUtils;
 
107
 
 
108
import com.ibatis.sqlmap.client.event.RowHandler;
 
109
 
 
110
/**
 
111
 * @author Lars Helge Overland
 
112
 * @version $Id: DefaultDhis14FileImportService.java 6425 2008-11-22 00:08:57Z larshelg $
 
113
 */
 
114
public class DefaultDhis14FileImportService
 
115
    extends ImportInternalProcess
 
116
{
 
117
    private final Log log = LogFactory.getLog( DefaultDhis14FileImportService.class );
 
118
    
 
119
    // -------------------------------------------------------------------------
 
120
    // Dependencies
 
121
    // -------------------------------------------------------------------------
 
122
 
 
123
    private QueryManager queryManager;
 
124
 
 
125
    public void setQueryManager( QueryManager queryManager )
 
126
    {
 
127
        this.queryManager = queryManager;
 
128
    }
 
129
 
 
130
    private ObjectMappingGenerator objectMappingGenerator;
 
131
 
 
132
    public void setObjectMappingGenerator( ObjectMappingGenerator objectMappingGenerator )
 
133
    {
 
134
        this.objectMappingGenerator = objectMappingGenerator;
 
135
    }
 
136
    
 
137
    private BatchHandlerFactory batchHandlerFactory;
 
138
 
 
139
    public void setBatchHandlerFactory( BatchHandlerFactory batchHandlerFactory )
 
140
    {
 
141
        this.batchHandlerFactory = batchHandlerFactory;
 
142
    }
 
143
    
 
144
    private ImportObjectService importObjectService;
 
145
 
 
146
    public void setImportObjectService( ImportObjectService importObjectService )
 
147
    {
 
148
        this.importObjectService = importObjectService;
 
149
    }
 
150
    
 
151
    private ExpressionService expressionService;
 
152
 
 
153
    public void setExpressionService( ExpressionService expressionService )
 
154
    {
 
155
        this.expressionService = expressionService;
 
156
    }
 
157
    
 
158
    private DataElementService dataElementService;
 
159
 
 
160
    public void setDataElementService( DataElementService dataElementService )
 
161
    {
 
162
        this.dataElementService = dataElementService;
 
163
    }
 
164
 
 
165
    private DataElementCategoryComboService categoryComboService;
 
166
    
 
167
    public void setCategoryComboService( DataElementCategoryComboService categoryComboService )
 
168
    {
 
169
        this.categoryComboService = categoryComboService;
 
170
    }
 
171
    
 
172
    private DataElementCategoryOptionComboService categoryOptionComboService;
 
173
 
 
174
    public void setCategoryOptionComboService( DataElementCategoryOptionComboService categoryOptionComboService )
 
175
    {
 
176
        this.categoryOptionComboService = categoryOptionComboService;
 
177
    }
 
178
    
 
179
    private PeriodService periodService;
 
180
 
 
181
    public void setPeriodService( PeriodService periodService )
 
182
    {
 
183
        this.periodService = periodService;
 
184
    }
 
185
    
 
186
    private DataSetService dataSetService;
 
187
 
 
188
    public void setDataSetService( DataSetService dataSetService )
 
189
    {
 
190
        this.dataSetService = dataSetService;
 
191
    }
 
192
    
 
193
    private OrganisationUnitService organisationUnitService;
 
194
 
 
195
    public void setOrganisationUnitService( OrganisationUnitService organisationUnitService )
 
196
    {
 
197
        this.organisationUnitService = organisationUnitService;
 
198
    }
 
199
    
 
200
    private OrganisationUnitGroupService organisationUnitGroupService;
 
201
 
 
202
    public void setOrganisationUnitGroupService( OrganisationUnitGroupService organisationUnitGroupService )
 
203
    {
 
204
        this.organisationUnitGroupService = organisationUnitGroupService;
 
205
    }
 
206
    
 
207
    private IndicatorService indicatorService;
 
208
 
 
209
    public void setIndicatorService( IndicatorService indicatorService )
 
210
    {
 
211
        this.indicatorService = indicatorService;
 
212
    }
 
213
    
 
214
    private DataValueService dataValueService;
 
215
 
 
216
    public void setDataValueService( DataValueService dataValueService )
 
217
    {
 
218
        this.dataValueService = dataValueService;
 
219
    }
 
220
    
 
221
    private ImportAnalyser importAnalyser;
 
222
 
 
223
    public void setImportAnalyser( ImportAnalyser importAnalyser )
 
224
    {
 
225
        this.importAnalyser = importAnalyser;
 
226
    }    
 
227
 
 
228
    private HibernateCacheManager cacheManager;
 
229
 
 
230
    public void setCacheManager( HibernateCacheManager cacheManager )
 
231
    {
 
232
        this.cacheManager = cacheManager;
 
233
    }
 
234
 
 
235
    // -------------------------------------------------------------------------
 
236
    // Constructor
 
237
    // -------------------------------------------------------------------------
 
238
 
 
239
    public DefaultDhis14FileImportService()
 
240
    {
 
241
        super();
 
242
    }
 
243
    
 
244
    // -------------------------------------------------------------------------
 
245
    // ImportInternalProcess implementation
 
246
    // -------------------------------------------------------------------------
 
247
 
 
248
    @Override
 
249
    public void importData( ImportParams params, InputStream inputStream )
 
250
    {
 
251
        if ( params.isPreview() )
 
252
        {
 
253
            importObjectService.deleteImportObjects();
 
254
        }
 
255
 
 
256
        importDataElements( params );
 
257
        importCalculatedDataElements( params, createCalculatedDataElementEntryMap() );
 
258
        importIndicatorTypes( params );
 
259
        importIndicators( params );
 
260
        importDataElementGroups( params );
 
261
        importDataElementGroupMembers( params );
 
262
        importIndicatorGroups( params );
 
263
        importIndicatorGroupMembers( params );
 
264
        
 
265
        importDataSets( params );
 
266
        importDataSetMembers( params );
 
267
 
 
268
        importOrganisationUnits( params );            
 
269
        importOrganisationUnitGroups( params );
 
270
        importOrganisationUnitGroupMembers( params );
 
271
        importGroupSets( params );
 
272
        importGroupSetMembers( params );
 
273
        importOrganisationUnitRelationships( params );
 
274
        importOrganisationUnitHierarchy();
 
275
 
 
276
        importDataSetOrganisationUnitAssociations( params );
 
277
        
 
278
        if ( params.isDataValues() && !params.isAnalysis() )
 
279
        {
 
280
            importPeriods( params );
 
281
            importRoutineDataValues( params );
 
282
 
 
283
            importOnChangePeriods( params );
 
284
            importSemiPermanentDataValues( params );
 
285
        }
 
286
        
 
287
        if ( params.isAnalysis() )
 
288
        {
 
289
            setOutput( importAnalyser.getImportAnalysis() );
 
290
        }
 
291
        
 
292
        setMessage( "import_process_done" );
 
293
        
 
294
        NameMappingUtil.clearMapping();
 
295
        Dhis14PeriodUtil.clear();
 
296
        
 
297
        cacheManager.clearCache();
 
298
    }
 
299
    
 
300
    // -------------------------------------------------------------------------
 
301
    // DataElement and Indicator
 
302
    // -------------------------------------------------------------------------
 
303
 
 
304
    private void importDataElements( ImportParams params )
 
305
    {
 
306
        setMessage( "importing_data_elements" );
 
307
        
 
308
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataElementBatchHandler.class );
 
309
        
 
310
        DataElementCategoryCombo categoryCombo = categoryComboService.
 
311
            getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME );
 
312
        
 
313
        RowHandler rowHandler = new DataElementRowHandler( batchHandler,
 
314
            importObjectService,
 
315
            dataElementService, 
 
316
            params,
 
317
            categoryCombo,
 
318
            importAnalyser );
 
319
 
 
320
        batchHandler.init();
 
321
        
 
322
        queryManager.queryWithRowhandler( "getDataElements", rowHandler );
 
323
 
 
324
        batchHandler.flush();
 
325
        
 
326
        log.info( "Imported DataElements" );
 
327
    }
 
328
 
 
329
    private void importCalculatedDataElements( ImportParams params, Map<Integer, String> calculatedEntryMap )
 
330
    {
 
331
        setMessage( "importing_data_elements" );
 
332
        
 
333
        DataElementCategoryCombo categoryCombo = categoryComboService.
 
334
            getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME );
 
335
        
 
336
        RowHandler rowHandler = new CalculatedDataElementRowHandler( importObjectService,
 
337
            dataElementService, 
 
338
            params,
 
339
            categoryCombo,
 
340
            importAnalyser,
 
341
            expressionService,
 
342
            calculatedEntryMap,
 
343
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ),
 
344
            getCategoryOptionComboMapping() );
 
345
 
 
346
        queryManager.queryWithRowhandler( "getCalculatedDataElements", rowHandler );
 
347
 
 
348
        log.info( "Imported CalculatedDataElements" );
 
349
    }
 
350
    
 
351
    private void importIndicatorTypes( ImportParams params )
 
352
    {
 
353
        setMessage( "importing_indicator_types" );
 
354
        
 
355
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( IndicatorTypeBatchHandler.class );
 
356
        
 
357
        RowHandler rowHandler = new IndicatorTypeRowHandler( batchHandler,
 
358
            importObjectService,
 
359
            indicatorService,
 
360
            params );
 
361
        
 
362
        batchHandler.init();
 
363
        
 
364
        queryManager.queryWithRowhandler( "getIndicatorTypes", rowHandler );
 
365
        
 
366
        batchHandler.flush();
 
367
        
 
368
        log.info( "Imported IndicatorTypes" );
 
369
    }
 
370
    
 
371
    private void importIndicators( ImportParams params )
 
372
    {
 
373
        setMessage( "importing_indicators" );
 
374
        
 
375
        BatchHandler indicatorBatchHandler = batchHandlerFactory.createBatchHandler( IndicatorBatchHandler.class );
 
376
        BatchHandler dataElementBatchHandler = batchHandlerFactory.createBatchHandler( DataElementBatchHandler.class );
 
377
        BatchHandler indicatorTypeBatchHandler = batchHandlerFactory.createBatchHandler( IndicatorTypeBatchHandler.class );
 
378
        
 
379
        RowHandler rowHandler = new IndicatorRowHandler( indicatorBatchHandler,
 
380
            importObjectService,
 
381
            indicatorService,
 
382
            objectMappingGenerator.getIndicatorTypeMapping( params.skipMapping() ), 
 
383
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ),
 
384
            categoryOptionComboService.getDefaultDataElementCategoryOptionCombo(),
 
385
            params,
 
386
            importAnalyser );
 
387
        
 
388
        indicatorBatchHandler.init();
 
389
        dataElementBatchHandler.init();
 
390
        indicatorTypeBatchHandler.init();
 
391
        
 
392
        queryManager.queryWithRowhandler( "getIndicators", rowHandler );
 
393
        
 
394
        indicatorBatchHandler.flush();
 
395
        dataElementBatchHandler.flush();
 
396
        indicatorTypeBatchHandler.flush();
 
397
        
 
398
        log.info( "Imported Indicators" );
 
399
    }
 
400
    
 
401
    private void importDataElementGroups( ImportParams params )
 
402
    {
 
403
        setMessage( "importing_data_element_groups" );
 
404
        
 
405
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataElementGroupBatchHandler.class );
 
406
        
 
407
        RowHandler rowHandler = new DataElementGroupRowHandler( batchHandler,
 
408
            importObjectService,
 
409
            dataElementService, 
 
410
            params );
 
411
        
 
412
        batchHandler.init();        
 
413
        
 
414
        queryManager.queryWithRowhandler( "getDataElementGroups", rowHandler );
 
415
        
 
416
        batchHandler.flush();        
 
417
        
 
418
        log.info( "Imported DataElementGroups" );
 
419
    }
 
420
    
 
421
    private void importIndicatorGroups( ImportParams params )
 
422
    {
 
423
        setMessage( "importing_indicator_groups" );
 
424
        
 
425
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( IndicatorGroupBatchHandler.class );
 
426
        
 
427
        RowHandler rowHandler = new IndicatorGroupRowHandler( batchHandler,
 
428
            importObjectService,
 
429
            indicatorService, 
 
430
            params );
 
431
        
 
432
        batchHandler.init();
 
433
        
 
434
        queryManager.queryWithRowhandler( "getIndicatorGroups", rowHandler );
 
435
        
 
436
        batchHandler.flush();
 
437
        
 
438
        log.info( "Imported IndicatorGroups" );
 
439
    }
 
440
    
 
441
    private void importDataElementGroupMembers( ImportParams params )
 
442
    {
 
443
        setMessage( "importing_data_element_group_members" );
 
444
        
 
445
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataElementGroupMemberBatchHandler.class );
 
446
        
 
447
        RowHandler rowHandler = new DataElementGroupMemberRowHandler( batchHandler,
 
448
            importObjectService,
 
449
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ),
 
450
            objectMappingGenerator.getDataElementGroupMapping( params.skipMapping() ),
 
451
            params );
 
452
        
 
453
        batchHandler.init();
 
454
        
 
455
        queryManager.queryWithRowhandler( "getDataElementGroupMembers", rowHandler );
 
456
        
 
457
        batchHandler.flush();
 
458
        
 
459
        log.info( "Imported DataElementGroup members" );
 
460
    }
 
461
 
 
462
    private void importIndicatorGroupMembers( ImportParams params )
 
463
    {
 
464
        setMessage( "importing_indicator_group_members" );
 
465
        
 
466
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( IndicatorGroupMemberBatchHandler.class );
 
467
        
 
468
        RowHandler rowHandler = new IndicatorGroupMemberRowHandler( batchHandler,
 
469
            importObjectService,
 
470
            objectMappingGenerator.getIndicatorMapping( params.skipMapping() ),
 
471
            objectMappingGenerator.getIndicatorGroupMapping( params.skipMapping() ),
 
472
            params );
 
473
        
 
474
        batchHandler.init();
 
475
        
 
476
        queryManager.queryWithRowhandler( "getIndicatorGroupMembers", rowHandler );
 
477
        
 
478
        batchHandler.flush();
 
479
        
 
480
        log.info( "Imported IndicatorGroup members" );
 
481
    }
 
482
    
 
483
    // -------------------------------------------------------------------------
 
484
    // DataSet
 
485
    // -------------------------------------------------------------------------
 
486
 
 
487
    private void importDataSets( ImportParams params )
 
488
    {
 
489
        setMessage( "importing_data_sets" );
 
490
        
 
491
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataSetBatchHandler.class );
 
492
        
 
493
        RowHandler rowHandler = new DataSetRowHandler( batchHandler,
 
494
            importObjectService,
 
495
            dataSetService,
 
496
            objectMappingGenerator.getPeriodTypeMapping(),
 
497
            params,
 
498
            importAnalyser );
 
499
        
 
500
        batchHandler.init();
 
501
        
 
502
        queryManager.queryWithRowhandler( "getDataSets", rowHandler );
 
503
        
 
504
        batchHandler.flush();
 
505
        
 
506
        log.info( "Imported DataSets" );
 
507
    }
 
508
    
 
509
    private void importDataSetMembers( ImportParams params )
 
510
    {
 
511
        setMessage( "importing_data_set_members" );
 
512
        
 
513
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataSetMemberBatchHandler.class );
 
514
        
 
515
        RowHandler rowHandler = new DataSetMemberRowHandler( batchHandler,
 
516
            importObjectService,
 
517
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ), 
 
518
            objectMappingGenerator.getDataSetMapping( params.skipMapping() ),
 
519
            params );
 
520
        
 
521
        batchHandler.init();
 
522
        
 
523
        queryManager.queryWithRowhandler( "getDataSetMembers", rowHandler );
 
524
        
 
525
        batchHandler.flush();
 
526
        
 
527
        log.info( "Imported DataSet members" );
 
528
    }
 
529
 
 
530
    // -------------------------------------------------------------------------
 
531
    // OrganisatonUnit
 
532
    // -------------------------------------------------------------------------
 
533
 
 
534
    private void importOrganisationUnits( ImportParams params )
 
535
    {
 
536
        setMessage( "importing_organisation_units" );
 
537
        
 
538
        BatchHandler sourceBatchHandler = batchHandlerFactory.createBatchHandler( SourceBatchHandler.class );
 
539
        BatchHandler organisationUnitBatchHandler = batchHandlerFactory.createBatchHandler( OrganisationUnitBatchHandler.class );
 
540
        
 
541
        RowHandler rowHandler = new OrganisationUnitRowHandler( organisationUnitBatchHandler, 
 
542
            sourceBatchHandler,
 
543
            importObjectService,
 
544
            organisationUnitService,
 
545
            params,
 
546
            importAnalyser );
 
547
        
 
548
        sourceBatchHandler.init();
 
549
        organisationUnitBatchHandler.init();
 
550
        
 
551
        queryManager.queryWithRowhandler( "getOrganisationUnits", rowHandler );
 
552
        
 
553
        sourceBatchHandler.flush();
 
554
        organisationUnitBatchHandler.flush();
 
555
        
 
556
        log.info( "Imported OrganisationUnits" );       
 
557
    }
 
558
    
 
559
    private void importOrganisationUnitGroups( ImportParams params )
 
560
    {
 
561
        setMessage( "importing_organisation_unit_groups" );
 
562
        
 
563
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( OrganisationUnitGroupBatchHandler.class );
 
564
        
 
565
        RowHandler rowHandler = new OrganisationUnitGroupRowHandler( batchHandler,
 
566
            importObjectService,
 
567
            organisationUnitGroupService,
 
568
            params );
 
569
        
 
570
        batchHandler.init();
 
571
        
 
572
        queryManager.queryWithRowhandler( "getOrganisationUnitGroups", rowHandler );
 
573
        
 
574
        batchHandler.flush();
 
575
        
 
576
        log.info( "Imported OrganisationUnitGroups" );
 
577
    }
 
578
    
 
579
    private void importOrganisationUnitGroupMembers( ImportParams params )
 
580
    {
 
581
        setMessage( "importing_organisation_unit_group_members" );
 
582
        
 
583
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( OrganisationUnitGroupMemberBatchHandler.class );
 
584
        
 
585
        RowHandler rowHandler = new OrganisationUnitGroupMemberRowHandler( batchHandler,
 
586
            importObjectService,
 
587
            objectMappingGenerator.getOrganisationUnitMapping( params.skipMapping() ),
 
588
            objectMappingGenerator.getOrganisationUnitGroupMapping( params.skipMapping() ),
 
589
            params );
 
590
        
 
591
        batchHandler.init();
 
592
        
 
593
        queryManager.queryWithRowhandler( "getOrganisationUnitGroupMembers", rowHandler );
 
594
        
 
595
        batchHandler.flush();
 
596
        
 
597
        log.info( "Imported OrganisationUnitGroup members" );
 
598
    }
 
599
    
 
600
    private void importGroupSets( ImportParams params )
 
601
    {
 
602
        setMessage( "importing_organisation_unit_group_sets" );
 
603
        
 
604
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GroupSetBatchHandler.class );
 
605
        
 
606
        RowHandler rowHandler = new GroupSetRowHandler( batchHandler, 
 
607
            importObjectService,
 
608
            organisationUnitGroupService,
 
609
            params );
 
610
        
 
611
        batchHandler.init();
 
612
        
 
613
        queryManager.queryWithRowhandler( "getOrganisationUnitGroupSets", rowHandler );
 
614
        
 
615
        batchHandler.flush();
 
616
        
 
617
        log.info( "Imported OrganisationUnitGroupSets" );  
 
618
    }
 
619
    
 
620
    private void importGroupSetMembers( ImportParams params )
 
621
    {
 
622
        setMessage( "importing_organisation_unit_group_set_members" );
 
623
        
 
624
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GroupSetMemberBatchHandler.class );
 
625
        
 
626
        RowHandler rowHandler = new GroupSetMemberRowHandler( batchHandler,
 
627
            importObjectService,
 
628
            objectMappingGenerator.getOrganisationUnitGroupMapping( params.skipMapping() ),
 
629
            objectMappingGenerator.getOrganisationUnitGroupSetMapping( params.skipMapping() ),
 
630
            params );
 
631
        
 
632
        batchHandler.init();
 
633
        
 
634
        queryManager.queryWithRowhandler( "getOrganisationUnitGroupSetMembers", rowHandler );
 
635
        
 
636
        batchHandler.flush();
 
637
                
 
638
        log.info( "Imported OrganisationUnitGroupSet members" );
 
639
    }
 
640
 
 
641
    private void importOrganisationUnitRelationships( ImportParams params )
 
642
    {
 
643
        setMessage( "importing_organisation_unit_relationships" );
 
644
        
 
645
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( OrganisationUnitBatchHandler.class );
 
646
        
 
647
        RowHandler rowHandler = new OrganisationUnitRelationshipRowHandler( batchHandler,
 
648
            importObjectService,
 
649
            organisationUnitService,
 
650
            objectMappingGenerator.getOrganisationUnitMapping( params.skipMapping() ),
 
651
            params );
 
652
        
 
653
        batchHandler.init();
 
654
        
 
655
        queryManager.queryWithRowhandler( "getOrganisationUnitRelationships", rowHandler );
 
656
        
 
657
        batchHandler.flush();
 
658
        
 
659
        log.info( "Imported OrganisationUnitRelationships" );
 
660
    }
 
661
    
 
662
    private void importOrganisationUnitHierarchy()
 
663
    {
 
664
        setMessage( "importing_organisation_unit_hierarchy" );
 
665
        
 
666
        organisationUnitService.addOrganisationUnitHierarchy( DateUtils.getEpoch() );
 
667
        
 
668
        log.info( "Imported OrganisationUnitHierarchy" );
 
669
    }
 
670
 
 
671
    // -------------------------------------------------------------------------
 
672
    // DataSet - OrganisationUnit Associations
 
673
    // -------------------------------------------------------------------------
 
674
 
 
675
    private void importDataSetOrganisationUnitAssociations( ImportParams params )
 
676
    {
 
677
        setMessage( "importing_data_set_organisation_unit_associations" );
 
678
        
 
679
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataSetSourceAssociationBatchHandler.class );
 
680
        
 
681
        RowHandler rowHandler = new DataSetOrganisationUnitAssociationRowHandler( batchHandler,
 
682
            importObjectService,
 
683
            objectMappingGenerator.getDataSetMapping( params.skipMapping() ), 
 
684
            objectMappingGenerator.getOrganisationUnitMapping( params.skipMapping() ),
 
685
            params );
 
686
        
 
687
        batchHandler.init();
 
688
        
 
689
        queryManager.queryWithRowhandler( "getDataSetOrganisationUnitAssociations", rowHandler );
 
690
        
 
691
        batchHandler.flush();
 
692
        
 
693
        log.info( "Imported DataSet OrganisationUnit Associations" );
 
694
    }
 
695
 
 
696
    // -------------------------------------------------------------------------
 
697
    // Period
 
698
    // -------------------------------------------------------------------------
 
699
 
 
700
    private void importPeriods( ImportParams params )
 
701
    {   
 
702
        setMessage( "importing_periods" );
 
703
        
 
704
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( PeriodBatchHandler.class );
 
705
        
 
706
        RowHandler rowHandler = new PeriodRowHandler( batchHandler, 
 
707
            importObjectService,
 
708
            periodService,
 
709
            objectMappingGenerator.getPeriodTypeMapping(),
 
710
            params,
 
711
            getPeriodWithDataIdentifiers() );
 
712
        
 
713
        batchHandler.init();
 
714
        
 
715
        queryManager.queryWithRowhandler( "getPeriods", rowHandler );
 
716
        
 
717
        batchHandler.flush();
 
718
        
 
719
        log.info( "Imported Periods" );
 
720
    }
 
721
    
 
722
    // -------------------------------------------------------------------------
 
723
    // RoutineDataValue
 
724
    // -------------------------------------------------------------------------
 
725
 
 
726
    private void importRoutineDataValues( ImportParams params )
 
727
    {
 
728
        setMessage( "importing_routine_data_values" );
 
729
 
 
730
        DataElementCategoryCombo categoryCombo = categoryComboService.getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME );
 
731
        
 
732
        DataElementCategoryOptionCombo categoryOptionCombo = categoryCombo.getOptionCombos().iterator().next();
 
733
        
 
734
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataValueBatchHandler.class );
 
735
        
 
736
        BatchHandler importDataValueBatchHandler = batchHandlerFactory.createBatchHandler( ImportDataValueBatchHandler.class );
 
737
                
 
738
        RowHandler rowHandler = new RoutineDataValueRowHandler( batchHandler,
 
739
            importDataValueBatchHandler,
 
740
            dataValueService,
 
741
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ),
 
742
            objectMappingGenerator.getPeriodMapping( params.skipMapping() ),
 
743
            objectMappingGenerator.getOrganisationUnitMapping( params.skipMapping() ),
 
744
            categoryOptionCombo,
 
745
            params );
 
746
        
 
747
        batchHandler.init();
 
748
        
 
749
        importDataValueBatchHandler.init();
 
750
        
 
751
        if ( params.getLastUpdated() == null )
 
752
        {
 
753
            queryManager.queryWithRowhandler( "getRoutineDataValues", rowHandler );
 
754
        }
 
755
        else
 
756
        {            
 
757
            queryManager.queryWithRowhandler( "getRoutineDataValuesLastUpdated", rowHandler, params.getLastUpdated() );
 
758
        }
 
759
        
 
760
        batchHandler.flush();
 
761
        
 
762
        importDataValueBatchHandler.flush();
 
763
        
 
764
        log.info( "Imported RoutineDataValues" );
 
765
    }
 
766
 
 
767
    // -------------------------------------------------------------------------
 
768
    // OnChangePeriod
 
769
    // -------------------------------------------------------------------------
 
770
 
 
771
    private void importOnChangePeriods( ImportParams params )
 
772
    {
 
773
        setMessage( "importing_on_change_periods" );
 
774
        
 
775
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( PeriodBatchHandler.class );
 
776
        
 
777
        RowHandler rowHandler = new OnChangePeriodRowHandler( batchHandler,
 
778
            importObjectService,
 
779
            periodService,
 
780
            objectMappingGenerator.getPeriodTypeMapping(),
 
781
            params );
 
782
        
 
783
        batchHandler.init();
 
784
        
 
785
        queryManager.queryWithRowhandler( "getOnChangePeriods", rowHandler );
 
786
        
 
787
        batchHandler.flush();
 
788
        
 
789
        log.info( "Imported OnChangePeriods" );
 
790
    }
 
791
 
 
792
    // -------------------------------------------------------------------------
 
793
    // SemiPermanentDataValue
 
794
    // -------------------------------------------------------------------------
 
795
 
 
796
    private void importSemiPermanentDataValues( ImportParams params )
 
797
    {
 
798
        setMessage( "importing_semi_permanent_data_values" );
 
799
 
 
800
        DataElementCategoryCombo categoryCombo = categoryComboService.getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME );
 
801
        
 
802
        DataElementCategoryOptionCombo categoryOptionCombo = categoryCombo.getOptionCombos().iterator().next();
 
803
        
 
804
        BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataValueBatchHandler.class );
 
805
 
 
806
        BatchHandler importDataValueBatchHandler = batchHandlerFactory.createBatchHandler( ImportDataValueBatchHandler.class );
 
807
        
 
808
        RowHandler rowHandler = new SemiPermanentDataValueRowHandler( batchHandler,
 
809
            importDataValueBatchHandler,
 
810
            dataValueService,
 
811
            objectMappingGenerator.getDataElementMapping( params.skipMapping() ),
 
812
            objectMappingGenerator.getPeriodObjectMapping( params.skipMapping() ),
 
813
            objectMappingGenerator.getOrganisationUnitMapping( params.skipMapping() ),
 
814
            categoryOptionCombo,
 
815
            params );
 
816
        
 
817
        batchHandler.init();
 
818
        
 
819
        importDataValueBatchHandler.init();
 
820
        
 
821
        if ( params.getLastUpdated() == null )
 
822
        {
 
823
            queryManager.queryWithRowhandler( "getSemiPermanentDataValues", rowHandler );
 
824
        }
 
825
        else
 
826
        {
 
827
            queryManager.queryWithRowhandler( "getSemiPermanentDataValuesLastUpdated", rowHandler, params.getLastUpdated() );
 
828
        }
 
829
        
 
830
        batchHandler.flush();
 
831
        
 
832
        importDataValueBatchHandler.flush();
 
833
        
 
834
        log.info( "Imported SemiPermanentDataValues" );
 
835
    }
 
836
 
 
837
    // -------------------------------------------------------------------------
 
838
    // Supportive methods
 
839
    // -------------------------------------------------------------------------
 
840
 
 
841
    /**
 
842
     * Creates a map where key is the calculated data element identifier and the
 
843
     * value if the formula.
 
844
     */
 
845
    private Map<Integer, String> createCalculatedDataElementEntryMap()
 
846
    {
 
847
        int categoryOptionComboId = categoryOptionComboService.getDefaultDataElementCategoryOptionCombo().getId();
 
848
        
 
849
        List<?> calculatedDataElements = queryManager.queryForList( "getCalculatedDataElementEntries", null );
 
850
        
 
851
        Map<Integer, String> map = new AppendingHashMap<Integer, String>(); // Calculated data element id, formula
 
852
        
 
853
        //TODO factor should be double
 
854
        
 
855
        for ( Object element : calculatedDataElements )
 
856
        {
 
857
            Dhis14CalculatedDataElementEntry calculated = (Dhis14CalculatedDataElementEntry) element;
 
858
            
 
859
            String formula = "([" + calculated.getDataElementId() + SEPARATOR + categoryOptionComboId + "]*" + calculated.getFactor() + ")";
 
860
            
 
861
            if ( map.containsKey( calculated.getCalculatedDataElementId() ) )
 
862
            {
 
863
                formula = "+" + formula;
 
864
            }
 
865
            
 
866
            map.put( calculated.getCalculatedDataElementId(), formula );
 
867
        }
 
868
        
 
869
        return map;
 
870
    }
 
871
 
 
872
    /**
 
873
     * Returns a list of distinct period identifiers from the RoutineDataValue table,
 
874
     * ie. periods which have registered data. Could be used to avoid importing 
 
875
     * periods without data.
 
876
     */
 
877
    private Set<Integer> getPeriodWithDataIdentifiers()
 
878
    {
 
879
        Set<Integer> identifiers = new HashSet<Integer>();
 
880
 
 
881
        List<?> list = queryManager.queryForList( "getDistinctPeriodIdentifiers", null );
 
882
        
 
883
        for ( Object id : list )
 
884
        {
 
885
            identifiers.add( (Integer) id );
 
886
        }
 
887
        
 
888
        return identifiers;
 
889
    }
 
890
    
 
891
    /**
 
892
     * Returns a mapping for category option combo. Since DHIS 1.4 does not have
 
893
     * this it will always be default.
 
894
     */
 
895
    private Map<Object, Integer> getCategoryOptionComboMapping()
 
896
    {
 
897
        Integer categoryOptionComboId = categoryOptionComboService.getDefaultDataElementCategoryOptionCombo().getId();
 
898
        
 
899
        Map<Object, Integer> mapping = new HashMap<Object, Integer>();
 
900
        
 
901
        mapping.put( categoryOptionComboId, categoryOptionComboId );
 
902
        
 
903
        return mapping;
 
904
    }
 
905
}