← Back to branch summary

~gcrosswhite/charon/trunk

« back to all changes in this revision

Viewing changes to src/blacs_grid.h

  • Committer: Gregory Crosswhite
  • Date: 2009-04-14 20:37:52 UTC
  • Revision ID: gcross@phys.washington.edu-20090414203752-aayz9j7fqmrzccun
Cleared out all but the simplest and most solid functionality.

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/blacs_grid.h
1
 
//@+leo-ver=4-thin
2
 
//@+node:gcross.20081019105739.3:@thin blacs_grid.h
3
 
//@@language cplusplus
4
 
 
5
 
#ifndef BLACS_GRID_H
6
 
#define BLACS_GRID_H
7
 
 
8
 
#include <iostream> 
9
 
using namespace std;
10
 
 
11
 
#include "charm++.h"
12
 
#include "ampi.h"
13
 
#include "pup_cmialloc.h"
14
 
#include "tcharm.h"
15
 
#include "charon.h"
16
 
#include "C++BLAS/c++pblas.h"
17
 
#include "ampi_controller.h"
18
 
 
19
 
#include "blacs_grid.decl.h"
20
 
 
21
 
extern "C" {
22
 
    void BLACS_initialize_thread_variable();
23
 
    void BLACS_initialize();
24
 
}
25
 
 
26
 
//@+others
27
 
//@+node:gcross.20081019105739.12:BLACSGridSlave Declaration
28
 
class BLACSGridSlave : public AMPISlave {
29
 
 
30
 
    protected:
31
 
 
32
 
        int number_of_rows, number_of_columns;
33
 
 
34
 
    public:
35
 
        BLACSGridSlave(int number_of_rows_, int number_of_columns_) :
36
 
            AMPISlave(number_of_rows_*number_of_columns_),
37
 
            number_of_rows(number_of_rows_),
38
 
            number_of_columns(number_of_columns_)
39
 
        {
40
 
        }
41
 
        BLACSGridSlave(CkMigrateMessage* msg) : AMPISlave(msg) { }
42
 
 
43
 
        static BLACSGridSlave* getThread() { return dynamic_cast<BLACSGridSlave*>(AMPISlave::getThread()); }
44
 
 
45
 
        int blacs_context;
46
 
 
47
 
        virtual void initialize();
48
 
 
49
 
};
50
 
//@-node:gcross.20081019105739.12:BLACSGridSlave Declaration
51
 
//@+node:gcross.20081030170932.4:UpdateMatrixDescriptorOperation
52
 
template<typename T_numtype> class UpdateMatrixDescriptorOperation : public AbstractOperation<T_numtype,2> {
53
 
 
54
 
PUPable_decl_template(UpdateMatrixDescriptorOperation)
55
 
 
56
 
public:
57
 
 
58
 
    typedef AbstractOperation<T_numtype,2> BaseType;
59
 
    typedef DistributedArraySegment<T_numtype,2> ArraySegmentType;
60
 
 
61
 
    UpdateMatrixDescriptorOperation() { }
62
 
 
63
 
    UpdateMatrixDescriptorOperation(CkMigrateMessage* msg) { }
64
 
 
65
 
    virtual void perform_on(ArraySegmentType& segment) {
66
 
        segment.descriptor = descriptor;
67
 
        segment.descriptor.leading_dimension() = segment.rows();
68
 
    }
69
 
 
70
 
    virtual void pup(PUP::er &p) {
71
 
        BaseType::pup(p);
72
 
        p|descriptor;
73
 
    }
74
 
 
75
 
    Descriptor descriptor;
76
 
 
77
 
};
78
 
//@-node:gcross.20081030170932.4:UpdateMatrixDescriptorOperation
79
 
//@+node:gcross.20081019105739.4:BLACSGridMaster Declaration
80
 
class BLACSGridMaster : public AMPIMaster {
81
 
 
82
 
protected:
83
 
 
84
 
    friend class BLACSGridSlave;
85
 
    typedef TinyVector<int,2> Coordinates;
86
 
 
87
 
    const Coordinates PE_dimensions;
88
 
 
89
 
public:
90
 
 
91
 
    //CProxy_BLACSGridThreadArray thread_array_proxy;
92
 
 
93
 
    BLACSGridMaster(int number_of_rows, int number_of_columns);
94
 
 
95
 
    //@    << inner class DistributedMatrix >>
96
 
    //@+node:gcross.20081025093011.2:<< inner class DistributedMatrix >>
97
 
    template<typename T_numtype> class DistributedMatrix : public BlockCyclicArray<T_numtype,2> {
98
 
    public:
99
 
        typedef BlockCyclicArray<T_numtype,2> BaseType;
100
 
        BLACSGridMaster& parent;
101
 
        DistributedMatrix(BLACSGridMaster& parent_, int number_of_rows, int number_of_columns, int row_block_size, int column_block_size) :
102
 
            BaseType(parent_.PE_dimensions,Coordinates(number_of_rows,number_of_columns),Coordinates(row_block_size,column_block_size),fortranArray,parent_.thread_array_proxy),
103
 
            parent(parent_)
104
 
        { 
105
 
            UpdateMatrixDescriptorOperation<T_numtype> op;
106
 
            op.descriptor.type() = 1;
107
 
            op.descriptor.number_of_rows() = number_of_rows;
108
 
            op.descriptor.number_of_columns() = number_of_columns;
109
 
            op.descriptor.row_blocking_factor() = row_block_size;
110
 
            op.descriptor.column_blocking_factor() = column_block_size;
111
 
            op.descriptor.first_process_row() = 0;
112
 
            op.descriptor.first_process_column() = 0;
113
 
            broadcast_operation(op);
114
 
        }
115
 
 
116
 
        using BaseType::operator=;
117
 
    };
118
 
    //@-node:gcross.20081025093011.2:<< inner class DistributedMatrix >>
119
 
    //@nl
120
 
 
121
 
};
122
 
 
123
 
//@-node:gcross.20081019105739.4:BLACSGridMaster Declaration
124
 
//@+node:gcross.20081030231935.10:Operations
125
 
//@+node:gcross.20081030231935.6:AbstractDistributedMatrixOperation
126
 
template<typename T_numtype> class AbstractDistributedMatrixOperation : public DistributedArraySegmentOperation<T_numtype,2> {
127
 
 
128
 
protected:
129
 
 
130
 
    typedef DistributedArraySegmentOperation<T_numtype,2> BaseType;
131
 
    typedef DistributedArraySegment<T_numtype,2> ArraySegmentType;
132
 
 
133
 
    friend class PUP::able;
134
 
 
135
 
    AbstractDistributedMatrixOperation() { }
136
 
 
137
 
    AbstractDistributedMatrixOperation(int operation_id_, CProxy_BLACSGridSlave grid_proxy_, int number_of_arguments_, int argument_position_) :
138
 
        operation_id(operation_id_),
139
 
        grid_proxy(grid_proxy_),
140
 
        number_of_arguments(number_of_arguments_),
141
 
        argument_position(argument_position_)
142
 
    { }
143
 
 
144
 
    AbstractDistributedMatrixOperation(CkMigrateMessage* msg) { }
145
 
 
146
 
    CProxy_BLACSGridSlave grid_proxy;
147
 
    int operation_id;
148
 
    int number_of_arguments, argument_position;
149
 
 
150
 
    ArraySegmentType* segment;
151
 
 
152
 
    static void Function(vector<Operation*>& arguments) {
153
 
        for(int i = 0; i < arguments.size(); i ++) {
154
 
            dynamic_cast<AbstractDistributedMatrixOperation*>(arguments[i])->segment->descriptor.context() = BLACSGridSlave::getThread()->blacs_context;
155
 
        }
156
 
        dynamic_cast<AbstractDistributedMatrixOperation*>(arguments[0])->operate(arguments);
157
 
    }
158
 
 
159
 
    virtual void operate(vector<Operation*>& arguments) = 0;
160
 
 
161
 
public:
162
 
 
163
 
    virtual bool perform(ArraySegmentType& segment_) {
164
 
        segment = &segment_;
165
 
        BLACSGridSlave* thread = grid_proxy[segment_.thisIndex].ckLocal();
166
 
        CkAssert(thread != NULL);
167
 
        thread->perform_synchronized_operation(operation_id,number_of_arguments,argument_position,this,Function);
168
 
        return false;
169
 
    }
170
 
 
171
 
    virtual void pup(PUP::er &p) {
172
 
        BaseType::pup(p);
173
 
        p|grid_proxy;
174
 
        p|operation_id;
175
 
        p|number_of_arguments;
176
 
        p|argument_position;
177
 
    }
178
 
 
179
 
 
180
 
};
181
 
//@-node:gcross.20081030231935.6:AbstractDistributedMatrixOperation
182
 
//@+node:gcross.20081030231935.7:TransposeOperation
183
 
template<typename T_numtype> class TransposeOperation : public AbstractDistributedMatrixOperation<T_numtype> {
184
 
 
185
 
PUPable_decl_template(TransposeOperation)
186
 
 
187
 
protected:
188
 
 
189
 
    virtual void operate(vector<Operation*>& arguments) {
190
 
        CkAssert(arguments.size()==2);
191
 
        TransposeOperation& A = *dynamic_cast<TransposeOperation*>(arguments[0]);
192
 
        TransposeOperation& C = *dynamic_cast<TransposeOperation*>(arguments[1]);
193
 
        cout << "BEFORE" << endl;
194
 
        cout << "A=" << (*A.segment) << endl;
195
 
        cout << "C=" << (*C.segment) << endl;
196
 
        PBLAS::transpose(
197
 
            C.segment->descriptor.number_of_rows(),
198
 
            C.segment->descriptor.number_of_columns(),
199
 
            A.coefficient,A.segment->data(),1,1,A.segment->descriptor,
200
 
            C.coefficient,C.segment->data(),1,1,C.segment->descriptor
201
 
        );
202
 
        cout << "AFTER" << endl;
203
 
        cout << "A=" << (*A.segment) << endl;
204
 
        cout << "C=" << (*C.segment) << endl;
205
 
    }
206
 
 
207
 
    T_numtype coefficient;
208
 
 
209
 
public:
210
 
 
211
 
    typedef AbstractDistributedMatrixOperation<T_numtype> BaseType;
212
 
    typedef DistributedArraySegment<T_numtype,2> ArraySegmentType;
213
 
 
214
 
    TransposeOperation(int operation_id_, CProxy_BLACSGridSlave grid_proxy_,int argument_position_, T_numtype coefficient_) :
215
 
        BaseType(operation_id_,grid_proxy_,2,argument_position_),
216
 
        coefficient(coefficient_)
217
 
    { }
218
 
 
219
 
    TransposeOperation(CkMigrateMessage* msg) { }
220
 
 
221
 
 
222
 
    virtual void pup(PUP::er &p) {
223
 
        BaseType::pup(p);
224
 
        p|coefficient;
225
 
    }
226
 
 
227
 
};
228
 
//@-node:gcross.20081030231935.7:TransposeOperation
229
 
//@+node:gcross.20081102101609.2:SolveOperation
230
 
template<typename T_numtype> class SolveOperation : public AbstractDistributedMatrixOperation<T_numtype> {
231
 
 
232
 
PUPable_decl_template(SolveOperation)
233
 
 
234
 
protected:
235
 
 
236
 
    virtual void operate(vector<Operation*>& arguments) {
237
 
        CkAssert(arguments.size()==2);
238
 
        SolveOperation& A = *dynamic_cast<SolveOperation*>(arguments[0]);
239
 
        SolveOperation& B = *dynamic_cast<SolveOperation*>(arguments[1]);
240
 
        CkAssert(A.segment->descriptor.number_of_rows()==A.segment->descriptor.number_of_columns());
241
 
 
242
 
        int IPIV[100]; // just for testing!!!!
243
 
 
244
 
        int info = PBLAS::solve(
245
 
            A.segment->descriptor.number_of_rows(),
246
 
            B.segment->descriptor.number_of_columns(),
247
 
            A.segment->data(),1,1,A.segment->descriptor,
248
 
            IPIV,
249
 
            B.segment->data(),1,1,B.segment->descriptor
250
 
        );
251
 
 
252
 
    }
253
 
 
254
 
 
255
 
public:
256
 
 
257
 
    typedef AbstractDistributedMatrixOperation<T_numtype> BaseType;
258
 
    typedef DistributedArraySegment<T_numtype,2> ArraySegmentType;
259
 
 
260
 
    SolveOperation(int operation_id_, CProxy_BLACSGridSlave grid_proxy_,int argument_position_) :
261
 
        BaseType(operation_id_,grid_proxy_,2,argument_position_)
262
 
    { }
263
 
 
264
 
    SolveOperation(CkMigrateMessage* msg) { }
265
 
 
266
 
 
267
 
    virtual void pup(PUP::er &p) {
268
 
        BaseType::pup(p);
269
 
    }
270
 
 
271
 
};
272
 
//@-node:gcross.20081102101609.2:SolveOperation
273
 
//@-node:gcross.20081030231935.10:Operations
274
 
//@+node:gcross.20081030231935.8:Functions
275
 
//@+node:gcross.20081030231935.9:transpose
276
 
template<typename T> void transpose(BLACSGridMaster::DistributedMatrix<T>& A,BLACSGridMaster::DistributedMatrix<T>& C) {
277
 
    CkAssert(&A.parent==&C.parent);
278
 
 
279
 
    BLACSGridMaster& grid = A.parent;
280
 
 
281
 
    int operation_id = grid.allocate_next_operation_id();
282
 
 
283
 
    TransposeOperation<T> A_op(operation_id, grid.thread_array_proxy, 0, 1.0);
284
 
    A.broadcast_operation(A_op);
285
 
 
286
 
    TransposeOperation<T> C_op(operation_id, grid.thread_array_proxy, 1, 0.0);
287
 
    C.broadcast_operation(C_op);
288
 
}
289
 
//@-node:gcross.20081030231935.9:transpose
290
 
//@+node:gcross.20081102101609.4:solve
291
 
template<typename T> void solve(BLACSGridMaster::DistributedMatrix<T>& A,BLACSGridMaster::DistributedMatrix<T>& B) {
292
 
    CkAssert(&A.parent==&B.parent);
293
 
    CkAssert(A.rows()==A.columns());
294
 
    CkAssert(A.columns()==B.rows());
295
 
 
296
 
    BLACSGridMaster& grid = A.parent;
297
 
 
298
 
    int operation_id = grid.allocate_next_operation_id();
299
 
 
300
 
    SolveOperation<T> A_op(operation_id, static_cast<CProxy_BLACSGridSlave&>(grid.thread_array_proxy), 0);
301
 
    A.broadcast_operation(A_op);
302
 
 
303
 
    SolveOperation<T> B_op(operation_id, static_cast<CProxy_BLACSGridSlave&>(grid.thread_array_proxy), 1);
304
 
    B.broadcast_operation(B_op);
305
 
 
306
 
}
307
 
//@-node:gcross.20081102101609.4:solve
308
 
//@-node:gcross.20081030231935.8:Functions
309
 
//@-others
310
 
 
311
 
#endif // BLACS_GRID_H
312
 
//@-node:gcross.20081019105739.3:@thin blacs_grid.h
313
 
//@-leo