5
(generic((p)->op)==INDIR && (p)->kids[0]->op==VREG+P)
6
#define setsrc(d) ((d) && (d)->x.regnode && \
7
(d)->x.regnode->set == src->x.regnode->set && \
8
(d)->x.regnode->mask&src->x.regnode->mask)
10
#define relink(a, b) ((b)->x.prev = (a), (a)->x.next = (b))
12
static Symbol askfixedreg(Symbol);
13
static Symbol askreg(Symbol, unsigned*);
14
static void blkunroll(int, int, int, int, int, int, int[]);
15
static void docall(Node);
16
static void dumpcover(Node, int, int);
17
static void dumpregs(char *, char *, char *);
18
static void dumprule(int);
19
static void dumptree(Node);
20
static unsigned emitasm(Node, int);
21
static void genreload(Node, Symbol, int);
22
static void genspill(Symbol, Node, Symbol);
23
static Symbol getreg(Symbol, unsigned*, Node);
24
static int getrule(Node, int);
25
static void linearize(Node, Node);
26
static int moveself(Node);
27
static void prelabel(Node);
28
static Node* prune(Node, Node*);
29
static void putreg(Symbol);
30
static void ralloc(Node);
31
static void reduce(Node, int);
32
static int reprune(Node*, int, int, Node);
33
static int requate(Node);
34
static Node reuse(Node, int);
35
static void rewrite(Node);
36
static Symbol spillee(Symbol, unsigned mask[], Node);
37
static void spillr(Symbol, Node);
38
static int uses(Node, Regnode);
50
int bflag = 0; /* omit */
55
unsigned (*emitter)(Node, int) = emitasm;
56
static char NeedsReg[] = {
61
0, 0, 1, 1, /* - - CVF CVI */
62
1, 0, 1, 1, /* CVP - CVU NEG */
66
1, 1, 1, /* ADDRG ADDRF ADDRL */
67
1, 1, 1, 1, 1, /* ADD SUB LSH MOD RSH */
68
1, 1, 1, 1, /* BAND BCOM BOR BXOR */
70
0, 0, 0, 0, 0, 0, /* EQ GE GT LE LT NE */
79
Symbol mkreg(char *fmt, int n, int mask, int set) {
83
p->name = p->x.name = stringf(fmt, n);
84
NEW0(p->x.regnode, PERM);
85
p->x.regnode->number = n;
86
p->x.regnode->mask = mask<<n;
87
p->x.regnode->set = set;
90
Symbol mkwildcard(Symbol *syms) {
94
p->name = p->x.name = "wildcard";
98
void mkauto(Symbol p) {
99
assert(p->sclass == AUTO);
100
offset = roundup(offset + p->type->size, p->type->align);
101
p->x.offset = -offset;
102
p->x.name = stringd(-offset);
104
void blockbeg(Env *e) {
106
e->freemask[IREG] = freemask[IREG];
107
e->freemask[FREG] = freemask[FREG];
109
void blockend(Env *e) {
110
if (offset > maxoffset)
113
freemask[IREG] = e->freemask[IREG];
114
freemask[FREG] = e->freemask[FREG];
116
int mkactual(int align, int size) {
117
int n = roundup(argoffset, align);
119
argoffset = n + size;
122
static void docall(Node p) {
123
p->syms[1] = p->syms[0];
124
p->syms[0] = intconst(argoffset);
125
if (argoffset > maxargoffset)
126
maxargoffset = argoffset;
129
void blkcopy(int dreg, int doff, int sreg, int soff, int size, int tmp[]) {
134
blkunroll(size, dreg, doff, sreg, soff, size, tmp);
135
else if (size == 3) {
136
blkunroll(2, dreg, doff, sreg, soff, 2, tmp);
137
blkunroll(1, dreg, doff+2, sreg, soff+2, 1, tmp);
139
else if (size <= 16) {
140
blkunroll(4, dreg, doff, sreg, soff, size&~3, tmp);
141
blkcopy(dreg, doff+(size&~3),
142
sreg, soff+(size&~3), size&3, tmp);
145
(*IR->x.blkloop)(dreg, doff, sreg, soff, size, tmp);
147
static void blkunroll(int k, int dreg, int doff, int sreg, int soff, int size, int tmp[]) {
150
assert(IR->x.max_unaligned_load);
151
if (k > IR->x.max_unaligned_load
152
&& (k > salign || k > dalign))
153
k = IR->x.max_unaligned_load;
154
for (i = 0; i+k < size; i += 2*k) {
155
(*IR->x.blkfetch)(k, soff+i, sreg, tmp[0]);
156
(*IR->x.blkfetch)(k, soff+i+k, sreg, tmp[1]);
157
(*IR->x.blkstore)(k, doff+i, dreg, tmp[0]);
158
(*IR->x.blkstore)(k, doff+i+k, dreg, tmp[1]);
161
(*IR->x.blkfetch)(k, i+soff, sreg, tmp[0]);
162
(*IR->x.blkstore)(k, i+doff, dreg, tmp[0]);
165
void parseflags(int argc, char *argv[]) {
168
for (i = 0; i < argc; i++)
169
if (strcmp(argv[i], "-d") == 0)
171
else if (strcmp(argv[i], "-b") == 0) /* omit */
172
bflag = 1; /* omit */
174
static int getrule(Node p, int nt) {
178
rulenum = (*IR->x._rule)(p->x.state, nt);
180
fprint(stderr, "(%x->op=%s at %w is corrupt.)\n", p, opname(p->op), &src);
185
static void reduce(Node p, int nt) {
191
rulenum = getrule(p, nt);
192
nts = IR->x._nts[rulenum];
193
(*IR->x._kids)(p, rulenum, kids);
194
for (i = 0; nts[i]; i++)
195
reduce(kids[i], nts[i]);
196
if (IR->x._isinstruction[rulenum]) {
197
assert(p->x.inst == 0 || p->x.inst == nt);
199
if (p->syms[RX] && p->syms[RX]->temporary) {
200
debug(fprint(stderr, "(using %s)\n", p->syms[RX]->name));
201
p->syms[RX]->x.usecount++;
205
static Node reuse(Node p, int nt) {
209
Symbol r = p->syms[RX];
211
if (generic(p->op) == INDIR && p->kids[0]->op == VREG+P
212
&& r->u.t.cse && p->x.mayrecalc
213
&& ((struct _state*)r->u.t.cse->x.state)->cost[nt] == 0)
219
int mayrecalc(Node p) {
222
assert(p && p->syms[RX]);
223
if (p->syms[RX]->u.t.cse == NULL)
225
op = generic(p->syms[RX]->u.t.cse->op);
226
if (op == CNST || op == ADDRF || op == ADDRG || op == ADDRL) {
232
static Node *prune(Node p, Node pp[]) {
235
p->x.kids[0] = p->x.kids[1] = p->x.kids[2] = NULL;
237
return prune(p->kids[1], prune(p->kids[0], pp));
238
else if (p->syms[RX] && p->syms[RX]->temporary
239
&& p->syms[RX]->x.usecount < 2) {
241
debug(fprint(stderr, "(clobbering %s)\n", p->syms[RX]->name));
242
return prune(p->kids[1], prune(p->kids[0], pp));
245
prune(p->kids[1], prune(p->kids[0], &p->x.kids[0]));
251
#define ck(i) return (i) ? 0 : LBURG_MAX
253
int range(Node p, int lo, int hi) {
254
Symbol s = p->syms[0];
256
switch (specific(p->op)) {
258
case ADDRL+P: ck(s->x.offset >= lo && s->x.offset <= hi);
259
case CNST+I: ck(s->u.c.v.i >= lo && s->u.c.v.i <= hi);
260
case CNST+U: ck(s->u.c.v.u >= lo && s->u.c.v.u <= hi);
261
case CNST+P: ck(s->u.c.v.p == 0 && lo <= 0 && hi >= 0);
265
static void dumptree(Node p) {
266
if (p->op == VREG+P && p->syms[0]) {
267
fprint(stderr, "VREGP(%s)", p->syms[0]->name);
269
} else if (generic(p->op) == LOAD) {
270
fprint(stderr, "LOAD(");
271
dumptree(p->kids[0]);
275
fprint(stderr, "%s(", opname(p->op));
276
switch (generic(p->op)) {
277
case CNST: case LABEL:
278
case ADDRG: case ADDRF: case ADDRL:
280
fprint(stderr, "%s", p->syms[0]->name);
284
dumptree(p->kids[0]);
286
case CVF: case CVI: case CVP: case CVU: case JUMP:
287
case ARG: case BCOM: case NEG: case INDIR:
288
dumptree(p->kids[0]);
291
if (optype(p->op) != B) {
292
dumptree(p->kids[0]);
296
case EQ: case NE: case GT: case GE: case LE: case LT:
297
case ASGN: case BOR: case BAND: case BXOR: case RSH: case LSH:
298
case ADD: case SUB: case DIV: case MUL: case MOD:
299
dumptree(p->kids[0]);
300
fprint(stderr, ", ");
301
dumptree(p->kids[1]);
307
static void dumpcover(Node p, int nt, int in) {
313
rulenum = getrule(p, nt);
314
nts = IR->x._nts[rulenum];
315
fprint(stderr, "dumpcover(%x) = ", p);
316
for (i = 0; i < in; i++)
319
(*IR->x._kids)(p, rulenum, kids);
320
for (i = 0; nts[i]; i++)
321
dumpcover(kids[i], nts[i], in+1);
324
static void dumprule(int rulenum) {
326
fprint(stderr, "%s / %s", IR->x._string[rulenum],
327
IR->x._templates[rulenum]);
328
if (!IR->x._isinstruction[rulenum])
329
fprint(stderr, "\n");
331
static unsigned emitasm(Node p, int nt) {
338
rulenum = getrule(p, nt);
339
nts = IR->x._nts[rulenum];
340
fmt = IR->x._templates[rulenum];
342
if (IR->x._isinstruction[rulenum] && p->x.emitted)
343
print("%s", p->syms[RX]->x.name);
344
else if (*fmt == '#')
350
if (p->syms[RX] == p->x.kids[0]->syms[RX])
351
while (*fmt++ != '\n')
354
for ((*IR->x._kids)(p, rulenum, kids); *fmt; fmt++)
357
else if (*++fmt == 'F')
358
print("%d", framesize);
359
else if (*fmt >= '0' && *fmt <= '9')
360
emitasm(kids[*fmt - '0'], nts[*fmt - '0']);
361
else if (*fmt >= 'a' && *fmt < 'a' + NELEMS(p->syms))
362
fputs(p->syms[*fmt - 'a']->x.name, stdout);
369
for (; p; p = p->x.next) {
370
assert(p->x.registered);
371
if ((p->x.equatable && requate(p)) || moveself(p))
374
(*emitter)(p, p->x.inst);
378
static int moveself(Node p) {
380
&& p->syms[RX]->x.name == p->x.kids[0]->syms[RX]->x.name;
386
static int requate(Node q) {
387
Symbol src = q->x.kids[0]->syms[RX];
388
Symbol tmp = q->syms[RX];
392
debug(fprint(stderr, "(requate(%x): tmp=%s src=%s)\n", q, tmp->x.name, src->x.name));
393
for (p = q->x.next; p; p = p->x.next)
394
if (p->x.copy && p->syms[RX] == src
395
&& p->x.kids[0]->syms[RX] == tmp)
396
debug(fprint(stderr, "(requate arm 0 at %x)\n", p)),
398
else if (setsrc(p->syms[RX]) && !moveself(p) && !readsreg(p))
400
else if (p->x.spills)
402
else if (generic(p->op) == CALL && p->x.next)
404
else if (p->op == LABEL+V && p->x.next)
406
else if (p->syms[RX] == tmp && readsreg(p))
407
debug(fprint(stderr, "(requate arm 5 at %x)\n", p)),
409
else if (p->syms[RX] == tmp)
411
debug(fprint(stderr, "(requate arm 7 at %x)\n", p));
413
for (p = q->x.next; p; p = p->x.next)
414
if (p->syms[RX] == tmp && readsreg(p)) {
421
static void prelabel(Node p) {
424
prelabel(p->kids[0]);
425
prelabel(p->kids[1]);
426
if (NeedsReg[opindex(p->op)])
427
setreg(p, (*IR->x.rmap)(opkind(p->op)));
428
switch (generic(p->op)) {
429
case ADDRF: case ADDRL:
430
if (p->syms[0]->sclass == REGISTER)
434
if (p->kids[0]->op == VREG+P)
435
setreg(p, p->kids[0]->syms[0]);
438
if (p->kids[0]->op == VREG+P)
439
rtarget(p, 1, p->kids[0]->syms[0]);
441
case CVI: case CVU: case CVP:
442
if (optype(p->op) != F
443
&& opsize(p->op) <= p->syms[0]->u.c.v.i)
444
p->op = LOAD + opkind(p->op);
449
void setreg(Node p, Symbol r) {
452
void rtarget(Node p, int n, Symbol r) {
457
assert(r->sclass == REGISTER || !r->x.wildcard);
459
if (r != q->syms[RX] && !q->syms[RX]->x.wildcard) {
460
q = newnode(LOAD + opkind(q->op),
461
q, NULL, q->syms[0]);
462
if (r->u.t.cse == p->kids[n])
464
p->kids[n] = p->x.kids[n] = q;
465
q->x.kids[0] = q->kids[0];
468
debug(fprint(stderr, "(targeting %x->x.kids[%d]=%x to %s)\n", p, n, p->kids[n], r->x.name));
470
static void rewrite(Node p) {
471
assert(p->x.inst == 0);
474
debug(fprint(stderr, "\n"));
476
debug(dumpcover(p, 1, 0));
479
Node gen(Node forest) {
481
struct node sentinel;
485
for (p = forest; p; p = p->link) {
486
assert(p->count == 0);
487
if (generic(p->op) == CALL)
489
else if ( generic(p->op) == ASGN
490
&& generic(p->kids[1]->op) == CALL)
492
else if (generic(p->op) == ARG)
497
for (p = forest; p; p = p->link)
499
relink(&sentinel, &sentinel);
500
for (p = forest; p; p = p->link)
501
linearize(p, &sentinel);
502
forest = sentinel.x.next;
504
sentinel.x.next->x.prev = NULL;
505
sentinel.x.prev->x.next = NULL;
506
for (p = forest; p; p = p->x.next)
507
for (i = 0; i < NELEMS(p->x.kids) && p->x.kids[i]; i++) {
508
assert(p->x.kids[i]->syms[RX]);
509
if (p->x.kids[i]->syms[RX]->temporary) {
510
p->x.kids[i]->x.prevuse =
511
p->x.kids[i]->syms[RX]->x.lastuse;
512
p->x.kids[i]->syms[RX]->x.lastuse = p->x.kids[i];
515
for (p = forest; p; p = p->x.next) {
517
if (p->x.listed && NeedsReg[opindex(p->op)]
518
&& (*IR->x.rmap)(opkind(p->op))) {
519
assert(generic(p->op) == CALL || generic(p->op) == LOAD);
525
int notarget(Node p) {
526
return p->syms[RX]->x.wildcard ? 0 : LBURG_MAX;
528
static void putreg(Symbol r) {
529
assert(r && r->x.regnode);
530
freemask[r->x.regnode->set] |= r->x.regnode->mask;
531
debug(dumpregs("(freeing %s)\n", r->x.name, NULL));
533
static Symbol askfixedreg(Symbol s) {
534
Regnode r = s->x.regnode;
537
if (r->mask&~freemask[n])
540
freemask[n] &= ~r->mask;
541
usedmask[n] |= r->mask;
545
static Symbol askreg(Symbol rs, unsigned rmask[]) {
548
if (rs->x.wildcard == NULL)
549
return askfixedreg(rs);
550
for (i = 31; i >= 0; i--) {
551
Symbol r = rs->x.wildcard[i];
553
&& !(r->x.regnode->mask&~rmask[r->x.regnode->set])
560
static Symbol getreg(Symbol s, unsigned mask[], Node p) {
561
Symbol r = askreg(s, mask);
563
r = spillee(s, mask, p);
564
assert(r && r->x.regnode);
565
spill(r->x.regnode->mask, r->x.regnode->set, p);
568
assert(r && r->x.regnode);
569
r->x.regnode->vbl = NULL;
572
int askregvar(Symbol p, Symbol regs) {
576
if (p->sclass != REGISTER)
578
else if (!isscalar(p->type)) {
582
else if (p->temporary) {
586
else if ((r = askreg(regs, vmask)) != NULL) {
587
p->x.regnode = r->x.regnode;
588
p->x.regnode->vbl = p;
589
p->x.name = r->x.name;
590
debug(dumpregs("(allocating %s to symbol %s)\n", p->x.name, p->name));
598
static void linearize(Node p, Node next) {
601
for (i = 0; i < NELEMS(p->x.kids) && p->x.kids[i]; i++)
602
linearize(p->x.kids[i], next);
603
relink(next->x.prev, p);
605
debug(fprint(stderr, "(listing %x)\n", p));
607
static void ralloc(Node p) {
614
debug(fprint(stderr, "(rallocing %x)\n", p));
615
for (i = 0; i < NELEMS(p->x.kids) && p->x.kids[i]; i++) {
616
Node kid = p->x.kids[i];
617
Symbol r = kid->syms[RX];
618
assert(r && kid->x.registered);
619
if (r->sclass != REGISTER && r->x.lastuse == kid)
622
if (!p->x.registered && NeedsReg[opindex(p->op)]
623
&& (*IR->x.rmap)(opkind(p->op))) {
624
Symbol sym = p->syms[RX], set = sym;
627
set = (*IR->x.rmap)(opkind(p->op));
629
if (set->sclass != REGISTER) {
631
if (*IR->x._templates[getrule(p, p->x.inst)] == '?')
632
for (i = 1; i < NELEMS(p->x.kids) && p->x.kids[i]; i++) {
633
Symbol r = p->x.kids[i]->syms[RX];
634
assert(p->x.kids[i]->x.registered);
635
assert(r && r->x.regnode);
636
assert(sym->x.wildcard || sym != r);
637
mask[r->x.regnode->set] &= ~r->x.regnode->mask;
639
r = getreg(set, mask, p);
640
if (sym->temporary) {
642
r->x.lastuse = sym->x.lastuse;
643
for (q = sym->x.lastuse; q; q = q->x.prevuse) {
646
if (sym->u.t.cse && q->x.copy)
653
debug(dumpregs("(allocating %s to node %x)\n", r->x.name, (char *) p));
659
static Symbol spillee(Symbol set, unsigned mask[], Node here) {
660
Symbol bestreg = NULL;
661
int bestdist = -1, i;
664
if (!set->x.wildcard)
667
for (i = 31; i >= 0; i--) {
668
Symbol ri = set->x.wildcard[i];
672
(ri->x.regnode->mask&tmask[ri->x.regnode->set]&mask[ri->x.regnode->set])
674
Regnode rn = ri->x.regnode;
677
for (; q && !uses(q, rn); q = q->x.next)
679
if (q && dist > bestdist) {
686
assert(bestreg); /* Must be able to spill something. Reconfigure the register allocator
687
to ensure that we can allocate a register for all nodes without spilling
688
the node's necessary input regs. */
689
assert(bestreg->x.regnode->vbl == NULL); /* Can't spill register variables because
690
the reload site might be in other blocks. Reconfigure the register allocator
691
to ensure that this register is never allocated to a variable. */
694
static int uses(Node p, Regnode rn) {
697
for (i = 0; i < NELEMS(p->x.kids); i++)
700
p->x.kids[i]->x.registered &&
701
rn->set == p->x.kids[i]->syms[RX]->x.regnode->set &&
702
(rn->mask&p->x.kids[i]->syms[RX]->x.regnode->mask)
707
static void spillr(Symbol r, Node here) {
710
Node p = r->x.lastuse;
713
assert(r == p->syms[RX]),
715
assert(p->x.registered && !readsreg(p));
716
tmp = newtemp(AUTO, optype(p->op), opsize(p->op));
718
for (p = here->x.next; p; p = p->x.next)
719
for (i = 0; i < NELEMS(p->x.kids) && p->x.kids[i]; i++) {
720
Node k = p->x.kids[i];
721
if (k->x.registered && k->syms[RX] == r)
722
genreload(p, tmp, i);
726
static void genspill(Symbol r, Node last, Symbol tmp) {
731
debug(fprint(stderr, "(spilling %s to local %s)\n", r->x.name, tmp->x.name));
732
debug(fprint(stderr, "(genspill: "));
733
debug(dumptree(last));
734
debug(fprint(stderr, ")\n"));
735
ty = opkind(last->op);
737
s->sclass = REGISTER;
738
s->name = s->x.name = r->x.name;
739
s->x.regnode = r->x.regnode;
740
q = newnode(ADDRL+P + sizeop(IR->ptrmetric.size), NULL, NULL, s);
741
q = newnode(INDIR + ty, q, NULL, NULL);
742
p = newnode(ADDRL+P + sizeop(IR->ptrmetric.size), NULL, NULL, tmp);
743
p = newnode(ASGN + ty, p, q, NULL);
749
for (p = last->x.next; p != q; p = p->x.next) {
751
assert(!p->x.listed || !NeedsReg[opindex(p->op)] || !(*IR->x.rmap)(opkind(p->op)));
755
static void genreload(Node p, Symbol tmp, int i) {
759
debug(fprint(stderr, "(replacing %x with a reload from %s)\n", p->x.kids[i], tmp->x.name));
760
debug(fprint(stderr, "(genreload: "));
761
debug(dumptree(p->x.kids[i]));
762
debug(fprint(stderr, ")\n"));
763
ty = opkind(p->x.kids[i]->op);
764
q = newnode(ADDRL+P + sizeop(IR->ptrmetric.size), NULL, NULL, tmp);
765
p->x.kids[i] = newnode(INDIR + ty, q, NULL, NULL);
766
rewrite(p->x.kids[i]);
767
prune(p->x.kids[i], &q);
768
reprune(&p->kids[1], reprune(&p->kids[0], 0, i, p), i, p);
770
linearize(p->x.kids[i], p);
772
static int reprune(Node *pp, int k, int n, Node p) {
773
struct node x, *q = *pp;
775
if (q == NULL || k > n)
777
else if (q->x.inst == 0)
778
return reprune(&q->kids[1],
779
reprune(&q->kids[0], k, n, p), n, p);
781
debug(fprint(stderr, "(reprune changes %x from %x to %x)\n", pp, *pp, p->x.kids[n]));
788
void spill(unsigned mask, int n, Node here) {
794
if (mask&~freemask[n]) {
796
assert( /* It makes no sense for a node to clobber() its target. */
797
here->x.registered == 0 || /* call isn't coming through clobber() */
798
here->syms[RX] == NULL ||
799
here->syms[RX]->x.regnode == NULL ||
800
here->syms[RX]->x.regnode->set != n ||
801
(here->syms[RX]->x.regnode->mask&mask) == 0
804
for (p = here; p; p = p->x.next)
805
for (i = 0; i < NELEMS(p->x.kids) && p->x.kids[i]; i++) {
806
Symbol r = p->x.kids[i]->syms[RX];
808
if (p->x.kids[i]->x.registered && r->x.regnode->set == n
809
&& r->x.regnode->mask&mask)
814
static void dumpregs(char *msg, char *a, char *b) {
815
fprint(stderr, msg, a, b);
816
fprint(stderr, "(free[0]=%x)\n", freemask[0]);
817
fprint(stderr, "(free[1]=%x)\n", freemask[1]);
820
int getregnum(Node p) {
821
assert(p && p->syms[RX] && p->syms[RX]->x.regnode);
822
return p->syms[RX]->x.regnode->number;
826
unsigned regloc(Symbol p) {
827
assert(p && p->sclass == REGISTER && p->sclass == REGISTER && p->x.regnode);
828
return p->x.regnode->set<<8 | p->x.regnode->number;