100
100
project_src(nir_builder *b, nir_tex_instr *tex)
102
/* Find the projector in the srcs list, if present. */
103
int proj_index = nir_tex_instr_src_index(tex, nir_tex_src_projector);
102
nir_ssa_def *proj = nir_steal_tex_src(tex, nir_tex_src_projector);
107
106
b->cursor = nir_before_instr(&tex->instr);
109
nir_ssa_def *inv_proj =
110
nir_frcp(b, nir_ssa_for_src(b, tex->src[proj_index].src, 1));
107
nir_ssa_def *inv_proj = nir_frcp(b, proj);
112
109
/* Walk through the sources projecting the arguments. */
113
110
for (unsigned i = 0; i < tex->num_srcs; i++) {
156
153
nir_src_for_ssa(projected));
159
nir_tex_instr_remove_src(tex, proj_index);
164
160
lower_offset(nir_builder *b, nir_tex_instr *tex)
166
int offset_index = nir_tex_instr_src_index(tex, nir_tex_src_offset);
167
if (offset_index < 0)
162
nir_ssa_def *offset = nir_steal_tex_src(tex, nir_tex_src_offset);
170
166
int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord);
171
167
assert(coord_index >= 0);
173
assert(tex->src[offset_index].src.is_ssa);
174
169
assert(tex->src[coord_index].src.is_ssa);
175
nir_ssa_def *offset = tex->src[offset_index].src.ssa;
176
170
nir_ssa_def *coord = tex->src[coord_index].src.ssa;
178
172
b->cursor = nir_before_instr(&tex->instr);
182
176
if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) {
183
177
offset_coord = nir_fadd(b, coord, nir_i2f32(b, offset));
185
nir_ssa_def *txs = nir_i2f32(b, nir_get_texture_size(b, tex));
186
nir_ssa_def *scale = nir_frcp(b, txs);
179
nir_ssa_def *scale = NULL;
181
if (b->shader->options->has_texture_scaling) {
182
nir_ssa_def *idx = nir_imm_int(b, tex->texture_index);
183
scale = nir_load_texture_scale(b, 32, idx);
185
nir_ssa_def *txs = nir_i2f32(b, nir_get_texture_size(b, tex));
186
scale = nir_frcp(b, txs);
188
189
offset_coord = nir_fadd(b, coord,
211
212
nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src,
212
213
nir_src_for_ssa(offset_coord));
214
nir_tex_instr_remove_src(tex, offset_index);
243
242
b->cursor = nir_before_instr(&tex->instr);
245
244
nir_ssa_def *idx = nir_imm_int(b, tex->texture_index);
246
nir_ssa_def *scale = nir_build_load_texture_rect_scaling(b, 32, idx);
245
nir_ssa_def *scale = nir_load_texture_scale(b, 32, idx);
247
246
int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord);
249
248
if (coord_index != -1) {
263
262
assert(nir_tex_instr_src_index(tex, nir_tex_src_ddx) < 0);
264
263
assert(nir_tex_instr_src_index(tex, nir_tex_src_ddy) < 0);
266
int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias);
268
/* If we have a bias, add it in */
269
lod = nir_fadd(b, lod, nir_ssa_for_src(b, tex->src[bias_idx].src, 1));
270
nir_tex_instr_remove_src(tex, bias_idx);
265
/* If we have a bias, add it in */
266
nir_ssa_def *bias = nir_steal_tex_src(tex, nir_tex_src_bias);
268
lod = nir_fadd(b, lod, bias);
273
int min_lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod);
274
if (min_lod_idx >= 0) {
275
/* If we have a minimum LOD, clamp LOD accordingly */
276
lod = nir_fmax(b, lod, nir_ssa_for_src(b, tex->src[min_lod_idx].src, 1));
277
nir_tex_instr_remove_src(tex, min_lod_idx);
270
/* If we have a minimum LOD, clamp LOD accordingly */
271
nir_ssa_def *min_lod = nir_steal_tex_src(tex, nir_tex_src_min_lod);
273
lod = nir_fmax(b, lod, min_lod);
280
275
nir_tex_instr_add_src(tex, nir_tex_src_lod, nir_src_for_ssa(lod));
281
276
tex->op = nir_texop_txl;
318
313
nir_src_copy(&plane_tex->src[i].src, &tex->src[i].src, &plane_tex->instr);
319
314
plane_tex->src[i].src_type = tex->src[i].src_type;
321
plane_tex->src[tex->num_srcs].src = nir_src_for_ssa(nir_imm_int(b, plane));
322
plane_tex->src[tex->num_srcs].src_type = nir_tex_src_plane;
316
plane_tex->src[tex->num_srcs] = nir_tex_src_for_ssa(nir_tex_src_plane,
317
nir_imm_int(b, plane));
323
318
plane_tex->op = nir_texop_tex;
324
319
plane_tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
325
320
plane_tex->dest_type = nir_type_float | nir_dest_bit_size(tex->dest);
329
324
plane_tex->sampler_index = tex->sampler_index;
331
326
nir_ssa_dest_init(&plane_tex->instr, &plane_tex->dest, 4,
332
nir_dest_bit_size(tex->dest), NULL);
327
nir_dest_bit_size(tex->dest));
334
329
nir_builder_instr_insert(b, &plane_tex->instr);
415
lower_y_vu_external(nir_builder *b, nir_tex_instr *tex,
416
const nir_lower_tex_options *options,
417
unsigned texture_index)
419
b->cursor = nir_after_instr(&tex->instr);
421
nir_ssa_def *y = sample_plane(b, tex, 0, options);
422
nir_ssa_def *vu = sample_plane(b, tex, 1, options);
424
convert_yuv_to_rgb(b, tex,
425
nir_channel(b, y, 0),
426
nir_channel(b, vu, 1),
427
nir_channel(b, vu, 0),
428
nir_imm_float(b, 1.0f),
420
434
lower_y_u_v_external(nir_builder *b, nir_tex_instr *tex,
421
435
const nir_lower_tex_options *options,
422
436
unsigned texture_index)
473
lower_yx_xvxu_external(nir_builder *b, nir_tex_instr *tex,
474
const nir_lower_tex_options *options,
475
unsigned texture_index)
477
b->cursor = nir_after_instr(&tex->instr);
479
nir_ssa_def *y = sample_plane(b, tex, 0, options);
480
nir_ssa_def *xvxu = sample_plane(b, tex, 1, options);
482
convert_yuv_to_rgb(b, tex,
483
nir_channel(b, y, 0),
484
nir_channel(b, xvxu, 3),
485
nir_channel(b, xvxu, 1),
486
nir_imm_float(b, 1.0f),
459
492
lower_xy_uxvx_external(nir_builder *b, nir_tex_instr *tex,
460
493
const nir_lower_tex_options *options,
461
494
unsigned texture_index)
511
lower_xy_vxux_external(nir_builder *b, nir_tex_instr *tex,
512
const nir_lower_tex_options *options,
513
unsigned texture_index)
515
b->cursor = nir_after_instr(&tex->instr);
517
nir_ssa_def *y = sample_plane(b, tex, 0, options);
518
nir_ssa_def *vxux = sample_plane(b, tex, 1, options);
520
convert_yuv_to_rgb(b, tex,
521
nir_channel(b, y, 1),
522
nir_channel(b, vxux, 2),
523
nir_channel(b, vxux, 0),
524
nir_imm_float(b, 1.0f),
478
530
lower_ayuv_external(nir_builder *b, nir_tex_instr *tex,
479
531
const nir_lower_tex_options *options,
480
532
unsigned texture_index)
620
lower_yv_yu_external(nir_builder *b, nir_tex_instr *tex,
621
const nir_lower_tex_options *options,
622
unsigned texture_index)
624
b->cursor = nir_after_instr(&tex->instr);
626
nir_ssa_def *yuv = sample_plane(b, tex, 0, options);
628
convert_yuv_to_rgb(b, tex,
629
nir_channel(b, yuv, 2),
630
nir_channel(b, yuv, 1),
631
nir_channel(b, yuv, 0),
632
nir_imm_float(b, 1.0f),
568
638
* Converts a nir_texop_txd instruction to nir_texop_txl with the given lod
569
639
* computed from the gradients.
576
646
nir_tex_instr_remove_src(tex, nir_tex_instr_src_index(tex, nir_tex_src_ddx));
577
647
nir_tex_instr_remove_src(tex, nir_tex_instr_src_index(tex, nir_tex_src_ddy));
579
int min_lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod);
580
if (min_lod_idx >= 0) {
581
/* If we have a minimum LOD, clamp LOD accordingly */
582
lod = nir_fmax(b, lod, nir_ssa_for_src(b, tex->src[min_lod_idx].src, 1));
583
nir_tex_instr_remove_src(tex, min_lod_idx);
649
/* If we have a minimum LOD, clamp LOD accordingly */
650
nir_ssa_def *min_lod = nir_steal_tex_src(tex, nir_tex_src_min_lod);
652
lod = nir_fmax(b, lod, min_lod);
586
654
nir_tex_instr_add_src(tex, nir_tex_src_lod, nir_src_for_ssa(lod));
587
655
tex->op = nir_texop_txl;
704
772
nir_ssa_def *rcp_Q_z = nir_frcp(b, nir_channel(b, Q, 2));
706
nir_ssa_def *Q_xy = nir_channels(b, Q, 0x3);
774
nir_ssa_def *Q_xy = nir_trim_vector(b, Q, 2);
707
775
nir_ssa_def *tmp = nir_fmul(b, Q_xy, rcp_Q_z);
709
nir_ssa_def *dQdx_xy = nir_channels(b, dQdx, 0x3);
777
nir_ssa_def *dQdx_xy = nir_trim_vector(b, dQdx, 2);
710
778
nir_ssa_def *dQdx_z = nir_channel(b, dQdx, 2);
711
779
nir_ssa_def *dx =
712
780
nir_fmul(b, rcp_Q_z, nir_fsub(b, dQdx_xy, nir_fmul(b, tmp, dQdx_z)));
714
nir_ssa_def *dQdy_xy = nir_channels(b, dQdy, 0x3);
782
nir_ssa_def *dQdy_xy = nir_trim_vector(b, dQdy, 2);
715
783
nir_ssa_def *dQdy_z = nir_channel(b, dQdy, 2);
716
784
nir_ssa_def *dy =
717
785
nir_fmul(b, rcp_Q_z, nir_fsub(b, dQdy_xy, nir_fmul(b, tmp, dQdy_z)));
819
887
assert(coord >= 0);
820
888
nir_ssa_def *dfdx = nir_fddx(b, tex->src[coord].src.ssa);
821
889
nir_ssa_def *dfdy = nir_fddy(b, tex->src[coord].src.ssa);
822
txd->src[tex->num_srcs].src = nir_src_for_ssa(dfdx);
823
txd->src[tex->num_srcs].src_type = nir_tex_src_ddx;
824
txd->src[tex->num_srcs + 1].src = nir_src_for_ssa(dfdy);
825
txd->src[tex->num_srcs + 1].src_type = nir_tex_src_ddy;
890
txd->src[tex->num_srcs] = nir_tex_src_for_ssa(nir_tex_src_ddx, dfdx);
891
txd->src[tex->num_srcs + 1] = nir_tex_src_for_ssa(nir_tex_src_ddy, dfdy);
827
nir_ssa_dest_init(&txd->instr, &txd->dest, nir_dest_num_components(tex->dest),
828
nir_dest_bit_size(tex->dest), NULL);
893
nir_ssa_dest_init(&txd->instr, &txd->dest,
894
nir_dest_num_components(tex->dest),
895
nir_dest_bit_size(tex->dest));
829
896
nir_builder_instr_insert(b, &txd->instr);
830
897
nir_ssa_def_rewrite_uses(&tex->dest.ssa, &txd->dest.ssa);
831
898
nir_instr_remove(&tex->instr);
861
928
int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias);
862
929
assert(bias_idx >= 0);
863
930
lod = nir_fadd(b, nir_channel(b, lod, 1), nir_ssa_for_src(b, tex->src[bias_idx].src, 1));
864
txl->src[tex->num_srcs - 1].src = nir_src_for_ssa(lod);
865
txl->src[tex->num_srcs - 1].src_type = nir_tex_src_lod;
931
txl->src[tex->num_srcs - 1] = nir_tex_src_for_ssa(nir_tex_src_lod, lod);
867
nir_ssa_dest_init(&txl->instr, &txl->dest, nir_dest_num_components(tex->dest),
868
nir_dest_bit_size(tex->dest), NULL);
933
nir_ssa_dest_init(&txl->instr, &txl->dest,
934
nir_dest_num_components(tex->dest),
935
nir_dest_bit_size(tex->dest));
869
936
nir_builder_instr_insert(b, &txl->instr);
870
937
nir_ssa_def_rewrite_uses(&tex->dest.ssa, &txl->dest.ssa);
871
938
nir_instr_remove(&tex->instr);
1010
1077
b->cursor = nir_after_instr(&tex->instr);
1012
1079
nir_ssa_def *rgb =
1013
nir_format_srgb_to_linear(b, nir_channels(b, &tex->dest.ssa, 0x7));
1080
nir_format_srgb_to_linear(b, nir_trim_vector(b, &tex->dest.ssa, 3));
1015
1082
/* alpha is untouched: */
1016
1083
nir_ssa_def *result = nir_vec4(b,
1104
lower_array_layer_round_even(nir_builder *b, nir_tex_instr *tex)
1106
int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord);
1107
if (coord_index < 0 || nir_tex_instr_src_type(tex, coord_index) != nir_type_float)
1110
assert(tex->src[coord_index].src.is_ssa);
1111
nir_ssa_def *coord = tex->src[coord_index].src.ssa;
1113
b->cursor = nir_before_instr(&tex->instr);
1115
unsigned layer = tex->coord_components - 1;
1116
nir_ssa_def *rounded_layer = nir_fround_even(b, nir_channel(b, coord, layer));
1117
nir_ssa_def *new_coord = nir_vector_insert_imm(b, coord, rounded_layer, layer);
1119
nir_instr_rewrite_src_ssa(&tex->instr, &tex->src[coord_index].src, new_coord);
1125
1171
sampler_index_lt(nir_tex_instr *tex, unsigned max)
1127
1173
assert(nir_tex_instr_src_index(tex, nir_tex_src_sampler_deref) == -1);
1160
1206
tex_copy->is_shadow = tex->is_shadow;
1161
1207
tex_copy->is_new_style_shadow = tex->is_new_style_shadow;
1162
1208
tex_copy->is_sparse = tex->is_sparse;
1209
tex_copy->is_gather_implicit_lod = tex->is_gather_implicit_lod;
1163
1210
tex_copy->component = tex->component;
1164
1211
tex_copy->dest_type = tex->dest_type;
1212
tex_copy->texture_index = tex->texture_index;
1213
tex_copy->sampler_index = tex->sampler_index;
1166
1215
for (unsigned j = 0; j < tex->num_srcs; ++j) {
1167
1216
nir_src_copy(&tex_copy->src[j].src, &tex->src[j].src, &tex_copy->instr);
1168
1217
tex_copy->src[j].src_type = tex->src[j].src_type;
1172
src.src = nir_src_for_ssa(nir_imm_ivec2(b, tex->tg4_offsets[i][0],
1173
tex->tg4_offsets[i][1]));
1174
src.src_type = nir_tex_src_offset;
1220
nir_ssa_def *offset = nir_imm_ivec2(b, tex->tg4_offsets[i][0],
1221
tex->tg4_offsets[i][1]);
1222
nir_tex_src src = nir_tex_src_for_ssa(nir_tex_src_offset, offset);
1175
1223
tex_copy->src[tex_copy->num_srcs - 1] = src;
1177
1225
nir_ssa_dest_init(&tex_copy->instr, &tex_copy->dest,
1178
nir_tex_instr_dest_size(tex), 32, NULL);
1226
nir_tex_instr_dest_size(tex), 32);
1180
1228
nir_builder_instr_insert(b, &tex_copy->instr);
1297
1345
fmask_fetch->is_array = tex->is_array;
1298
1346
fmask_fetch->texture_non_uniform = tex->texture_non_uniform;
1299
1347
fmask_fetch->dest_type = nir_type_uint32;
1300
nir_ssa_dest_init(&fmask_fetch->instr, &fmask_fetch->dest, 1, 32, NULL);
1348
nir_ssa_dest_init(&fmask_fetch->instr, &fmask_fetch->dest, 1, 32);
1302
1350
fmask_fetch->num_srcs = 0;
1303
1351
for (unsigned i = 0; i < tex->num_srcs; i++) {
1330
1378
nir_tex_instr *fmask_fetch = nir_instr_as_tex(nir_instr_clone(b->shader, &tex->instr));
1331
1379
fmask_fetch->op = nir_texop_fragment_mask_fetch_amd;
1332
1380
fmask_fetch->dest_type = nir_type_uint32;
1333
nir_ssa_dest_init(&fmask_fetch->instr, &fmask_fetch->dest, 1, 32, NULL);
1381
nir_ssa_dest_init(&fmask_fetch->instr, &fmask_fetch->dest, 1, 32);
1334
1382
nir_builder_instr_insert(b, &fmask_fetch->instr);
1336
1384
nir_ssa_def_rewrite_uses(&tex->dest.ssa, nir_ieq_imm(b, &fmask_fetch->dest.ssa, 0));
1346
1394
b->cursor = nir_after_instr(&tex->instr);
1348
nir_ssa_def *is_zero = nir_imm_bool(b, true);
1396
nir_ssa_def *is_zero = nir_imm_true(b);
1349
1397
for (unsigned i = 0; i < tex->coord_components; i++) {
1350
1398
nir_ssa_def *coord = nir_channel(b, tex->src[coord_index].src.ssa, i);
1355
1403
nir_ssa_def *fwidth = nir_fadd(b, nir_fabs(b, dfdx), nir_fabs(b, dfdy));
1357
1405
/* Check if the sum is 0. */
1358
is_zero = nir_iand(b, is_zero, nir_feq(b, fwidth, nir_imm_float(b, 0.0)));
1406
is_zero = nir_iand(b, is_zero, nir_feq_imm(b, fwidth, 0.0));
1361
1409
/* Replace the raw LOD by -FLT_MAX if the sum is 0 for all coordinates. */
1420
1468
/* mask of src coords to saturate (clamp): */
1421
1469
unsigned sat_mask = 0;
1423
if ((1 << tex->sampler_index) & options->saturate_r)
1424
sat_mask |= (1 << 2); /* .z */
1425
if ((1 << tex->sampler_index) & options->saturate_t)
1426
sat_mask |= (1 << 1); /* .y */
1427
if ((1 << tex->sampler_index) & options->saturate_s)
1428
sat_mask |= (1 << 0); /* .x */
1470
/* ignore saturate for txf ops: these don't use samplers and can't GL_CLAMP */
1471
if (nir_tex_instr_need_sampler(tex)) {
1472
if ((1 << tex->sampler_index) & options->saturate_r)
1473
sat_mask |= (1 << 2); /* .z */
1474
if ((1 << tex->sampler_index) & options->saturate_t)
1475
sat_mask |= (1 << 1); /* .y */
1476
if ((1 << tex->sampler_index) & options->saturate_s)
1477
sat_mask |= (1 << 0); /* .x */
1430
1480
if (options->lower_index_to_offset)
1431
1481
progress |= lower_index_to_offset(b, tex);
1451
1501
tex->op != nir_texop_txf) {
1452
1502
if (nir_tex_instr_is_query(tex))
1453
1503
tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
1454
else if (compiler_options->has_txs)
1504
else if (compiler_options->has_texture_scaling)
1505
lower_rect_tex_scale(b, tex);
1455
1507
lower_rect(b, tex);
1457
lower_rect_tex_scale(b, tex);
1459
1509
progress = true;
1474
1524
progress = true;
1527
if (texture_mask & options->lower_y_vu_external) {
1528
lower_y_vu_external(b, tex, options, texture_index);
1477
1532
if (texture_mask & options->lower_y_u_v_external) {
1478
1533
lower_y_u_v_external(b, tex, options, texture_index);
1479
1534
progress = true;
1484
1539
progress = true;
1542
if (texture_mask & options->lower_yx_xvxu_external) {
1543
lower_yx_xvxu_external(b, tex, options, texture_index);
1487
1547
if (texture_mask & options->lower_xy_uxvx_external) {
1488
1548
lower_xy_uxvx_external(b, tex, options, texture_index);
1489
1549
progress = true;
1552
if (texture_mask & options->lower_xy_vxux_external) {
1553
lower_xy_vxux_external(b, tex, options, texture_index);
1492
1557
if (texture_mask & options->lower_ayuv_external) {
1493
1558
lower_ayuv_external(b, tex, options, texture_index);
1494
1559
progress = true;
1509
1574
progress = true;
1577
if ((1 << tex->texture_index) & options->lower_yv_yu_external) {
1578
lower_yv_yu_external(b, tex, options, texture_index);
1512
1582
if ((1 << tex->texture_index) & options->lower_y41x_external) {
1513
1583
lower_y41x_external(b, tex, options, texture_index);
1514
1584
progress = true;
1558
1628
progress = true;
1561
if (options->lower_array_layer_round_even && tex->is_array &&
1562
tex->op != nir_texop_lod) {
1563
progress |= lower_array_layer_round_even(b, tex);
1566
1631
if (tex->op == nir_texop_txd &&
1567
1632
(options->lower_txd ||
1568
1633
(options->lower_txd_shadow && tex->is_shadow) ||
1657
1722
const struct nir_shader_compiler_options *compiler_options)
1659
1724
bool progress = false;
1660
nir_builder builder;
1661
nir_builder_init(&builder, impl);
1725
nir_builder builder = nir_builder_create(impl);
1663
1727
nir_foreach_block(block, impl) {
1664
1728
progress |= nir_lower_tex_block(block, &builder, options, compiler_options);
1675
1739
bool progress = false;
1677
nir_foreach_function(function, shader) {
1679
progress |= nir_lower_tex_impl(function->impl, options, shader->options);
1741
/* lower_tg4_offsets injects new tg4 instructions that won't be lowered
1742
* if lower_tg4_broadcom_swizzle is also requested so when both are set
1743
* we want to run lower_tg4_offsets in a separate pass first.
1745
if (options->lower_tg4_offsets && options->lower_tg4_broadcom_swizzle) {
1746
nir_lower_tex_options _options = {
1747
.lower_tg4_offsets = true,
1749
progress = nir_lower_tex(shader, &_options);
1752
nir_foreach_function_impl(impl, shader) {
1753
progress |= nir_lower_tex_impl(impl, options, shader->options);
1682
1756
return progress;
added
removed
src/compiler/nir/nir_lower_tex.c
