1
/**************************************************************************
3
* Copyright 2009 VMware, Inc.
4
* Copyright 2007 VMware, Inc.
7
* Permission is hereby granted, free of charge, to any person obtaining a
8
* copy of this software and associated documentation files (the
9
* "Software"), to deal in the Software without restriction, including
10
* without limitation the rights to use, copy, modify, merge, publish,
11
* distribute, sub license, and/or sell copies of the Software, and to
12
* permit persons to whom the Software is furnished to do so, subject to
13
* the following conditions:
15
* The above copyright notice and this permission notice (including the
16
* next paragraph) shall be included in all copies or substantial portions
19
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
23
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
**************************************************************************/
31
* Code generate the whole fragment pipeline.
33
* The fragment pipeline consists of the following stages:
37
* - depth/stencil test
40
* This file has only the glue to assemble the fragment pipeline. The actual
41
* plumbing of converting Gallium state into LLVM IR is done elsewhere, in the
42
* lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we
43
* muster the LLVM JIT execution engine to create a function that follows an
44
* established binary interface and that can be called from C directly.
46
* A big source of complexity here is that we often want to run different
47
* stages with different precisions and data types and precisions. For example,
48
* the fragment shader needs typically to be done in floats, but the
49
* depth/stencil test and blending is better done in the type that most closely
50
* matches the depth/stencil and color buffer respectively.
52
* Since the width of a SIMD vector register stays the same regardless of the
53
* element type, different types imply different number of elements, so we must
54
* code generate more instances of the stages with larger types to be able to
55
* feed/consume the stages with smaller types.
57
* @author Jose Fonseca <jfonseca@vmware.com>
61
#include "pipe/p_defines.h"
62
#include "util/u_inlines.h"
63
#include "util/u_memory.h"
64
#include "util/u_pointer.h"
65
#include "util/format/u_format.h"
66
#include "util/u_dump.h"
67
#include "util/u_string.h"
68
#include "util/simple_list.h"
69
#include "util/u_dual_blend.h"
70
#include "util/u_upload_mgr.h"
71
#include "util/os_time.h"
72
#include "pipe/p_shader_tokens.h"
73
#include "draw/draw_context.h"
74
#include "tgsi/tgsi_dump.h"
75
#include "tgsi/tgsi_scan.h"
76
#include "tgsi/tgsi_parse.h"
77
#include "gallivm/lp_bld_type.h"
78
#include "gallivm/lp_bld_const.h"
79
#include "gallivm/lp_bld_conv.h"
80
#include "gallivm/lp_bld_init.h"
81
#include "gallivm/lp_bld_intr.h"
82
#include "gallivm/lp_bld_logic.h"
83
#include "gallivm/lp_bld_tgsi.h"
84
#include "gallivm/lp_bld_nir.h"
85
#include "gallivm/lp_bld_swizzle.h"
86
#include "gallivm/lp_bld_flow.h"
87
#include "gallivm/lp_bld_debug.h"
88
#include "gallivm/lp_bld_arit.h"
89
#include "gallivm/lp_bld_bitarit.h"
90
#include "gallivm/lp_bld_pack.h"
91
#include "gallivm/lp_bld_format.h"
92
#include "gallivm/lp_bld_quad.h"
93
#include "gallivm/lp_bld_gather.h"
95
#include "lp_bld_alpha.h"
96
#include "lp_bld_blend.h"
97
#include "lp_bld_depth.h"
98
#include "lp_bld_interp.h"
99
#include "lp_context.h"
100
#include "lp_debug.h"
102
#include "lp_setup.h"
103
#include "lp_state.h"
104
#include "lp_tex_sample.h"
105
#include "lp_flush.h"
106
#include "lp_state_fs.h"
108
#include "nir/nir_to_tgsi_info.h"
110
#include "lp_screen.h"
111
#include "compiler/nir/nir_serialize.h"
112
#include "util/mesa-sha1.h"
113
/** Fragment shader number (for debugging) */
114
static unsigned fs_no = 0;
117
load_unswizzled_block(struct gallivm_state *gallivm,
118
LLVMValueRef base_ptr,
120
unsigned block_width,
121
unsigned block_height,
123
struct lp_type dst_type,
125
unsigned dst_alignment);
127
* Checks if a format description is an arithmetic format
129
* A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5.
131
static inline boolean
132
is_arithmetic_format(const struct util_format_description *format_desc)
134
boolean arith = false;
137
for (i = 0; i < format_desc->nr_channels; ++i) {
138
arith |= format_desc->channel[i].size != format_desc->channel[0].size;
139
arith |= (format_desc->channel[i].size % 8) != 0;
146
* Checks if this format requires special handling due to required expansion
147
* to floats for blending, and furthermore has "natural" packed AoS -> unpacked
150
static inline boolean
151
format_expands_to_float_soa(const struct util_format_description *format_desc)
153
if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT ||
154
format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
162
* Retrieves the type representing the memory layout for a format
164
* e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte
167
lp_mem_type_from_format_desc(const struct util_format_description *format_desc,
168
struct lp_type* type)
173
if (format_expands_to_float_soa(format_desc)) {
174
/* just make this a uint with width of block */
175
type->floating = false;
179
type->width = format_desc->block.bits;
184
for (i = 0; i < 4; i++)
185
if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
189
memset(type, 0, sizeof(struct lp_type));
190
type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
191
type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
192
type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
193
type->norm = format_desc->channel[chan].normalized;
195
if (is_arithmetic_format(format_desc)) {
199
for (i = 0; i < format_desc->nr_channels; ++i) {
200
type->width += format_desc->channel[i].size;
203
type->width = format_desc->channel[chan].size;
204
type->length = format_desc->nr_channels;
209
* Expand the relevant bits of mask_input to a n*4-dword mask for the
210
* n*four pixels in n 2x2 quads. This will set the n*four elements of the
211
* quad mask vector to 0 or ~0.
212
* Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid
213
* quad arguments with fs length 8.
215
* \param first_quad which quad(s) of the quad group to test, in [0,3]
216
* \param mask_input bitwise mask for the whole 4x4 stamp
219
generate_quad_mask(struct gallivm_state *gallivm,
220
struct lp_type fs_type,
223
LLVMValueRef mask_input) /* int64 */
225
LLVMBuilderRef builder = gallivm->builder;
226
struct lp_type mask_type;
227
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
228
LLVMValueRef bits[16];
229
LLVMValueRef mask, bits_vec;
233
* XXX: We'll need a different path for 16 x u8
235
assert(fs_type.width == 32);
236
assert(fs_type.length <= ARRAY_SIZE(bits));
237
mask_type = lp_int_type(fs_type);
240
* mask_input >>= (quad * 4)
242
switch (first_quad) {
247
assert(fs_type.length == 4);
254
assert(fs_type.length == 4);
262
mask_input = LLVMBuildLShr(builder, mask_input, lp_build_const_int64(gallivm, 16 * sample), "");
263
mask_input = LLVMBuildTrunc(builder, mask_input,
265
mask_input = LLVMBuildAnd(builder, mask_input, lp_build_const_int32(gallivm, 0xffff), "");
267
mask_input = LLVMBuildLShr(builder,
269
LLVMConstInt(i32t, shift, 0),
273
* mask = { mask_input & (1 << i), for i in [0,3] }
275
mask = lp_build_broadcast(gallivm,
276
lp_build_vec_type(gallivm, mask_type),
279
for (i = 0; i < fs_type.length / 4; i++) {
280
unsigned j = 2 * (i % 2) + (i / 2) * 8;
281
bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0);
282
bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0);
283
bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0);
284
bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0);
286
bits_vec = LLVMConstVector(bits, fs_type.length);
287
mask = LLVMBuildAnd(builder, mask, bits_vec, "");
290
* mask = mask == bits ? ~0 : 0
292
mask = lp_build_compare(gallivm,
293
mask_type, PIPE_FUNC_EQUAL,
300
#define EARLY_DEPTH_TEST 0x1
301
#define LATE_DEPTH_TEST 0x2
302
#define EARLY_DEPTH_WRITE 0x4
303
#define LATE_DEPTH_WRITE 0x8
304
#define EARLY_DEPTH_TEST_INFERRED 0x10 //only with EARLY_DEPTH_TEST
307
find_output_by_semantic( const struct tgsi_shader_info *info,
313
for (i = 0; i < info->num_outputs; i++)
314
if (info->output_semantic_name[i] == semantic &&
315
info->output_semantic_index[i] == index)
323
* Fetch the specified lp_jit_viewport structure for a given viewport_index.
326
lp_llvm_viewport(LLVMValueRef context_ptr,
327
struct gallivm_state *gallivm,
328
LLVMValueRef viewport_index)
330
LLVMBuilderRef builder = gallivm->builder;
333
struct lp_type viewport_type =
334
lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS);
336
ptr = lp_jit_context_viewports(gallivm, context_ptr);
337
ptr = LLVMBuildPointerCast(builder, ptr,
338
LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), "");
340
res = lp_build_pointer_get(builder, ptr, viewport_index);
347
lp_build_depth_clamp(struct gallivm_state *gallivm,
348
LLVMBuilderRef builder,
350
LLVMValueRef context_ptr,
351
LLVMValueRef thread_data_ptr,
354
LLVMValueRef viewport, min_depth, max_depth;
355
LLVMValueRef viewport_index;
356
struct lp_build_context f32_bld;
358
assert(type.floating);
359
lp_build_context_init(&f32_bld, gallivm, type);
362
* Assumes clamping of the viewport index will occur in setup/gs. Value
363
* is passed through the rasterization stage via lp_rast_shader_inputs.
365
* See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping
368
viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm,
372
* Load the min and max depth from the lp_jit_context.viewports
373
* array of lp_jit_viewport structures.
375
viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index);
377
/* viewports[viewport_index].min_depth */
378
min_depth = LLVMBuildExtractElement(builder, viewport,
379
lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), "");
380
min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth);
382
/* viewports[viewport_index].max_depth */
383
max_depth = LLVMBuildExtractElement(builder, viewport,
384
lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), "");
385
max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth);
388
* Clamp to the min and max depth values for the given viewport.
390
return lp_build_clamp(&f32_bld, z, min_depth, max_depth);
394
lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm,
396
unsigned coverage_samples,
397
LLVMValueRef num_loop,
398
LLVMValueRef loop_counter,
399
LLVMValueRef coverage_mask_store,
402
struct lp_build_context bld;
403
LLVMBuilderRef builder = gallivm->builder;
404
float step = 1.0 / coverage_samples;
406
lp_build_context_init(&bld, gallivm, type);
407
for (unsigned s = 0; s < coverage_samples; s++) {
408
LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s);
409
LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value);
411
LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, "");
412
s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, "");
413
LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, "");
414
LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
415
s_mask = LLVMBuildAnd(builder, s_mask, test, "");
416
LLVMBuildStore(builder, s_mask, s_mask_ptr);
420
struct lp_build_fs_llvm_iface {
421
struct lp_build_fs_iface base;
422
struct lp_build_interp_soa_context *interp;
423
struct lp_build_for_loop_state *loop_state;
424
LLVMValueRef mask_store;
425
LLVMValueRef sample_id;
426
LLVMValueRef color_ptr_ptr;
427
LLVMValueRef color_stride_ptr;
428
LLVMValueRef color_sample_stride_ptr;
429
const struct lp_fragment_shader_variant_key *key;
432
static LLVMValueRef fs_interp(const struct lp_build_fs_iface *iface,
433
struct lp_build_context *bld,
434
unsigned attrib, unsigned chan,
435
bool centroid, bool sample,
436
LLVMValueRef attrib_indir,
437
LLVMValueRef offsets[2])
439
struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
440
struct lp_build_interp_soa_context *interp = fs_iface->interp;
441
unsigned loc = TGSI_INTERPOLATE_LOC_CENTER;
443
loc = TGSI_INTERPOLATE_LOC_CENTROID;
445
loc = TGSI_INTERPOLATE_LOC_SAMPLE;
447
return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter,
448
fs_iface->mask_store,
449
attrib, chan, loc, attrib_indir, offsets);
452
static void fs_fb_fetch(const struct lp_build_fs_iface *iface,
453
struct lp_build_context *bld,
455
LLVMValueRef result[4])
457
assert(location >= FRAG_RESULT_DATA0 && location <= FRAG_RESULT_DATA7);
458
const int cbuf = location - FRAG_RESULT_DATA0;
460
struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
461
struct gallivm_state *gallivm = bld->gallivm;
462
LLVMBuilderRef builder = gallivm->builder;
463
const struct lp_fragment_shader_variant_key *key = fs_iface->key;
464
LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
465
LLVMValueRef color_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_ptr_ptr, &index, 1, ""), "");
466
LLVMValueRef stride = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_stride_ptr, &index, 1, ""), "");
468
enum pipe_format cbuf_format = key->cbuf_format[cbuf];
469
const struct util_format_description* out_format_desc = util_format_description(cbuf_format);
470
if (out_format_desc->format == PIPE_FORMAT_NONE) {
471
result[0] = result[1] = result[2] = result[3] = bld->undef;
475
unsigned block_size = bld->type.length;
476
unsigned block_height = key->resource_1d ? 1 : 2;
477
unsigned block_width = block_size / block_height;
479
if (key->multisample) {
480
LLVMValueRef sample_stride = LLVMBuildLoad(builder,
481
LLVMBuildGEP(builder, fs_iface->color_sample_stride_ptr,
483
LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, fs_iface->sample_id, "");
484
color_ptr = LLVMBuildGEP(builder, color_ptr, &sample_offset, 1, "");
486
/* fragment shader executes on 4x4 blocks. depending on vector width it can execute 2 or 4 iterations.
487
* only move to the next row once the top row has completed 8 wide 1 iteration, 4 wide 2 iterations */
488
LLVMValueRef x_offset = NULL, y_offset = NULL;
489
if (!key->resource_1d) {
490
LLVMValueRef counter = fs_iface->loop_state->counter;
492
if (block_size == 4) {
493
x_offset = LLVMBuildShl(builder,
494
LLVMBuildAnd(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""),
495
lp_build_const_int32(gallivm, 1), "");
496
counter = LLVMBuildLShr(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), "");
498
y_offset = LLVMBuildMul(builder, counter, lp_build_const_int32(gallivm, 2), "");
501
LLVMValueRef offsets[4 * 4];
502
for (unsigned i = 0; i < block_size; i++) {
503
unsigned x = i % block_width;
504
unsigned y = i / block_width;
506
if (block_size == 8) {
507
/* remap the raw slots into the fragment shader execution mode. */
508
/* this math took me way too long to work out, I'm sure it's overkill. */
509
x = (i & 1) + ((i >> 2) << 1);
510
if (!key->resource_1d)
516
x_val = LLVMBuildAdd(builder, lp_build_const_int32(gallivm, x), x_offset, "");
517
x_val = LLVMBuildMul(builder, x_val, lp_build_const_int32(gallivm, out_format_desc->block.bits / 8), "");
519
x_val = lp_build_const_int32(gallivm, x * (out_format_desc->block.bits / 8));
522
LLVMValueRef y_val = lp_build_const_int32(gallivm, y);
524
y_val = LLVMBuildAdd(builder, y_val, y_offset, "");
525
y_val = LLVMBuildMul(builder, y_val, stride, "");
527
offsets[i] = LLVMBuildAdd(builder, x_val, y_val, "");
529
LLVMValueRef offset = lp_build_gather_values(gallivm, offsets, block_size);
531
struct lp_type texel_type = bld->type;
532
if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
533
out_format_desc->channel[0].pure_integer) {
534
if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
535
texel_type = lp_type_int_vec(bld->type.width, bld->type.width * bld->type.length);
537
else if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
538
texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length);
542
lp_build_fetch_rgba_soa(gallivm, out_format_desc, texel_type,
543
true, color_ptr, offset,
544
NULL, NULL, NULL, result);
548
* Generate the fragment shader, depth/stencil test, and alpha tests.
551
generate_fs_loop(struct gallivm_state *gallivm,
552
struct lp_fragment_shader *shader,
553
const struct lp_fragment_shader_variant_key *key,
554
LLVMBuilderRef builder,
556
LLVMValueRef context_ptr,
557
LLVMValueRef sample_pos_array,
558
LLVMValueRef num_loop,
559
struct lp_build_interp_soa_context *interp,
560
const struct lp_build_sampler_soa *sampler,
561
const struct lp_build_image_soa *image,
562
LLVMValueRef mask_store,
563
LLVMValueRef (*out_color)[4],
564
LLVMValueRef depth_base_ptr,
565
LLVMValueRef depth_stride,
566
LLVMValueRef depth_sample_stride,
567
LLVMValueRef color_ptr_ptr,
568
LLVMValueRef color_stride_ptr,
569
LLVMValueRef color_sample_stride_ptr,
571
LLVMValueRef thread_data_ptr)
573
const struct util_format_description *zs_format_desc = NULL;
574
const struct tgsi_token *tokens = shader->base.tokens;
575
struct lp_type int_type = lp_int_type(type);
576
LLVMTypeRef vec_type, int_vec_type;
577
LLVMValueRef mask_ptr = NULL, mask_val = NULL;
578
LLVMValueRef consts_ptr, num_consts_ptr;
579
LLVMValueRef ssbo_ptr, num_ssbo_ptr;
581
LLVMValueRef z_value, s_value;
582
LLVMValueRef z_fb, s_fb;
583
LLVMValueRef depth_ptr;
584
LLVMValueRef stencil_refs[2];
585
LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
586
LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples);
587
LLVMValueRef z_out = NULL, s_out = NULL;
588
struct lp_build_for_loop_state loop_state, sample_loop_state = {0};
589
struct lp_build_mask_context mask;
591
* TODO: figure out if simple_shader optimization is really worthwile to
592
* keep. Disabled because it may hide some real bugs in the (depth/stencil)
593
* code since tests tend to take another codepath than real shaders.
595
boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 &&
596
shader->info.base.num_inputs < 3 &&
597
shader->info.base.num_instructions < 8) && 0;
598
const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
599
util_blend_state_is_dual(&key->blend, 0);
600
const bool post_depth_coverage = shader->info.base.properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE];
606
struct lp_bld_tgsi_system_values system_values;
608
memset(&system_values, 0, sizeof(system_values));
610
/* truncate then sign extend. */
611
system_values.front_facing = LLVMBuildTrunc(gallivm->builder, facing, LLVMInt1TypeInContext(gallivm->context), "");
612
system_values.front_facing = LLVMBuildSExt(gallivm->builder, system_values.front_facing, LLVMInt32TypeInContext(gallivm->context), "");
613
system_values.view_index = lp_jit_thread_data_raster_state_view_index(gallivm,
615
if (key->depth.enabled ||
616
key->stencil[0].enabled) {
618
zs_format_desc = util_format_description(key->zsbuf_format);
619
assert(zs_format_desc);
621
if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL])
622
depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE;
623
else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil &&
624
!shader->info.base.uses_fbfetch) {
625
if (shader->info.base.writes_memory)
626
depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
627
else if (key->alpha.enabled ||
628
key->blend.alpha_to_coverage ||
629
shader->info.base.uses_kill ||
630
shader->info.base.writes_samplemask) {
631
/* With alpha test and kill, can do the depth test early
632
* and hopefully eliminate some quads. But need to do a
633
* special deferred depth write once the final mask value
634
* is known. This only works though if there's either no
635
* stencil test or the stencil value isn't written.
637
if (key->stencil[0].enabled && (key->stencil[0].writemask ||
638
(key->stencil[1].enabled &&
639
key->stencil[1].writemask)))
640
depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
642
depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
645
depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
648
depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
651
if (!(key->depth.enabled && key->depth.writemask) &&
652
!(key->stencil[0].enabled && (key->stencil[0].writemask ||
653
(key->stencil[1].enabled &&
654
key->stencil[1].writemask))))
655
depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE);
661
vec_type = lp_build_vec_type(gallivm, type);
662
int_vec_type = lp_build_vec_type(gallivm, int_type);
664
stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr);
665
stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr);
666
/* convert scalar stencil refs into vectors */
667
stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]);
668
stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]);
670
consts_ptr = lp_jit_context_constants(gallivm, context_ptr);
671
num_consts_ptr = lp_jit_context_num_constants(gallivm, context_ptr);
673
ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr);
674
num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr);
676
memset(outputs, 0, sizeof outputs);
678
/* Allocate color storage for each fragment sample */
679
LLVMValueRef color_store_size = num_loop;
680
if (key->min_samples > 1)
681
color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), "");
683
for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
684
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
685
out_color[cbuf][chan] = lp_build_array_alloca(gallivm,
686
lp_build_vec_type(gallivm,
688
color_store_size, "color");
691
if (dual_source_blend) {
692
assert(key->nr_cbufs <= 1);
693
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
694
out_color[1][chan] = lp_build_array_alloca(gallivm,
695
lp_build_vec_type(gallivm,
697
color_store_size, "color1");
700
if (shader->info.base.writes_z) {
701
z_out = lp_build_array_alloca(gallivm,
702
lp_build_vec_type(gallivm, type),
703
color_store_size, "depth");
706
if (shader->info.base.writes_stencil) {
707
s_out = lp_build_array_alloca(gallivm,
708
lp_build_vec_type(gallivm, type),
709
color_store_size, "depth");
712
lp_build_for_loop_begin(&loop_state, gallivm,
713
lp_build_const_int32(gallivm, 0),
716
lp_build_const_int32(gallivm, 1));
718
LLVMValueRef sample_mask_in;
719
if (key->multisample) {
720
sample_mask_in = lp_build_const_int_vec(gallivm, type, 0);
721
/* create shader execution mask by combining all sample masks. */
722
for (unsigned s = 0; s < key->coverage_samples; s++) {
723
LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), "");
724
s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
725
LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx);
729
mask_val = LLVMBuildOr(builder, s_mask, mask_val, "");
731
LLVMValueRef mask_in = LLVMBuildAnd(builder, s_mask, lp_build_const_int_vec(gallivm, type, (1ll << s)), "");
732
sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
735
sample_mask_in = lp_build_const_int_vec(gallivm, type, 1);
736
mask_ptr = LLVMBuildGEP(builder, mask_store,
737
&loop_state.counter, 1, "mask_ptr");
738
mask_val = LLVMBuildLoad(builder, mask_ptr, "");
740
LLVMValueRef mask_in = LLVMBuildAnd(builder, mask_val, lp_build_const_int_vec(gallivm, type, 1), "");
741
sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
744
/* 'mask' will control execution based on quad's pixel alive/killed state */
745
lp_build_mask_begin(&mask, gallivm, type, mask_val);
747
if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader)
748
lp_build_mask_check(&mask);
750
/* Create storage for recombining sample masks after early Z pass. */
751
LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth");
752
LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or);
754
/* Create storage for post depth sample mask */
755
LLVMValueRef post_depth_sample_mask_in = NULL;
756
if (post_depth_coverage)
757
post_depth_sample_mask_in = lp_build_alloca(gallivm, int_vec_type, "post_depth_sample_mask_in");
759
LLVMValueRef s_mask = NULL, s_mask_ptr = NULL;
760
LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL;
761
LLVMValueRef z_fb_store = NULL, s_fb_store = NULL;
762
LLVMTypeRef z_type = NULL, z_fb_type = NULL;
764
/* Run early depth once per sample */
765
if (key->multisample) {
767
if (zs_format_desc) {
768
struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length);
769
struct lp_type z_type = zs_type;
770
struct lp_type s_type = zs_type;
771
if (zs_format_desc->block.bits < type.width)
772
z_type.width = type.width;
773
if (zs_format_desc->block.bits == 8)
774
s_type.width = type.width;
776
else if (zs_format_desc->block.bits > 32) {
777
z_type.width = z_type.width / 2;
778
s_type.width = s_type.width / 2;
781
z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
782
zs_samples, "z_sample_store");
783
s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
784
zs_samples, "s_sample_store");
785
z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type),
786
zs_samples, "z_fb_store");
787
s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type),
788
zs_samples, "s_fb_store");
790
lp_build_for_loop_begin(&sample_loop_state, gallivm,
791
lp_build_const_int32(gallivm, 0),
792
LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
793
lp_build_const_int32(gallivm, 1));
795
LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
796
s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
797
s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
799
s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
800
s_mask = LLVMBuildAnd(builder, s_mask, mask_val, "");
804
/* for multisample Z needs to be interpolated at sample points for testing. */
805
lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
808
depth_ptr = depth_base_ptr;
809
if (key->multisample) {
810
LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
811
depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
814
if (depth_mode & EARLY_DEPTH_TEST) {
816
* Clamp according to ARB_depth_clamp semantics.
818
if (key->depth_clamp) {
819
z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
822
lp_build_depth_stencil_load_swizzled(gallivm, type,
823
zs_format_desc, key->resource_1d,
824
depth_ptr, depth_stride,
825
&z_fb, &s_fb, loop_state.counter);
826
lp_build_depth_stencil_test(gallivm,
831
key->multisample ? NULL : &mask,
837
!simple_shader && !key->multisample);
839
if (depth_mode & EARLY_DEPTH_WRITE) {
840
lp_build_depth_stencil_write_swizzled(gallivm, type,
841
zs_format_desc, key->resource_1d,
842
NULL, NULL, NULL, loop_state.counter,
843
depth_ptr, depth_stride,
847
* Note mask check if stencil is enabled must be after ds write not after
848
* stencil test otherwise new stencil values may not get written if all
849
* fragments got killed by depth/stencil test.
851
if (!simple_shader && key->stencil[0].enabled && !key->multisample)
852
lp_build_mask_check(&mask);
854
if (key->multisample) {
855
z_fb_type = LLVMTypeOf(z_fb);
856
z_type = LLVMTypeOf(z_value);
857
lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), ""));
858
lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), ""));
859
lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb);
860
lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb);
864
if (key->multisample) {
866
* Store the post-early Z coverage mask.
867
* Recombine the resulting coverage masks post early Z into the fragment
868
* shader execution mask.
870
LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
871
tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, "");
872
LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or);
874
if (post_depth_coverage) {
875
LLVMValueRef mask_bit_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
876
LLVMValueRef post_depth_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
877
mask_bit_idx = LLVMBuildAnd(builder, s_mask, lp_build_broadcast(gallivm, int_vec_type, mask_bit_idx), "");
878
post_depth_mask_in = LLVMBuildOr(builder, post_depth_mask_in, mask_bit_idx, "");
879
LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
882
LLVMBuildStore(builder, s_mask, s_mask_ptr);
884
lp_build_for_loop_end(&sample_loop_state);
886
/* recombined all the coverage masks in the shader exec mask. */
887
tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
888
lp_build_mask_update(&mask, tmp_s_mask_or);
890
if (key->min_samples == 1) {
891
/* for multisample Z needs to be re interpolated at pixel center */
892
lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL);
894
lp_build_mask_update(&mask, tmp_s_mask_or);
897
if (post_depth_coverage) {
898
LLVMValueRef post_depth_mask_in = LLVMBuildAnd(builder, lp_build_mask_value(&mask), lp_build_const_int_vec(gallivm, type, 1), "");
899
LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
903
LLVMValueRef out_sample_mask_storage = NULL;
904
if (shader->info.base.writes_samplemask) {
905
out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask");
906
if (key->min_samples > 1)
907
LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage);
910
if (post_depth_coverage) {
911
system_values.sample_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
914
system_values.sample_mask_in = sample_mask_in;
915
if (key->multisample && key->min_samples > 1) {
916
lp_build_for_loop_begin(&sample_loop_state, gallivm,
917
lp_build_const_int32(gallivm, 0),
919
lp_build_const_int32(gallivm, key->min_samples),
920
lp_build_const_int32(gallivm, 1));
922
LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
923
s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
924
s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
925
s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
926
lp_build_mask_force(&mask, s_mask);
927
lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter);
928
system_values.sample_id = sample_loop_state.counter;
929
system_values.sample_mask_in = LLVMBuildAnd(builder, system_values.sample_mask_in,
930
lp_build_broadcast(gallivm, int_vec_type,
931
LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "")), "");
933
system_values.sample_id = lp_build_const_int32(gallivm, 0);
936
system_values.sample_pos = sample_pos_array;
938
lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter);
940
struct lp_build_fs_llvm_iface fs_iface = {
941
.base.interp_fn = fs_interp,
942
.base.fb_fetch = fs_fb_fetch,
944
.loop_state = &loop_state,
945
.sample_id = system_values.sample_id,
946
.mask_store = mask_store,
947
.color_ptr_ptr = color_ptr_ptr,
948
.color_stride_ptr = color_stride_ptr,
949
.color_sample_stride_ptr = color_sample_stride_ptr,
953
struct lp_build_tgsi_params params;
954
memset(¶ms, 0, sizeof(params));
958
params.fs_iface = &fs_iface.base;
959
params.consts_ptr = consts_ptr;
960
params.const_sizes_ptr = num_consts_ptr;
961
params.system_values = &system_values;
962
params.inputs = interp->inputs;
963
params.context_ptr = context_ptr;
964
params.thread_data_ptr = thread_data_ptr;
965
params.sampler = sampler;
966
params.info = &shader->info.base;
967
params.ssbo_ptr = ssbo_ptr;
968
params.ssbo_sizes_ptr = num_ssbo_ptr;
969
params.image = image;
970
params.aniso_filter_table = lp_jit_context_aniso_filter_table(gallivm, context_ptr);
972
/* Build the actual shader */
973
if (shader->base.type == PIPE_SHADER_IR_TGSI)
974
lp_build_tgsi_soa(gallivm, tokens, ¶ms,
977
lp_build_nir_soa(gallivm, shader->base.ir.nir, ¶ms,
981
if (key->alpha.enabled) {
982
int color0 = find_output_by_semantic(&shader->info.base,
986
if (color0 != -1 && outputs[color0][3]) {
987
const struct util_format_description *cbuf_format_desc;
988
LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
989
LLVMValueRef alpha_ref_value;
991
alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr);
992
alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value);
994
cbuf_format_desc = util_format_description(key->cbuf_format[0]);
996
lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc,
997
&mask, alpha, alpha_ref_value,
998
((depth_mode & LATE_DEPTH_TEST) != 0) && !key->multisample);
1002
/* Emulate Alpha to Coverage with Alpha test */
1003
if (key->blend.alpha_to_coverage) {
1004
int color0 = find_output_by_semantic(&shader->info.base,
1005
TGSI_SEMANTIC_COLOR,
1008
if (color0 != -1 && outputs[color0][3]) {
1009
LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
1011
if (!key->multisample) {
1012
lp_build_alpha_to_coverage(gallivm, type,
1014
(depth_mode & LATE_DEPTH_TEST) != 0);
1016
lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop,
1022
if (key->blend.alpha_to_one && key->multisample) {
1023
for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) {
1024
unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1025
if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
1026
((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
1027
if (outputs[cbuf][3]) {
1028
LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0), outputs[cbuf][3]);
1032
if (shader->info.base.writes_samplemask) {
1033
LLVMValueRef output_smask = NULL;
1034
int smaski = find_output_by_semantic(&shader->info.base,
1035
TGSI_SEMANTIC_SAMPLEMASK,
1037
struct lp_build_context smask_bld;
1038
lp_build_context_init(&smask_bld, gallivm, int_type);
1040
assert(smaski >= 0);
1041
output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask");
1042
output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, "");
1043
if (!key->multisample && key->no_ms_sample_mask_out) {
1044
output_smask = lp_build_and(&smask_bld, output_smask, smask_bld.one);
1045
output_smask = lp_build_cmp(&smask_bld, PIPE_FUNC_NOTEQUAL, output_smask, smask_bld.zero);
1046
lp_build_mask_update(&mask, output_smask);
1049
if (key->min_samples > 1) {
1050
/* only the bit corresponding to this sample is to be used. */
1051
LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask");
1052
LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1053
LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), "");
1054
output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, "");
1057
LLVMBuildStore(builder, output_smask, out_sample_mask_storage);
1060
if (shader->info.base.writes_z) {
1061
int pos0 = find_output_by_semantic(&shader->info.base,
1062
TGSI_SEMANTIC_POSITION,
1064
LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos0][2], "");
1065
LLVMValueRef idx = loop_state.counter;
1066
if (key->min_samples > 1)
1067
idx = LLVMBuildAdd(builder, idx,
1068
LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1069
LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1070
LLVMBuildStore(builder, out, ptr);
1073
if (shader->info.base.writes_stencil) {
1074
int sten_out = find_output_by_semantic(&shader->info.base,
1075
TGSI_SEMANTIC_STENCIL,
1077
LLVMValueRef out = LLVMBuildLoad(builder, outputs[sten_out][1], "output.s");
1078
LLVMValueRef idx = loop_state.counter;
1079
if (key->min_samples > 1)
1080
idx = LLVMBuildAdd(builder, idx,
1081
LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1082
LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1083
LLVMBuildStore(builder, out, ptr);
1086
bool has_cbuf0_write = false;
1087
/* Color write - per fragment sample */
1088
for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib)
1090
unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1091
if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
1092
((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
1094
if (cbuf == 0 && shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]) {
1095
/* XXX: there is an edge case with FB fetch where gl_FragColor and gl_LastFragData[0]
1096
* are used together. This creates both FRAG_RESULT_COLOR and FRAG_RESULT_DATA* output
1097
* variables. This loop then writes to cbuf 0 twice, owerwriting the correct value
1098
* from gl_FragColor with some garbage. This case is excercised in one of deqp tests.
1099
* A similar bug can happen if gl_SecondaryFragColorEXT and gl_LastFragData[1]
1100
* are mixed in the same fashion...
1101
* This workaround will break if gl_LastFragData[0] goes in outputs list before
1102
* gl_FragColor. This doesn't seem to happen though.
1104
if (has_cbuf0_write)
1106
has_cbuf0_write = true;
1109
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1110
if(outputs[attrib][chan]) {
1111
/* XXX: just initialize outputs to point at colors[] and
1114
LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
1115
LLVMValueRef color_ptr;
1116
LLVMValueRef color_idx = loop_state.counter;
1117
if (key->min_samples > 1)
1118
color_idx = LLVMBuildAdd(builder, color_idx,
1119
LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1120
color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan],
1122
lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]);
1123
LLVMBuildStore(builder, out, color_ptr);
1129
if (key->multisample && key->min_samples > 1) {
1130
LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr);
1131
lp_build_for_loop_end(&sample_loop_state);
1134
if (key->multisample) {
1135
/* execute depth test for each sample */
1136
lp_build_for_loop_begin(&sample_loop_state, gallivm,
1137
lp_build_const_int32(gallivm, 0),
1138
LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
1139
lp_build_const_int32(gallivm, 1));
1141
/* load the per-sample coverage mask */
1142
LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
1143
s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
1144
s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
1146
/* combine the execution mask post fragment shader with the coverage mask. */
1147
s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
1148
if (key->min_samples == 1)
1149
s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), "");
1151
/* if the shader writes sample mask use that,
1152
* but only if this isn't genuine early-depth to avoid breaking occlusion query */
1153
if (shader->info.base.writes_samplemask &&
1154
(!(depth_mode & EARLY_DEPTH_TEST) || (depth_mode & (EARLY_DEPTH_TEST_INFERRED)))) {
1155
LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1156
out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1157
LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1158
LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1159
LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1160
smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1162
s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1166
depth_ptr = depth_base_ptr;
1167
if (key->multisample) {
1168
LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
1169
depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
1173
if (depth_mode & LATE_DEPTH_TEST) {
1174
if (shader->info.base.writes_z) {
1175
LLVMValueRef idx = loop_state.counter;
1176
if (key->min_samples > 1)
1177
idx = LLVMBuildAdd(builder, idx,
1178
LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1179
LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1180
z = LLVMBuildLoad(builder, ptr, "output.z");
1182
if (key->multisample) {
1183
lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
1189
* Clamp according to ARB_depth_clamp semantics.
1191
if (key->depth_clamp) {
1192
z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
1193
thread_data_ptr, z);
1195
struct lp_build_context f32_bld;
1196
lp_build_context_init(&f32_bld, gallivm, type);
1197
z = lp_build_clamp(&f32_bld, z,
1198
lp_build_const_vec(gallivm, type, 0.0),
1199
lp_build_const_vec(gallivm, type, 1.0));
1202
if (shader->info.base.writes_stencil) {
1203
LLVMValueRef idx = loop_state.counter;
1204
if (key->min_samples > 1)
1205
idx = LLVMBuildAdd(builder, idx,
1206
LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1207
LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1208
stencil_refs[0] = LLVMBuildLoad(builder, ptr, "output.s");
1209
/* there's only one value, and spec says to discard additional bits */
1210
LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255);
1211
stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, "");
1212
stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, "");
1213
stencil_refs[1] = stencil_refs[0];
1216
lp_build_depth_stencil_load_swizzled(gallivm, type,
1217
zs_format_desc, key->resource_1d,
1218
depth_ptr, depth_stride,
1219
&z_fb, &s_fb, loop_state.counter);
1221
lp_build_depth_stencil_test(gallivm,
1226
key->multisample ? NULL : &mask,
1234
if (depth_mode & LATE_DEPTH_WRITE) {
1235
lp_build_depth_stencil_write_swizzled(gallivm, type,
1236
zs_format_desc, key->resource_1d,
1237
NULL, NULL, NULL, loop_state.counter,
1238
depth_ptr, depth_stride,
1242
else if ((depth_mode & EARLY_DEPTH_TEST) &&
1243
(depth_mode & LATE_DEPTH_WRITE))
1245
/* Need to apply a reduced mask to the depth write. Reload the
1246
* depth value, update from zs_value with the new mask value and
1249
if (key->multisample) {
1250
z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");;
1251
s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter);
1252
z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, "");
1253
s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter);
1255
lp_build_depth_stencil_write_swizzled(gallivm, type,
1256
zs_format_desc, key->resource_1d,
1257
key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter,
1258
depth_ptr, depth_stride,
1262
if (key->occlusion_count) {
1263
LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr);
1264
lp_build_name(counter, "counter");
1266
lp_build_occlusion_count(gallivm, type,
1267
key->multisample ? s_mask : lp_build_mask_value(&mask), counter);
1270
/* if this is genuine early-depth in the shader, write samplemask now
1271
* after occlusion count has been updated
1273
if (key->multisample && shader->info.base.writes_samplemask &&
1274
(depth_mode & (EARLY_DEPTH_TEST_INFERRED | EARLY_DEPTH_TEST)) == EARLY_DEPTH_TEST) {
1275
/* if the shader writes sample mask use that */
1276
LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1277
out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1278
LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1279
LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1280
LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1281
smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1283
s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1287
if (key->multisample) {
1288
/* store the sample mask for this loop */
1289
LLVMBuildStore(builder, s_mask, s_mask_ptr);
1290
lp_build_for_loop_end(&sample_loop_state);
1293
mask_val = lp_build_mask_end(&mask);
1294
if (!key->multisample)
1295
LLVMBuildStore(builder, mask_val, mask_ptr);
1296
lp_build_for_loop_end(&loop_state);
1301
* This function will reorder pixels from the fragment shader SoA to memory layout AoS
1303
* Fragment Shader outputs pixels in small 2x2 blocks
1304
* e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ...
1306
* However in memory pixels are stored in rows
1307
* e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ...
1309
* @param type fragment shader type (4x or 8x float)
1310
* @param num_fs number of fs_src
1311
* @param is_1d whether we're outputting to a 1d resource
1312
* @param dst_channels number of output channels
1313
* @param fs_src output from fragment shader
1314
* @param dst pointer to store result
1315
* @param pad_inline is channel padding inline or at end of row
1316
* @return the number of dsts
1319
generate_fs_twiddle(struct gallivm_state *gallivm,
1320
struct lp_type type,
1322
unsigned dst_channels,
1323
LLVMValueRef fs_src[][4],
1327
LLVMValueRef src[16];
1333
unsigned pixels = type.length / 4;
1334
unsigned reorder_group;
1335
unsigned src_channels;
1339
src_channels = dst_channels < 3 ? dst_channels : 4;
1340
src_count = num_fs * src_channels;
1342
assert(pixels == 2 || pixels == 1);
1343
assert(num_fs * src_channels <= ARRAY_SIZE(src));
1346
* Transpose from SoA -> AoS
1348
for (i = 0; i < num_fs; ++i) {
1349
lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]);
1353
* Pick transformation options
1355
swizzle_pad = false;
1360
if (dst_channels == 1) {
1366
} else if (dst_channels == 2) {
1370
} else if (dst_channels > 2) {
1377
if (!pad_inline && dst_channels == 3 && pixels > 1) {
1383
* Split the src in half
1386
for (i = num_fs; i > 0; --i) {
1387
src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4);
1388
src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4);
1396
* Ensure pixels are in memory order
1398
if (reorder_group) {
1399
/* Twiddle pixels by reordering the array, e.g.:
1401
* src_count = 8 -> 0 2 1 3 4 6 5 7
1402
* src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15
1404
const unsigned reorder_sw[] = { 0, 2, 1, 3 };
1406
for (i = 0; i < src_count; ++i) {
1407
unsigned group = i / reorder_group;
1408
unsigned block = (group / 4) * 4 * reorder_group;
1409
unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group);
1412
} else if (twiddle) {
1413
/* Twiddle pixels across elements of array */
1415
* XXX: we should avoid this in some cases, but would need to tell
1416
* lp_build_conv to reorder (or deal with it ourselves).
1418
lp_bld_quad_twiddle(gallivm, type, src, src_count, dst);
1421
memcpy(dst, src, sizeof(LLVMValueRef) * src_count);
1425
* Moves any padding between pixels to the end
1426
* e.g. RGBXRGBX -> RGBRGBXX
1429
unsigned char swizzles[16];
1430
unsigned elems = pixels * dst_channels;
1432
for (i = 0; i < type.length; ++i) {
1434
swizzles[i] = i % dst_channels + (i / dst_channels) * 4;
1436
swizzles[i] = LP_BLD_SWIZZLE_DONTCARE;
1439
for (i = 0; i < src_count; ++i) {
1440
dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length);
1449
* Untwiddle and transpose, much like the above.
1450
* However, this is after conversion, so we get packed vectors.
1451
* At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data,
1452
* the vectors will look like:
1453
* r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may
1454
* be swizzled here). Extending to 16bit should be trivial.
1455
* Should also be extended to handle twice wide vectors with AVX2...
1458
fs_twiddle_transpose(struct gallivm_state *gallivm,
1459
struct lp_type type,
1465
struct lp_type type64, type16, type32;
1466
LLVMTypeRef type64_t, type8_t, type16_t, type32_t;
1467
LLVMBuilderRef builder = gallivm->builder;
1468
LLVMValueRef tmp[4], shuf[8];
1469
for (j = 0; j < 2; j++) {
1470
shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0);
1471
shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2);
1472
shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1);
1473
shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3);
1476
assert(src_count == 4 || src_count == 2 || src_count == 1);
1477
assert(type.width == 8);
1478
assert(type.length == 16);
1480
type8_t = lp_build_vec_type(gallivm, type);
1485
type64_t = lp_build_vec_type(gallivm, type64);
1490
type16_t = lp_build_vec_type(gallivm, type16);
1495
type32_t = lp_build_vec_type(gallivm, type32);
1497
lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp);
1499
if (src_count == 1) {
1500
/* transpose was no-op, just untwiddle */
1501
LLVMValueRef shuf_vec;
1502
shuf_vec = LLVMConstVector(shuf, 8);
1503
tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, "");
1504
tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, "");
1505
dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, "");
1506
} else if (src_count == 2) {
1507
LLVMValueRef shuf_vec;
1508
shuf_vec = LLVMConstVector(shuf, 4);
1510
for (i = 0; i < 2; i++) {
1511
tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, "");
1512
tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, "");
1513
dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, "");
1516
for (j = 0; j < 2; j++) {
1517
LLVMValueRef lo, hi, lo2, hi2;
1519
* Note that if we only really have 3 valid channels (rgb)
1520
* and we don't need alpha we could substitute a undef here
1521
* for the respective channel (causing llvm to drop conversion
1524
/* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */
1525
lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, "");
1526
hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, "");
1527
lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0);
1528
hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1);
1529
dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, "");
1530
dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, "");
1537
* Load an unswizzled block of pixels from memory
1540
load_unswizzled_block(struct gallivm_state *gallivm,
1541
LLVMValueRef base_ptr,
1542
LLVMValueRef stride,
1543
unsigned block_width,
1544
unsigned block_height,
1546
struct lp_type dst_type,
1548
unsigned dst_alignment)
1550
LLVMBuilderRef builder = gallivm->builder;
1551
unsigned row_size = dst_count / block_height;
1554
/* Ensure block exactly fits into dst */
1555
assert((block_width * block_height) % dst_count == 0);
1557
for (i = 0; i < dst_count; ++i) {
1558
unsigned x = i % row_size;
1559
unsigned y = i / row_size;
1561
LLVMValueRef bx = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length);
1562
LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1564
LLVMValueRef gep[2];
1565
LLVMValueRef dst_ptr;
1567
gep[0] = lp_build_const_int32(gallivm, 0);
1568
gep[1] = LLVMBuildAdd(builder, bx, by, "");
1570
dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1571
dst_ptr = LLVMBuildBitCast(builder, dst_ptr,
1572
LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), "");
1574
dst[i] = LLVMBuildLoad(builder, dst_ptr, "");
1576
LLVMSetAlignment(dst[i], dst_alignment);
1582
* Store an unswizzled block of pixels to memory
1585
store_unswizzled_block(struct gallivm_state *gallivm,
1586
LLVMValueRef base_ptr,
1587
LLVMValueRef stride,
1588
unsigned block_width,
1589
unsigned block_height,
1591
struct lp_type src_type,
1593
unsigned src_alignment)
1595
LLVMBuilderRef builder = gallivm->builder;
1596
unsigned row_size = src_count / block_height;
1599
/* Ensure src exactly fits into block */
1600
assert((block_width * block_height) % src_count == 0);
1602
for (i = 0; i < src_count; ++i) {
1603
unsigned x = i % row_size;
1604
unsigned y = i / row_size;
1606
LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length);
1607
LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1609
LLVMValueRef gep[2];
1610
LLVMValueRef src_ptr;
1612
gep[0] = lp_build_const_int32(gallivm, 0);
1613
gep[1] = LLVMBuildAdd(builder, bx, by, "");
1615
src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1616
src_ptr = LLVMBuildBitCast(builder, src_ptr,
1617
LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), "");
1619
src_ptr = LLVMBuildStore(builder, src[i], src_ptr);
1621
LLVMSetAlignment(src_ptr, src_alignment);
1628
* Retrieves the type for a format which is usable in the blending code.
1630
* e.g. RGBA16F = 4x float, R3G3B2 = 3x byte
1633
lp_blend_type_from_format_desc(const struct util_format_description *format_desc,
1634
struct lp_type* type)
1639
if (format_expands_to_float_soa(format_desc)) {
1640
/* always use ordinary floats for blending */
1641
type->floating = true;
1642
type->fixed = false;
1650
for (i = 0; i < 4; i++)
1651
if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1655
memset(type, 0, sizeof(struct lp_type));
1656
type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
1657
type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
1658
type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
1659
type->norm = format_desc->channel[chan].normalized;
1660
type->width = format_desc->channel[chan].size;
1661
type->length = format_desc->nr_channels;
1663
for (i = 1; i < format_desc->nr_channels; ++i) {
1664
if (format_desc->channel[i].size > type->width)
1665
type->width = format_desc->channel[i].size;
1668
if (type->floating) {
1671
if (type->width <= 8) {
1673
} else if (type->width <= 16) {
1680
if (is_arithmetic_format(format_desc) && type->length == 3) {
1687
* Scale a normalized value from src_bits to dst_bits.
1689
* The exact calculation is
1691
* dst = iround(src * dst_mask / src_mask)
1693
* or with integer rounding
1695
* dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask)
1699
* src_mask = (1 << src_bits) - 1
1700
* dst_mask = (1 << dst_bits) - 1
1702
* but we try to avoid division and multiplication through shifts.
1704
static inline LLVMValueRef
1705
scale_bits(struct gallivm_state *gallivm,
1709
struct lp_type src_type)
1711
LLVMBuilderRef builder = gallivm->builder;
1712
LLVMValueRef result = src;
1714
if (dst_bits < src_bits) {
1715
int delta_bits = src_bits - dst_bits;
1717
if (delta_bits <= dst_bits) {
1719
if (dst_bits == 4) {
1720
struct lp_type flt_type = lp_type_float_vec(32, src_type.length * 32);
1722
result = lp_build_unsigned_norm_to_float(gallivm, src_bits, flt_type, src);
1723
result = lp_build_clamped_float_to_unsigned_norm(gallivm, flt_type, dst_bits, result);
1724
result = LLVMBuildTrunc(gallivm->builder, result, lp_build_int_vec_type(gallivm, src_type), "");
1729
* Approximate the rescaling with a single shift.
1731
* This gives the wrong rounding.
1734
result = LLVMBuildLShr(builder,
1736
lp_build_const_int_vec(gallivm, src_type, delta_bits),
1741
* Try more accurate rescaling.
1745
* Drop the least significant bits to make space for the multiplication.
1747
* XXX: A better approach would be to use a wider integer type as intermediate. But
1748
* this is enough to convert alpha from 16bits -> 2 when rendering to
1749
* PIPE_FORMAT_R10G10B10A2_UNORM.
1751
result = LLVMBuildLShr(builder,
1753
lp_build_const_int_vec(gallivm, src_type, dst_bits),
1757
result = LLVMBuildMul(builder,
1759
lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1),
1763
* Add a rounding term before the division.
1765
* TODO: Handle signed integers too.
1767
if (!src_type.sign) {
1768
result = LLVMBuildAdd(builder,
1770
lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))),
1775
* Approximate the division by src_mask with a src_bits shift.
1777
* Given the src has already been shifted by dst_bits, all we need
1778
* to do is to shift by the difference.
1781
result = LLVMBuildLShr(builder,
1783
lp_build_const_int_vec(gallivm, src_type, delta_bits),
1787
} else if (dst_bits > src_bits) {
1789
int db = dst_bits - src_bits;
1791
/* Shift left by difference in bits */
1792
result = LLVMBuildShl(builder,
1794
lp_build_const_int_vec(gallivm, src_type, db),
1797
if (db <= src_bits) {
1798
/* Enough bits in src to fill the remainder */
1799
LLVMValueRef lower = LLVMBuildLShr(builder,
1801
lp_build_const_int_vec(gallivm, src_type, src_bits - db),
1804
result = LLVMBuildOr(builder, result, lower, "");
1805
} else if (db > src_bits) {
1806
/* Need to repeatedly copy src bits to fill remainder in dst */
1809
for (n = src_bits; n < dst_bits; n *= 2) {
1810
LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n);
1812
result = LLVMBuildOr(builder,
1814
LLVMBuildLShr(builder, result, shuv, ""),
1824
* If RT is a smallfloat (needing denorms) format
1827
have_smallfloat_format(struct lp_type dst_type,
1828
enum pipe_format format)
1830
return ((dst_type.floating && dst_type.width != 32) ||
1831
/* due to format handling hacks this format doesn't have floating set
1832
* here (and actually has width set to 32 too) so special case this. */
1833
(format == PIPE_FORMAT_R11G11B10_FLOAT));
1838
* Convert from memory format to blending format
1840
* e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending
1843
convert_to_blend_type(struct gallivm_state *gallivm,
1844
unsigned block_size,
1845
const struct util_format_description *src_fmt,
1846
struct lp_type src_type,
1847
struct lp_type dst_type,
1848
LLVMValueRef* src, // and dst
1851
LLVMValueRef *dst = src;
1852
LLVMBuilderRef builder = gallivm->builder;
1853
struct lp_type blend_type;
1854
struct lp_type mem_type;
1856
unsigned pixels = block_size / num_srcs;
1860
* full custom path for packed floats and srgb formats - none of the later
1861
* functions would do anything useful, and given the lp_type representation they
1862
* can't be fixed. Should really have some SoA blend path for these kind of
1863
* formats rather than hacking them in here.
1865
if (format_expands_to_float_soa(src_fmt)) {
1866
LLVMValueRef tmpsrc[4];
1868
* This is pretty suboptimal for this case blending in SoA would be much
1869
* better, since conversion gets us SoA values so need to convert back.
1871
assert(src_type.width == 32 || src_type.width == 16);
1872
assert(dst_type.floating);
1873
assert(dst_type.width == 32);
1874
assert(dst_type.length % 4 == 0);
1875
assert(num_srcs % 4 == 0);
1877
if (src_type.width == 16) {
1878
/* expand 4x16bit values to 4x32bit */
1879
struct lp_type type32x4 = src_type;
1880
LLVMTypeRef ltype32x4;
1881
unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
1882
type32x4.width = 32;
1883
ltype32x4 = lp_build_vec_type(gallivm, type32x4);
1884
for (i = 0; i < num_fetch; i++) {
1885
src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, "");
1887
src_type.width = 32;
1889
for (i = 0; i < 4; i++) {
1892
for (i = 0; i < num_srcs / 4; i++) {
1893
LLVMValueRef tmpsoa[4];
1894
LLVMValueRef tmps = tmpsrc[i];
1895
if (dst_type.length == 8) {
1896
LLVMValueRef shuffles[8];
1898
/* fetch was 4 values but need 8-wide output values */
1899
tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2);
1901
* for 8-wide aos transpose would give us wrong order not matching
1902
* incoming converted fs values and mask. ARGH.
1904
for (j = 0; j < 4; j++) {
1905
shuffles[j] = lp_build_const_int32(gallivm, j * 2);
1906
shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1);
1908
tmps = LLVMBuildShuffleVector(builder, tmps, tmps,
1909
LLVMConstVector(shuffles, 8), "");
1911
if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
1912
lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa);
1915
lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa);
1917
lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]);
1922
lp_mem_type_from_format_desc(src_fmt, &mem_type);
1923
lp_blend_type_from_format_desc(src_fmt, &blend_type);
1925
/* Is the format arithmetic */
1926
is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length;
1927
is_arith &= !(mem_type.width == 16 && mem_type.floating);
1929
/* Pad if necessary */
1930
if (!is_arith && src_type.length < dst_type.length) {
1931
for (i = 0; i < num_srcs; ++i) {
1932
dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length);
1935
src_type.length = dst_type.length;
1938
/* Special case for half-floats */
1939
if (mem_type.width == 16 && mem_type.floating) {
1940
assert(blend_type.width == 32 && blend_type.floating);
1941
lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
1949
src_type.width = blend_type.width * blend_type.length;
1950
blend_type.length *= pixels;
1951
src_type.length *= pixels / (src_type.length / mem_type.length);
1953
for (i = 0; i < num_srcs; ++i) {
1955
LLVMValueRef res = NULL;
1957
dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
1959
for (j = 0; j < src_fmt->nr_channels; ++j) {
1961
unsigned sa = src_fmt->channel[j].shift;
1962
#if UTIL_ARCH_LITTLE_ENDIAN
1963
unsigned from_lsb = j;
1965
unsigned from_lsb = (blend_type.length / pixels) - j - 1;
1968
mask = (1 << src_fmt->channel[j].size) - 1;
1970
/* Extract bits from source */
1971
chans = LLVMBuildLShr(builder,
1973
lp_build_const_int_vec(gallivm, src_type, sa),
1976
chans = LLVMBuildAnd(builder,
1978
lp_build_const_int_vec(gallivm, src_type, mask),
1982
if (src_type.norm) {
1983
chans = scale_bits(gallivm, src_fmt->channel[j].size,
1984
blend_type.width, chans, src_type);
1987
/* Insert bits into correct position */
1988
chans = LLVMBuildShl(builder,
1990
lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width),
1996
res = LLVMBuildOr(builder, res, chans, "");
2000
dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), "");
2006
* Convert from blending format to memory format
2008
* e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory
2011
convert_from_blend_type(struct gallivm_state *gallivm,
2012
unsigned block_size,
2013
const struct util_format_description *src_fmt,
2014
struct lp_type src_type,
2015
struct lp_type dst_type,
2016
LLVMValueRef* src, // and dst
2019
LLVMValueRef* dst = src;
2021
struct lp_type mem_type;
2022
struct lp_type blend_type;
2023
LLVMBuilderRef builder = gallivm->builder;
2024
unsigned pixels = block_size / num_srcs;
2028
* full custom path for packed floats and srgb formats - none of the later
2029
* functions would do anything useful, and given the lp_type representation they
2030
* can't be fixed. Should really have some SoA blend path for these kind of
2031
* formats rather than hacking them in here.
2033
if (format_expands_to_float_soa(src_fmt)) {
2035
* This is pretty suboptimal for this case blending in SoA would be much
2036
* better - we need to transpose the AoS values back to SoA values for
2037
* conversion/packing.
2039
assert(src_type.floating);
2040
assert(src_type.width == 32);
2041
assert(src_type.length % 4 == 0);
2042
assert(dst_type.width == 32 || dst_type.width == 16);
2044
for (i = 0; i < num_srcs / 4; i++) {
2045
LLVMValueRef tmpsoa[4], tmpdst;
2046
lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa);
2047
/* really really need SoA here */
2049
if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
2050
tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa);
2053
tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt,
2057
if (src_type.length == 8) {
2058
LLVMValueRef tmpaos, shuffles[8];
2061
* for 8-wide aos transpose has given us wrong order not matching
2062
* output order. HMPF. Also need to split the output values manually.
2064
for (j = 0; j < 4; j++) {
2065
shuffles[j * 2] = lp_build_const_int32(gallivm, j);
2066
shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4);
2068
tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst,
2069
LLVMConstVector(shuffles, 8), "");
2070
src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4);
2071
src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4);
2077
if (dst_type.width == 16) {
2078
struct lp_type type16x8 = dst_type;
2079
struct lp_type type32x4 = dst_type;
2080
LLVMTypeRef ltype16x4, ltypei64, ltypei128;
2081
unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
2082
type16x8.length = 8;
2083
type32x4.width = 32;
2084
ltypei128 = LLVMIntTypeInContext(gallivm->context, 128);
2085
ltypei64 = LLVMIntTypeInContext(gallivm->context, 64);
2086
ltype16x4 = lp_build_vec_type(gallivm, dst_type);
2087
/* We could do vector truncation but it doesn't generate very good code */
2088
for (i = 0; i < num_fetch; i++) {
2089
src[i] = lp_build_pack2(gallivm, type32x4, type16x8,
2090
src[i], lp_build_zero(gallivm, type32x4));
2091
src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, "");
2092
src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, "");
2093
src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, "");
2099
lp_mem_type_from_format_desc(src_fmt, &mem_type);
2100
lp_blend_type_from_format_desc(src_fmt, &blend_type);
2102
is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length);
2104
/* Special case for half-floats */
2105
if (mem_type.width == 16 && mem_type.floating) {
2106
int length = dst_type.length;
2107
assert(blend_type.width == 32 && blend_type.floating);
2109
dst_type.length = src_type.length;
2111
lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
2113
dst_type.length = length;
2117
/* Remove any padding */
2118
if (!is_arith && (src_type.length % mem_type.length)) {
2119
src_type.length -= (src_type.length % mem_type.length);
2121
for (i = 0; i < num_srcs; ++i) {
2122
dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length);
2126
/* No bit arithmetic to do */
2131
src_type.length = pixels;
2132
src_type.width = blend_type.length * blend_type.width;
2133
dst_type.length = pixels;
2135
for (i = 0; i < num_srcs; ++i) {
2137
LLVMValueRef res = NULL;
2139
dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
2141
for (j = 0; j < src_fmt->nr_channels; ++j) {
2143
unsigned sa = src_fmt->channel[j].shift;
2144
unsigned sz_a = src_fmt->channel[j].size;
2145
#if UTIL_ARCH_LITTLE_ENDIAN
2146
unsigned from_lsb = j;
2148
unsigned from_lsb = blend_type.length - j - 1;
2151
assert(blend_type.width > src_fmt->channel[j].size);
2153
for (k = 0; k < blend_type.width; ++k) {
2158
chans = LLVMBuildLShr(builder,
2160
lp_build_const_int_vec(gallivm, src_type,
2161
from_lsb * blend_type.width),
2164
chans = LLVMBuildAnd(builder,
2166
lp_build_const_int_vec(gallivm, src_type, mask),
2169
/* Scale down bits */
2170
if (src_type.norm) {
2171
chans = scale_bits(gallivm, blend_type.width,
2172
src_fmt->channel[j].size, chans, src_type);
2173
} else if (!src_type.floating && sz_a < blend_type.width) {
2174
LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1);
2175
LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans, mask_val, "");
2176
chans = LLVMBuildSelect(builder, mask, mask_val, chans, "");
2180
chans = LLVMBuildShl(builder,
2182
lp_build_const_int_vec(gallivm, src_type, sa),
2185
sa += src_fmt->channel[j].size;
2190
res = LLVMBuildOr(builder, res, chans, "");
2194
assert (dst_type.width != 24);
2196
dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), "");
2202
* Convert alpha to same blend type as src
2205
convert_alpha(struct gallivm_state *gallivm,
2206
struct lp_type row_type,
2207
struct lp_type alpha_type,
2208
const unsigned block_size,
2209
const unsigned block_height,
2210
const unsigned src_count,
2211
const unsigned dst_channels,
2212
const bool pad_inline,
2213
LLVMValueRef* src_alpha)
2215
LLVMBuilderRef builder = gallivm->builder;
2217
unsigned length = row_type.length;
2218
row_type.length = alpha_type.length;
2220
/* Twiddle the alpha to match pixels */
2221
lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha);
2224
* TODO this should use single lp_build_conv call for
2225
* src_count == 1 && dst_channels == 1 case (dropping the concat below)
2227
for (i = 0; i < block_height; ++i) {
2228
lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1);
2231
alpha_type = row_type;
2232
row_type.length = length;
2234
/* If only one channel we can only need the single alpha value per pixel */
2235
if (src_count == 1 && dst_channels == 1) {
2237
lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count);
2239
/* If there are more srcs than rows then we need to split alpha up */
2240
if (src_count > block_height) {
2241
for (i = src_count; i > 0; --i) {
2242
unsigned pixels = block_size / src_count;
2243
unsigned idx = i - 1;
2245
src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4],
2246
(idx * pixels) % 4, pixels);
2250
/* If there is a src for each pixel broadcast the alpha across whole row */
2251
if (src_count == block_size) {
2252
for (i = 0; i < src_count; ++i) {
2253
src_alpha[i] = lp_build_broadcast(gallivm,
2254
lp_build_vec_type(gallivm, row_type), src_alpha[i]);
2257
unsigned pixels = block_size / src_count;
2258
unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels;
2259
unsigned alpha_span = 1;
2260
LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
2262
/* Check if we need 2 src_alphas for our shuffles */
2263
if (pixels > alpha_type.length) {
2267
/* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */
2268
for (j = 0; j < row_type.length; ++j) {
2269
if (j < pixels * channels) {
2270
shuffles[j] = lp_build_const_int32(gallivm, j / channels);
2272
shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
2276
for (i = 0; i < src_count; ++i) {
2277
unsigned idx1 = i, idx2 = i;
2279
if (alpha_span > 1){
2284
src_alpha[i] = LLVMBuildShuffleVector(builder,
2287
LLVMConstVector(shuffles, row_type.length),
2296
* Generates the blend function for unswizzled colour buffers
2297
* Also generates the read & write from colour buffer
2300
generate_unswizzled_blend(struct gallivm_state *gallivm,
2302
struct lp_fragment_shader_variant *variant,
2303
enum pipe_format out_format,
2304
unsigned int num_fs,
2305
struct lp_type fs_type,
2306
LLVMValueRef* fs_mask,
2307
LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],
2308
LLVMValueRef context_ptr,
2309
LLVMValueRef color_ptr,
2310
LLVMValueRef stride,
2311
unsigned partial_mask,
2314
const unsigned alpha_channel = 3;
2315
const unsigned block_width = LP_RASTER_BLOCK_SIZE;
2316
const unsigned block_height = LP_RASTER_BLOCK_SIZE;
2317
const unsigned block_size = block_width * block_height;
2318
const unsigned lp_integer_vector_width = 128;
2320
LLVMBuilderRef builder = gallivm->builder;
2321
LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS];
2322
LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS];
2323
LLVMValueRef src_alpha[4 * 4];
2324
LLVMValueRef src1_alpha[4 * 4] = { NULL };
2325
LLVMValueRef src_mask[4 * 4];
2326
LLVMValueRef src[4 * 4];
2327
LLVMValueRef src1[4 * 4];
2328
LLVMValueRef dst[4 * 4];
2329
LLVMValueRef blend_color;
2330
LLVMValueRef blend_alpha;
2331
LLVMValueRef i32_zero;
2332
LLVMValueRef check_mask;
2333
LLVMValueRef undef_src_val;
2335
struct lp_build_mask_context mask_ctx;
2336
struct lp_type mask_type;
2337
struct lp_type blend_type;
2338
struct lp_type row_type;
2339
struct lp_type dst_type;
2340
struct lp_type ls_type;
2342
unsigned char swizzle[TGSI_NUM_CHANNELS];
2343
unsigned vector_width;
2344
unsigned src_channels = TGSI_NUM_CHANNELS;
2345
unsigned dst_channels;
2350
const struct util_format_description* out_format_desc = util_format_description(out_format);
2352
unsigned dst_alignment;
2354
bool pad_inline = is_arithmetic_format(out_format_desc);
2355
bool has_alpha = false;
2356
const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable &&
2357
util_blend_state_is_dual(&variant->key.blend, 0);
2359
const boolean is_1d = variant->key.resource_1d;
2360
boolean twiddle_after_convert = FALSE;
2361
unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs;
2362
LLVMValueRef fpstate = 0;
2364
/* Get type from output format */
2365
lp_blend_type_from_format_desc(out_format_desc, &row_type);
2366
lp_mem_type_from_format_desc(out_format_desc, &dst_type);
2369
* Technically this code should go into lp_build_smallfloat_to_float
2370
* and lp_build_float_to_smallfloat but due to the
2371
* http://llvm.org/bugs/show_bug.cgi?id=6393
2372
* llvm reorders the mxcsr intrinsics in a way that breaks the code.
2373
* So the ordering is important here and there shouldn't be any
2374
* llvm ir instrunctions in this function before
2375
* this, otherwise half-float format conversions won't work
2376
* (again due to llvm bug #6393).
2378
if (have_smallfloat_format(dst_type, out_format)) {
2379
/* We need to make sure that denorms are ok for half float
2381
fpstate = lp_build_fpstate_get(gallivm);
2382
lp_build_fpstate_set_denorms_zero(gallivm, FALSE);
2385
mask_type = lp_int32_vec4_type();
2386
mask_type.length = fs_type.length;
2388
for (i = num_fs; i < num_fullblock_fs; i++) {
2389
fs_mask[i] = lp_build_zero(gallivm, mask_type);
2392
/* Do not bother executing code when mask is empty.. */
2394
check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type));
2396
for (i = 0; i < num_fullblock_fs; ++i) {
2397
check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], "");
2400
lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask);
2401
lp_build_mask_check(&mask_ctx);
2404
partial_mask |= !variant->opaque;
2405
i32_zero = lp_build_const_int32(gallivm, 0);
2407
undef_src_val = lp_build_undef(gallivm, fs_type);
2409
row_type.length = fs_type.length;
2410
vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width;
2412
/* Compute correct swizzle and count channels */
2413
memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS);
2416
for (i = 0; i < TGSI_NUM_CHANNELS; ++i) {
2417
/* Ensure channel is used */
2418
if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) {
2422
/* Ensure not already written to (happens in case with GL_ALPHA) */
2423
if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) {
2427
/* Ensure we haven't already found all channels */
2428
if (dst_channels >= out_format_desc->nr_channels) {
2432
swizzle[out_format_desc->swizzle[i]] = i;
2435
if (i == alpha_channel) {
2440
if (format_expands_to_float_soa(out_format_desc)) {
2442
* the code above can't work for layout_other
2443
* for srgb it would sort of work but we short-circuit swizzles, etc.
2444
* as that is done as part of unpack / pack.
2446
dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */
2452
pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */
2455
/* If 3 channels then pad to include alpha for 4 element transpose */
2456
if (dst_channels == 3) {
2457
assert (!has_alpha);
2458
for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
2459
if (swizzle[i] > TGSI_NUM_CHANNELS)
2462
if (out_format_desc->nr_channels == 4) {
2465
* We use alpha from the color conversion, not separate one.
2466
* We had to include it for transpose, hence it will get converted
2467
* too (albeit when doing transpose after conversion, that would
2468
* no longer be the case necessarily).
2469
* (It works only with 4 channel dsts, e.g. rgbx formats, because
2470
* otherwise we really have padding, not alpha, included.)
2477
* Load shader output
2479
for (i = 0; i < num_fullblock_fs; ++i) {
2480
/* Always load alpha for use in blending */
2483
alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], "");
2486
alpha = undef_src_val;
2489
/* Load each channel */
2490
for (j = 0; j < dst_channels; ++j) {
2491
assert(swizzle[j] < 4);
2493
fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], "");
2496
fs_src[i][j] = undef_src_val;
2500
/* If 3 channels then pad to include alpha for 4 element transpose */
2502
* XXX If we include that here maybe could actually use it instead of
2503
* separate alpha for blending?
2504
* (Difficult though we actually convert pad channels, not alpha.)
2506
if (dst_channels == 3 && !has_alpha) {
2507
fs_src[i][3] = alpha;
2510
/* We split the row_mask and row_alpha as we want 128bit interleave */
2511
if (fs_type.length == 8) {
2512
src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i],
2514
src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i],
2515
src_channels, src_channels);
2517
src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2518
src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2519
src_channels, src_channels);
2521
src_mask[i] = fs_mask[i];
2522
src_alpha[i] = alpha;
2525
if (dual_source_blend) {
2526
/* same as above except different src/dst, skip masks and comments... */
2527
for (i = 0; i < num_fullblock_fs; ++i) {
2530
alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], "");
2533
alpha = undef_src_val;
2536
for (j = 0; j < dst_channels; ++j) {
2537
assert(swizzle[j] < 4);
2539
fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], "");
2542
fs_src1[i][j] = undef_src_val;
2545
if (dst_channels == 3 && !has_alpha) {
2546
fs_src1[i][3] = alpha;
2548
if (fs_type.length == 8) {
2549
src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2550
src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2551
src_channels, src_channels);
2553
src1_alpha[i] = alpha;
2558
if (util_format_is_pure_integer(out_format)) {
2560
* In this case fs_type was really ints or uints disguised as floats,
2563
fs_type.floating = 0;
2564
fs_type.sign = dst_type.sign;
2565
for (i = 0; i < num_fullblock_fs; ++i) {
2566
for (j = 0; j < dst_channels; ++j) {
2567
fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j],
2568
lp_build_vec_type(gallivm, fs_type), "");
2570
if (dst_channels == 3 && !has_alpha) {
2571
fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3],
2572
lp_build_vec_type(gallivm, fs_type), "");
2578
* We actually should generally do conversion first (for non-1d cases)
2579
* when the blend format is 8 or 16 bits. The reason is obvious,
2580
* there's 2 or 4 times less vectors to deal with for the interleave...
2581
* Albeit for the AVX (not AVX2) case there's no benefit with 16 bit
2582
* vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit
2583
* unpack only with 128bit vectors).
2584
* Note: for 16bit sizes really need matching pack conversion code
2586
if (!is_1d && dst_channels != 3 && dst_type.width == 8) {
2587
twiddle_after_convert = TRUE;
2591
* Pixel twiddle from fragment shader order to memory order
2593
if (!twiddle_after_convert) {
2594
src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs,
2595
dst_channels, fs_src, src, pad_inline);
2596
if (dual_source_blend) {
2597
generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels,
2598
fs_src1, src1, pad_inline);
2601
src_count = num_fullblock_fs * dst_channels;
2603
* We reorder things a bit here, so the cases for 4-wide and 8-wide
2604
* (AVX) turn out the same later when untwiddling/transpose (albeit
2605
* for true AVX2 path untwiddle needs to be different).
2606
* For now just order by colors first (so we can use unpack later).
2608
for (j = 0; j < num_fullblock_fs; j++) {
2609
for (i = 0; i < dst_channels; i++) {
2610
src[i*num_fullblock_fs + j] = fs_src[j][i];
2611
if (dual_source_blend) {
2612
src1[i*num_fullblock_fs + j] = fs_src1[j][i];
2618
src_channels = dst_channels < 3 ? dst_channels : 4;
2619
if (src_count != num_fullblock_fs * src_channels) {
2620
unsigned ds = src_count / (num_fullblock_fs * src_channels);
2621
row_type.length /= ds;
2622
fs_type.length = row_type.length;
2625
blend_type = row_type;
2626
mask_type.length = 4;
2628
/* Convert src to row_type */
2629
if (dual_source_blend) {
2630
struct lp_type old_row_type = row_type;
2631
lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2632
src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1);
2635
src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2638
/* If the rows are not an SSE vector, combine them to become SSE size! */
2639
if ((row_type.width * row_type.length) % 128) {
2640
unsigned bits = row_type.width * row_type.length;
2643
assert(src_count >= (vector_width / bits));
2645
dst_count = src_count / (vector_width / bits);
2647
combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count);
2648
if (dual_source_blend) {
2649
lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count);
2652
row_type.length *= combined;
2653
src_count /= combined;
2655
bits = row_type.width * row_type.length;
2656
assert(bits == 128 || bits == 256);
2659
if (twiddle_after_convert) {
2660
fs_twiddle_transpose(gallivm, row_type, src, src_count, src);
2661
if (dual_source_blend) {
2662
fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1);
2667
* Blend Colour conversion
2669
blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr);
2670
blend_color = LLVMBuildPointerCast(builder, blend_color,
2671
LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), "");
2672
blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color,
2673
&i32_zero, 1, ""), "");
2676
lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1);
2678
if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
2680
* since blending is done with floats, there was no conversion.
2681
* However, the rules according to fixed point renderbuffers still
2682
* apply, that is we must clamp inputs to 0.0/1.0.
2683
* (This would apply to separate alpha conversion too but we currently
2684
* force has_alpha to be true.)
2685
* TODO: should skip this with "fake" blend, since post-blend conversion
2686
* will clamp anyway.
2687
* TODO: could also skip this if fragment color clamping is enabled. We
2688
* don't support it natively so it gets baked into the shader however, so
2689
* can't really tell here.
2691
struct lp_build_context f32_bld;
2692
assert(row_type.floating);
2693
lp_build_context_init(&f32_bld, gallivm, row_type);
2694
for (i = 0; i < src_count; i++) {
2695
src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]);
2697
if (dual_source_blend) {
2698
for (i = 0; i < src_count; i++) {
2699
src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]);
2702
/* probably can't be different than row_type but better safe than sorry... */
2703
lp_build_context_init(&f32_bld, gallivm, blend_type);
2704
blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one);
2708
blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3));
2710
/* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */
2711
pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width;
2713
/* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */
2714
blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length);
2716
/* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */
2717
blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length);
2723
lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]);
2725
if (src_count < block_height) {
2726
lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count);
2727
} else if (src_count > block_height) {
2728
for (i = src_count; i > 0; --i) {
2729
unsigned pixels = block_size / src_count;
2730
unsigned idx = i - 1;
2732
src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4],
2733
(idx * pixels) % 4, pixels);
2737
assert(mask_type.width == 32);
2739
for (i = 0; i < src_count; ++i) {
2740
unsigned pixels = block_size / src_count;
2741
unsigned pixel_width = row_type.width * dst_channels;
2743
if (pixel_width == 24) {
2744
mask_type.width = 8;
2745
mask_type.length = vector_width / mask_type.width;
2747
mask_type.length = pixels;
2748
mask_type.width = row_type.width * dst_channels;
2751
* If mask_type width is smaller than 32bit, this doesn't quite
2752
* generate the most efficient code (could use some pack).
2754
src_mask[i] = LLVMBuildIntCast(builder, src_mask[i],
2755
lp_build_int_vec_type(gallivm, mask_type), "");
2757
mask_type.length *= dst_channels;
2758
mask_type.width /= dst_channels;
2761
src_mask[i] = LLVMBuildBitCast(builder, src_mask[i],
2762
lp_build_int_vec_type(gallivm, mask_type), "");
2763
src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length);
2770
struct lp_type alpha_type = fs_type;
2771
alpha_type.length = 4;
2772
convert_alpha(gallivm, row_type, alpha_type,
2773
block_size, block_height,
2774
src_count, dst_channels,
2775
pad_inline, src_alpha);
2776
if (dual_source_blend) {
2777
convert_alpha(gallivm, row_type, alpha_type,
2778
block_size, block_height,
2779
src_count, dst_channels,
2780
pad_inline, src1_alpha);
2786
* Load dst from memory
2788
if (src_count < block_height) {
2789
dst_count = block_height;
2791
dst_count = src_count;
2794
dst_type.length *= block_size / dst_count;
2796
if (format_expands_to_float_soa(out_format_desc)) {
2798
* we need multiple values at once for the conversion, so can as well
2799
* load them vectorized here too instead of concatenating later.
2800
* (Still need concatenation later for 8-wide vectors).
2802
dst_count = block_height;
2803
dst_type.length = block_width;
2807
* Compute the alignment of the destination pointer in bytes
2808
* We fetch 1-4 pixels, if the format has pot alignment then those fetches
2809
* are always aligned by MIN2(16, fetch_width) except for buffers (not
2810
* 1d tex but can't distinguish here) so need to stick with per-pixel
2811
* alignment in this case.
2814
dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8);
2817
dst_alignment = dst_type.length * dst_type.width / 8;
2819
/* Force power-of-two alignment by extracting only the least-significant-bit */
2820
dst_alignment = 1 << (ffs(dst_alignment) - 1);
2822
* Resource base and stride pointers are aligned to 16 bytes, so that's
2823
* the maximum alignment we can guarantee
2825
dst_alignment = MIN2(16, dst_alignment);
2829
if (dst_count > src_count) {
2830
if ((dst_type.width == 8 || dst_type.width == 16) &&
2831
util_is_power_of_two_or_zero(dst_type.length) &&
2832
dst_type.length * dst_type.width < 128) {
2834
* Never try to load values as 4xi8 which we will then
2835
* concatenate to larger vectors. This gives llvm a real
2836
* headache (the problem is the type legalizer (?) will
2837
* try to load that as 4xi8 zext to 4xi32 to fill the vector,
2838
* then the shuffles to concatenate are more or less impossible
2839
* - llvm is easily capable of generating a sequence of 32
2840
* pextrb/pinsrb instructions for that. Albeit it appears to
2841
* be fixed in llvm 4.0. So, load and concatenate with 32bit
2842
* width to avoid the trouble (16bit seems not as bad, llvm
2843
* probably recognizes the load+shuffle as only one shuffle
2844
* is necessary, but we can do just the same anyway).
2846
ls_type.length = dst_type.length * dst_type.width / 32;
2852
load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2853
dst, ls_type, dst_count / 4, dst_alignment);
2854
for (i = dst_count / 4; i < dst_count; i++) {
2855
dst[i] = lp_build_undef(gallivm, ls_type);
2860
load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2861
dst, ls_type, dst_count, dst_alignment);
2866
* Convert from dst/output format to src/blending format.
2868
* This is necessary as we can only read 1 row from memory at a time,
2869
* so the minimum dst_count will ever be at this point is 4.
2871
* With, for example, R8 format you can have all 16 pixels in a 128 bit vector,
2872
* this will take the 4 dsts and combine them into 1 src so we can perform blending
2873
* on all 16 pixels in that single vector at once.
2875
if (dst_count > src_count) {
2876
if (ls_type.length != dst_type.length && ls_type.length == 1) {
2877
LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type);
2878
LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1);
2879
for (i = 0; i < dst_count; i++) {
2880
dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, "");
2884
lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count);
2886
if (ls_type.length != dst_type.length) {
2887
struct lp_type tmp_type = dst_type;
2888
tmp_type.length = dst_type.length * 4 / src_count;
2889
for (i = 0; i < src_count; i++) {
2890
dst[i] = LLVMBuildBitCast(builder, dst[i],
2891
lp_build_vec_type(gallivm, tmp_type), "");
2899
/* XXX this is broken for RGB8 formats -
2900
* they get expanded from 12 to 16 elements (to include alpha)
2901
* by convert_to_blend_type then reduced to 15 instead of 12
2902
* by convert_from_blend_type (a simple fix though breaks A8...).
2903
* R16G16B16 also crashes differently however something going wrong
2904
* inside llvm handling npot vector sizes seemingly.
2905
* It seems some cleanup could be done here (like skipping conversion/blend
2908
convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type,
2909
row_type, dst, src_count);
2912
* FIXME: Really should get logic ops / masks out of generic blend / row
2913
* format. Logic ops will definitely not work on the blend float format
2914
* used for SRGB here and I think OpenGL expects this to work as expected
2915
* (that is incoming values converted to srgb then logic op applied).
2917
for (i = 0; i < src_count; ++i) {
2918
dst[i] = lp_build_blend_aos(gallivm,
2919
&variant->key.blend,
2924
has_alpha ? NULL : src_alpha[i],
2926
has_alpha ? NULL : src1_alpha[i],
2928
partial_mask ? src_mask[i] : NULL,
2930
has_alpha ? NULL : blend_alpha,
2932
pad_inline ? 4 : dst_channels);
2935
convert_from_blend_type(gallivm, block_size, out_format_desc,
2936
row_type, dst_type, dst, src_count);
2938
/* Split the blend rows back to memory rows */
2939
if (dst_count > src_count) {
2940
row_type.length = dst_type.length * (dst_count / src_count);
2942
if (src_count == 1) {
2943
dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2944
dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2946
row_type.length /= 2;
2950
dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2);
2951
dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2);
2952
dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2953
dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2955
row_type.length /= 2;
2960
* Store blend result to memory
2963
store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2964
dst, dst_type, dst_count / 4, dst_alignment);
2967
store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2968
dst, dst_type, dst_count, dst_alignment);
2971
if (have_smallfloat_format(dst_type, out_format)) {
2972
lp_build_fpstate_set(gallivm, fpstate);
2976
lp_build_mask_end(&mask_ctx);
2982
* Generate the runtime callable function for the whole fragment pipeline.
2983
* Note that the function which we generate operates on a block of 16
2984
* pixels at at time. The block contains 2x2 quads. Each quad contains
2988
generate_fragment(struct llvmpipe_context *lp,
2989
struct lp_fragment_shader *shader,
2990
struct lp_fragment_shader_variant *variant,
2991
unsigned partial_mask)
2993
struct gallivm_state *gallivm = variant->gallivm;
2994
struct lp_fragment_shader_variant_key *key = &variant->key;
2995
struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS];
2997
struct lp_type fs_type;
2998
struct lp_type blend_type;
2999
LLVMTypeRef fs_elem_type;
3000
LLVMTypeRef blend_vec_type;
3001
LLVMTypeRef arg_types[15];
3002
LLVMTypeRef func_type;
3003
LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
3004
LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context);
3005
LLVMValueRef context_ptr;
3008
LLVMValueRef a0_ptr;
3009
LLVMValueRef dadx_ptr;
3010
LLVMValueRef dady_ptr;
3011
LLVMValueRef color_ptr_ptr;
3012
LLVMValueRef stride_ptr;
3013
LLVMValueRef color_sample_stride_ptr;
3014
LLVMValueRef depth_ptr;
3015
LLVMValueRef depth_stride;
3016
LLVMValueRef depth_sample_stride;
3017
LLVMValueRef mask_input;
3018
LLVMValueRef thread_data_ptr;
3019
LLVMBasicBlockRef block;
3020
LLVMBuilderRef builder;
3021
struct lp_build_sampler_soa *sampler;
3022
struct lp_build_image_soa *image;
3023
struct lp_build_interp_soa_context interp;
3024
LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES];
3025
LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4];
3026
LLVMValueRef function;
3027
LLVMValueRef facing;
3032
boolean cbuf0_write_all;
3033
const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
3034
util_blend_state_is_dual(&key->blend, 0);
3036
assert(lp_native_vector_width / 32 >= 4);
3038
/* Adjust color input interpolation according to flatshade state:
3040
memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]);
3041
for (i = 0; i < shader->info.base.num_inputs; i++) {
3042
if (inputs[i].interp == LP_INTERP_COLOR) {
3044
inputs[i].interp = LP_INTERP_CONSTANT;
3046
inputs[i].interp = LP_INTERP_PERSPECTIVE;
3050
/* check if writes to cbuf[0] are to be copied to all cbufs */
3052
shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS];
3054
/* TODO: actually pick these based on the fs and color buffer
3055
* characteristics. */
3057
memset(&fs_type, 0, sizeof fs_type);
3058
fs_type.floating = TRUE; /* floating point values */
3059
fs_type.sign = TRUE; /* values are signed */
3060
fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
3061
fs_type.width = 32; /* 32-bit float */
3062
fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
3064
memset(&blend_type, 0, sizeof blend_type);
3065
blend_type.floating = FALSE; /* values are integers */
3066
blend_type.sign = FALSE; /* values are unsigned */
3067
blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */
3068
blend_type.width = 8; /* 8-bit ubyte values */
3069
blend_type.length = 16; /* 16 elements per vector */
3072
* Generate the function prototype. Any change here must be reflected in
3073
* lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
3076
fs_elem_type = lp_build_elem_type(gallivm, fs_type);
3078
blend_vec_type = lp_build_vec_type(gallivm, blend_type);
3080
snprintf(func_name, sizeof(func_name), "fs_variant_%s",
3081
partial_mask ? "partial" : "whole");
3083
arg_types[0] = variant->jit_context_ptr_type; /* context */
3084
arg_types[1] = int32_type; /* x */
3085
arg_types[2] = int32_type; /* y */
3086
arg_types[3] = int32_type; /* facing */
3087
arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */
3088
arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */
3089
arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */
3090
arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */
3091
arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */
3092
arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */
3093
arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */
3094
arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */
3095
arg_types[12] = int32_type; /* depth_stride */
3096
arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */
3097
arg_types[14] = int32_type; /* depth sample stride */
3099
func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
3100
arg_types, ARRAY_SIZE(arg_types), 0);
3102
function = LLVMAddFunction(gallivm->module, func_name, func_type);
3103
LLVMSetFunctionCallConv(function, LLVMCCallConv);
3105
variant->function[partial_mask] = function;
3107
/* XXX: need to propagate noalias down into color param now we are
3108
* passing a pointer-to-pointer?
3110
for(i = 0; i < ARRAY_SIZE(arg_types); ++i)
3111
if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
3112
lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
3114
if (variant->gallivm->cache->data_size)
3117
context_ptr = LLVMGetParam(function, 0);
3118
x = LLVMGetParam(function, 1);
3119
y = LLVMGetParam(function, 2);
3120
facing = LLVMGetParam(function, 3);
3121
a0_ptr = LLVMGetParam(function, 4);
3122
dadx_ptr = LLVMGetParam(function, 5);
3123
dady_ptr = LLVMGetParam(function, 6);
3124
color_ptr_ptr = LLVMGetParam(function, 7);
3125
depth_ptr = LLVMGetParam(function, 8);
3126
mask_input = LLVMGetParam(function, 9);
3127
thread_data_ptr = LLVMGetParam(function, 10);
3128
stride_ptr = LLVMGetParam(function, 11);
3129
depth_stride = LLVMGetParam(function, 12);
3130
color_sample_stride_ptr = LLVMGetParam(function, 13);
3131
depth_sample_stride = LLVMGetParam(function, 14);
3133
lp_build_name(context_ptr, "context");
3134
lp_build_name(x, "x");
3135
lp_build_name(y, "y");
3136
lp_build_name(a0_ptr, "a0");
3137
lp_build_name(dadx_ptr, "dadx");
3138
lp_build_name(dady_ptr, "dady");
3139
lp_build_name(color_ptr_ptr, "color_ptr_ptr");
3140
lp_build_name(depth_ptr, "depth");
3141
lp_build_name(mask_input, "mask_input");
3142
lp_build_name(thread_data_ptr, "thread_data");
3143
lp_build_name(stride_ptr, "stride_ptr");
3144
lp_build_name(depth_stride, "depth_stride");
3145
lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr");
3146
lp_build_name(depth_sample_stride, "depth_sample_stride");
3152
block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
3153
builder = gallivm->builder;
3155
LLVMPositionBuilderAtEnd(builder, block);
3158
* Must not count ps invocations if there's a null shader.
3159
* (It would be ok to count with null shader if there's d/s tests,
3160
* but only if there's d/s buffers too, which is different
3161
* to implicit rasterization disable which must not depend
3162
* on the d/s buffers.)
3163
* Could use popcount on mask, but pixel accuracy is not required.
3164
* Could disable if there's no stats query, but maybe not worth it.
3166
if (shader->info.base.num_instructions > 1) {
3167
LLVMValueRef invocs, val;
3168
invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr);
3169
val = LLVMBuildLoad(builder, invocs, "");
3170
val = LLVMBuildAdd(builder, val,
3171
LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0),
3173
LLVMBuildStore(builder, val, invocs);
3176
/* code generated texture sampling */
3177
sampler = lp_llvm_sampler_soa_create(lp_fs_variant_key_samplers(key), key->nr_samplers);
3178
image = lp_llvm_image_soa_create(lp_fs_variant_key_images(key), key->nr_images);
3180
num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */
3181
/* for 1d resources only run "upper half" of stamp */
3182
if (key->resource_1d)
3186
LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs);
3187
LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type);
3188
LLVMValueRef num_loop_samp = lp_build_const_int32(gallivm, num_fs * key->coverage_samples);
3189
LLVMValueRef mask_store = lp_build_array_alloca(gallivm, mask_type,
3190
num_loop_samp, "mask_store");
3192
LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context);
3193
LLVMValueRef glob_sample_pos = LLVMAddGlobal(gallivm->module, LLVMArrayType(flt_type, key->coverage_samples * 2), "");
3194
LLVMValueRef sample_pos_array;
3196
if (key->multisample && key->coverage_samples == 4) {
3197
LLVMValueRef sample_pos_arr[8];
3198
for (unsigned i = 0; i < 4; i++) {
3199
sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]);
3200
sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]);
3202
sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8);
3204
LLVMValueRef sample_pos_arr[2];
3205
sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5);
3206
sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5);
3207
sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2);
3209
LLVMSetInitializer(glob_sample_pos, sample_pos_array);
3211
LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS];
3212
boolean pixel_center_integer =
3213
shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER];
3216
* The shader input interpolation info is not explicitely baked in the
3217
* shader key, but everything it derives from (TGSI, and flatshade) is
3218
* already included in the shader key.
3220
lp_build_interp_soa_init(&interp,
3222
shader->info.base.num_inputs,
3224
pixel_center_integer,
3225
key->coverage_samples, glob_sample_pos,
3229
a0_ptr, dadx_ptr, dady_ptr,
3232
for (i = 0; i < num_fs; i++) {
3233
if (key->multisample) {
3234
LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), "");
3237
* For multisampling, extract the per-sample mask from the incoming 64-bit mask,
3238
* store to the per sample mask storage. Or all of them together to generate
3239
* the fragment shader mask. (sample shading TODO).
3240
* Take the incoming state coverage mask into account.
3242
for (unsigned s = 0; s < key->coverage_samples; s++) {
3243
LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs));
3244
LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store,
3245
&sindexi, 1, "sample_mask_ptr");
3246
LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type,
3247
i*fs_type.length/4, s, mask_input);
3249
LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), "");
3250
LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), "");
3251
smask_bit = LLVMBuildSExt(builder, cmp, int32_type, "");
3252
smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit);
3254
s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
3255
LLVMBuildStore(builder, s_mask, sample_mask_ptr);
3259
LLVMValueRef indexi = lp_build_const_int32(gallivm, i);
3260
LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store,
3261
&indexi, 1, "mask_ptr");
3264
mask = generate_quad_mask(gallivm, fs_type,
3265
i*fs_type.length/4, 0, mask_input);
3268
mask = lp_build_const_int_vec(gallivm, fs_type, ~0);
3270
LLVMBuildStore(builder, mask, mask_ptr);
3274
generate_fs_loop(gallivm,
3284
mask_store, /* output */
3288
depth_sample_stride,
3291
color_sample_stride_ptr,
3295
for (i = 0; i < num_fs; i++) {
3297
for (unsigned s = 0; s < key->coverage_samples; s++) {
3298
int idx = (i + (s * num_fs));
3299
LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3300
ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, "");
3302
fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask");
3305
for (unsigned s = 0; s < key->min_samples; s++) {
3306
/* This is fucked up need to reorganize things */
3307
int idx = s * num_fs + i;
3308
LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3309
for (cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3310
for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3311
ptr = LLVMBuildGEP(builder,
3312
color_store[cbuf * !cbuf0_write_all][chan],
3314
fs_out_color[s][cbuf][chan][i] = ptr;
3317
if (dual_source_blend) {
3318
/* only support one dual source blend target hence always use output 1 */
3319
for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3320
ptr = LLVMBuildGEP(builder,
3321
color_store[1][chan],
3323
fs_out_color[s][1][chan][i] = ptr;
3330
sampler->destroy(sampler);
3331
image->destroy(image);
3332
/* Loop over color outputs / color buffers to do blending.
3334
for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3335
if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) {
3336
LLVMValueRef color_ptr;
3337
LLVMValueRef stride;
3338
LLVMValueRef sample_stride = NULL;
3339
LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
3341
boolean do_branch = ((key->depth.enabled
3342
|| key->stencil[0].enabled
3343
|| key->alpha.enabled)
3344
&& !shader->info.base.uses_kill);
3346
color_ptr = LLVMBuildLoad(builder,
3347
LLVMBuildGEP(builder, color_ptr_ptr,
3351
stride = LLVMBuildLoad(builder,
3352
LLVMBuildGEP(builder, stride_ptr, &index, 1, ""),
3355
if (key->cbuf_nr_samples[cbuf] > 1)
3356
sample_stride = LLVMBuildLoad(builder,
3357
LLVMBuildGEP(builder, color_sample_stride_ptr,
3358
&index, 1, ""), "");
3360
for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) {
3361
unsigned mask_idx = num_fs * (key->multisample ? s : 0);
3362
unsigned out_idx = key->min_samples == 1 ? 0 : s;
3363
LLVMValueRef out_ptr = color_ptr;;
3365
if (sample_stride) {
3366
LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, lp_build_const_int32(gallivm, s), "");
3367
out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, "");
3369
out_ptr = LLVMBuildBitCast(builder, out_ptr, LLVMPointerType(blend_vec_type, 0), "");
3371
lp_build_name(out_ptr, "color_ptr%d", cbuf);
3373
generate_unswizzled_blend(gallivm, cbuf, variant,
3374
key->cbuf_format[cbuf],
3375
num_fs, fs_type, &fs_mask[mask_idx], fs_out_color[out_idx],
3376
context_ptr, out_ptr, stride,
3377
partial_mask, do_branch);
3382
LLVMBuildRetVoid(builder);
3384
gallivm_verify_function(gallivm, function);
3389
dump_fs_variant_key(struct lp_fragment_shader_variant_key *key)
3393
debug_printf("fs variant %p:\n", (void *) key);
3395
if (key->flatshade) {
3396
debug_printf("flatshade = 1\n");
3398
if (key->depth_clamp)
3399
debug_printf("depth_clamp = 1\n");
3401
if (key->multisample) {
3402
debug_printf("multisample = 1\n");
3403
debug_printf("coverage samples = %d\n", key->coverage_samples);
3404
debug_printf("min samples = %d\n", key->min_samples);
3406
for (i = 0; i < key->nr_cbufs; ++i) {
3407
debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i]));
3408
debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]);
3410
if (key->depth.enabled || key->stencil[0].enabled) {
3411
debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format));
3412
debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples);
3414
if (key->depth.enabled) {
3415
debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE));
3416
debug_printf("depth.writemask = %u\n", key->depth.writemask);
3419
for (i = 0; i < 2; ++i) {
3420
if (key->stencil[i].enabled) {
3421
debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE));
3422
debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE));
3423
debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE));
3424
debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE));
3425
debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask);
3426
debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask);
3430
if (key->alpha.enabled) {
3431
debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE));
3434
if (key->occlusion_count) {
3435
debug_printf("occlusion_count = 1\n");
3438
if (key->blend.logicop_enable) {
3439
debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE));
3441
else if (key->blend.rt[0].blend_enable) {
3442
debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE));
3443
debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE));
3444
debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE));
3445
debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE));
3446
debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE));
3447
debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE));
3449
debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask);
3450
if (key->blend.alpha_to_coverage) {
3451
debug_printf("blend.alpha_to_coverage is enabled\n");
3453
for (i = 0; i < key->nr_samplers; ++i) {
3454
const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3455
const struct lp_static_sampler_state *sampler = &samplers[i].sampler_state;
3456
debug_printf("sampler[%u] = \n", i);
3457
debug_printf(" .wrap = %s %s %s\n",
3458
util_str_tex_wrap(sampler->wrap_s, TRUE),
3459
util_str_tex_wrap(sampler->wrap_t, TRUE),
3460
util_str_tex_wrap(sampler->wrap_r, TRUE));
3461
debug_printf(" .min_img_filter = %s\n",
3462
util_str_tex_filter(sampler->min_img_filter, TRUE));
3463
debug_printf(" .min_mip_filter = %s\n",
3464
util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
3465
debug_printf(" .mag_img_filter = %s\n",
3466
util_str_tex_filter(sampler->mag_img_filter, TRUE));
3467
if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
3468
debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
3469
debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
3470
debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
3471
debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
3472
debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
3473
debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
3474
debug_printf(" .reduction_mode = %u\n", sampler->reduction_mode);
3475
debug_printf(" .aniso = %u\n", sampler->aniso);
3477
for (i = 0; i < key->nr_sampler_views; ++i) {
3478
const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3479
const struct lp_static_texture_state *texture = &samplers[i].texture_state;
3480
debug_printf("texture[%u] = \n", i);
3481
debug_printf(" .format = %s\n",
3482
util_format_name(texture->format));
3483
debug_printf(" .target = %s\n",
3484
util_str_tex_target(texture->target, TRUE));
3485
debug_printf(" .level_zero_only = %u\n",
3486
texture->level_zero_only);
3487
debug_printf(" .pot = %u %u %u\n",
3489
texture->pot_height,
3490
texture->pot_depth);
3492
struct lp_image_static_state *images = lp_fs_variant_key_images(key);
3493
for (i = 0; i < key->nr_images; ++i) {
3494
const struct lp_static_texture_state *image = &images[i].image_state;
3495
debug_printf("image[%u] = \n", i);
3496
debug_printf(" .format = %s\n",
3497
util_format_name(image->format));
3498
debug_printf(" .target = %s\n",
3499
util_str_tex_target(image->target, TRUE));
3500
debug_printf(" .level_zero_only = %u\n",
3501
image->level_zero_only);
3502
debug_printf(" .pot = %u %u %u\n",
3510
lp_debug_fs_kind(enum lp_fs_kind kind)
3513
case LP_FS_KIND_GENERAL:
3515
case LP_FS_KIND_BLIT_RGBA:
3517
case LP_FS_KIND_BLIT_RGB1:
3519
case LP_FS_KIND_AERO_MINIFICATION:
3520
return "AERO_MINIFICATION";
3521
case LP_FS_KIND_LLVM_LINEAR:
3522
return "LLVM_LINEAR";
3529
lp_debug_fs_variant(struct lp_fragment_shader_variant *variant)
3531
debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n",
3532
variant->shader->no, variant->no);
3533
if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
3534
tgsi_dump(variant->shader->base.tokens, 0);
3536
nir_print_shader(variant->shader->base.ir.nir, stderr);
3537
dump_fs_variant_key(&variant->key);
3538
debug_printf("variant->opaque = %u\n", variant->opaque);
3539
debug_printf("variant->potentially_opaque = %u\n", variant->potentially_opaque);
3540
debug_printf("variant->blit = %u\n", variant->blit);
3541
debug_printf("shader->kind = %s\n", lp_debug_fs_kind(variant->shader->kind));
3546
lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant *variant,
3547
unsigned char ir_sha1_cache_key[20])
3549
struct blob blob = { 0 };
3554
nir_serialize(&blob, variant->shader->base.ir.nir, true);
3555
ir_binary = blob.data;
3556
ir_size = blob.size;
3558
struct mesa_sha1 ctx;
3559
_mesa_sha1_init(&ctx);
3560
_mesa_sha1_update(&ctx, &variant->key, variant->shader->variant_key_size);
3561
_mesa_sha1_update(&ctx, ir_binary, ir_size);
3562
_mesa_sha1_final(&ctx, ir_sha1_cache_key);
3568
* Generate a new fragment shader variant from the shader code and
3569
* other state indicated by the key.
3571
static struct lp_fragment_shader_variant *
3572
generate_variant(struct llvmpipe_context *lp,
3573
struct lp_fragment_shader *shader,
3574
const struct lp_fragment_shader_variant_key *key)
3576
struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen);
3577
struct lp_fragment_shader_variant *variant;
3578
const struct util_format_description *cbuf0_format_desc = NULL;
3579
boolean fullcolormask;
3582
char module_name[64];
3583
unsigned char ir_sha1_cache_key[20];
3584
struct lp_cached_code cached = { 0 };
3585
bool needs_caching = false;
3586
variant = MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key);
3590
memset(variant, 0, sizeof(*variant));
3591
snprintf(module_name, sizeof(module_name), "fs%u_variant%u",
3592
shader->no, shader->variants_created);
3594
pipe_reference_init(&variant->reference, 1);
3595
lp_fs_reference(lp, &variant->shader, shader);
3597
memcpy(&variant->key, key, shader->variant_key_size);
3599
if (shader->base.ir.nir) {
3600
lp_fs_get_ir_cache_key(variant, ir_sha1_cache_key);
3602
lp_disk_cache_find_shader(screen, &cached, ir_sha1_cache_key);
3603
if (!cached.data_size)
3604
needs_caching = true;
3606
variant->gallivm = gallivm_create(module_name, lp->context, &cached);
3607
if (!variant->gallivm) {
3612
variant->list_item_global.base = variant;
3613
variant->list_item_local.base = variant;
3614
variant->no = shader->variants_created++;
3619
* Determine whether we are touching all channels in the color buffer.
3621
fullcolormask = FALSE;
3622
if (key->nr_cbufs == 1) {
3623
cbuf0_format_desc = util_format_description(key->cbuf_format[0]);
3624
fullcolormask = util_format_colormask_full(cbuf0_format_desc, key->blend.rt[0].colormask);
3627
/* The scissor is ignored here as only tiles inside the scissoring
3628
* rectangle will refer to this */
3631
!key->stencil[0].enabled &&
3632
!key->alpha.enabled &&
3633
!key->multisample &&
3634
!key->blend.alpha_to_coverage &&
3635
!key->depth.enabled &&
3636
!shader->info.base.uses_kill &&
3637
!shader->info.base.writes_samplemask &&
3638
!shader->info.base.uses_fbfetch;
3642
!key->blend.logicop_enable &&
3643
!key->blend.rt[0].blend_enable
3646
variant->potentially_opaque =
3648
!key->blend.logicop_enable &&
3649
key->blend.rt[0].blend_enable &&
3650
key->blend.rt[0].rgb_func == PIPE_BLEND_ADD &&
3651
key->blend.rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_INV_SRC_ALPHA &&
3652
key->blend.rt[0].alpha_func == key->blend.rt[0].rgb_func &&
3653
key->blend.rt[0].alpha_dst_factor == key->blend.rt[0].rgb_dst_factor &&
3654
shader->base.type == PIPE_SHADER_IR_TGSI &&
3656
* FIXME: for NIR, all of the fields of info.xxx (except info.base)
3657
* are zeros, hence shader analysis (here and elsewhere) using these
3658
* bits cannot work and will silently fail (cbuf is the only pointer
3659
* field, hence causing a crash).
3661
shader->info.cbuf[0][3].file != TGSI_FILE_NULL
3664
/* We only care about opaque blits for now */
3665
if (variant->opaque &&
3666
(shader->kind == LP_FS_KIND_BLIT_RGBA ||
3667
shader->kind == LP_FS_KIND_BLIT_RGB1)) {
3668
unsigned target, min_img_filter, mag_img_filter, min_mip_filter;
3669
enum pipe_format texture_format;
3670
struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0);
3672
texture_format = samp0->texture_state.format;
3673
target = samp0->texture_state.target;
3674
min_img_filter = samp0->sampler_state.min_img_filter;
3675
mag_img_filter = samp0->sampler_state.mag_img_filter;
3676
if (samp0->texture_state.level_zero_only) {
3677
min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
3679
min_mip_filter = samp0->sampler_state.min_mip_filter;
3682
if (target == PIPE_TEXTURE_2D &&
3683
min_img_filter == PIPE_TEX_FILTER_NEAREST &&
3684
mag_img_filter == PIPE_TEX_FILTER_NEAREST &&
3685
min_mip_filter == PIPE_TEX_MIPFILTER_NONE &&
3687
util_is_format_compatible(util_format_description(texture_format),
3688
cbuf0_format_desc)) ||
3689
(shader->kind == LP_FS_KIND_BLIT_RGB1 &&
3690
(texture_format == PIPE_FORMAT_B8G8R8A8_UNORM ||
3691
texture_format == PIPE_FORMAT_B8G8R8X8_UNORM) &&
3692
(key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3693
key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM))))
3698
/* Whether this is a candidate for the linear path */
3700
!key->stencil[0].enabled &&
3701
!key->depth.enabled &&
3702
!shader->info.base.uses_kill &&
3703
!key->blend.logicop_enable &&
3704
(key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3705
key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM);
3707
memcpy(&variant->key, key, sizeof *key);
3709
if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3710
lp_debug_fs_variant(variant);
3713
llvmpipe_fs_variant_fastpath(variant);
3715
lp_jit_init_types(variant);
3717
if (variant->jit_function[RAST_EDGE_TEST] == NULL)
3718
generate_fragment(lp, shader, variant, RAST_EDGE_TEST);
3720
if (variant->jit_function[RAST_WHOLE] == NULL) {
3721
if (variant->opaque) {
3722
/* Specialized shader, which doesn't need to read the color buffer. */
3723
generate_fragment(lp, shader, variant, RAST_WHOLE);
3728
/* Currently keeping both the old fastpaths and new linear path
3729
* active. The older code is still somewhat faster for the cases
3732
* XXX: consider restricting this to aero-mode only.
3734
if (fullcolormask &&
3735
!key->alpha.enabled &&
3736
!key->blend.alpha_to_coverage) {
3737
llvmpipe_fs_variant_linear_fastpath(variant);
3740
/* If the original fastpath doesn't cover this variant, try the new
3743
if (variant->jit_linear == NULL) {
3744
if (shader->kind == LP_FS_KIND_BLIT_RGBA ||
3745
shader->kind == LP_FS_KIND_BLIT_RGB1 ||
3746
shader->kind == LP_FS_KIND_LLVM_LINEAR) {
3747
llvmpipe_fs_variant_linear_llvm(lp, shader, variant);
3751
if (LP_DEBUG & DEBUG_LINEAR) {
3752
lp_debug_fs_variant(variant);
3753
debug_printf(" ----> no linear path for this variant\n");
3758
* Compile everything
3761
gallivm_compile_module(variant->gallivm);
3763
variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
3765
if (variant->function[RAST_EDGE_TEST]) {
3766
variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func)
3767
gallivm_jit_function(variant->gallivm,
3768
variant->function[RAST_EDGE_TEST]);
3771
if (variant->function[RAST_WHOLE]) {
3772
variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func)
3773
gallivm_jit_function(variant->gallivm,
3774
variant->function[RAST_WHOLE]);
3775
} else if (!variant->jit_function[RAST_WHOLE]) {
3776
variant->jit_function[RAST_WHOLE] = variant->jit_function[RAST_EDGE_TEST];
3780
if (variant->linear_function) {
3781
variant->jit_linear_llvm = (lp_jit_linear_llvm_func)
3782
gallivm_jit_function(variant->gallivm, variant->linear_function);
3786
* This must be done after LLVM compilation, as it will call the JIT'ed
3787
* code to determine active inputs.
3789
lp_linear_check_variant(variant);
3792
if (needs_caching) {
3793
lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key);
3796
gallivm_free_ir(variant->gallivm);
3803
llvmpipe_create_fs_state(struct pipe_context *pipe,
3804
const struct pipe_shader_state *templ)
3806
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3807
struct lp_fragment_shader *shader;
3809
int nr_sampler_views;
3813
shader = CALLOC_STRUCT(lp_fragment_shader);
3817
pipe_reference_init(&shader->reference, 1);
3818
shader->no = fs_no++;
3819
make_empty_list(&shader->variants);
3821
shader->base.type = templ->type;
3822
if (templ->type == PIPE_SHADER_IR_TGSI) {
3823
/* get/save the summary info for this shader */
3824
lp_build_tgsi_info(templ->tokens, &shader->info);
3826
/* we need to keep a local copy of the tokens */
3827
shader->base.tokens = tgsi_dup_tokens(templ->tokens);
3829
shader->base.ir.nir = templ->ir.nir;
3830
nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true);
3833
shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ);
3834
if (shader->draw_data == NULL) {
3835
FREE((void *) shader->base.tokens);
3840
nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
3841
nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
3842
nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
3843
shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers, nr_sampler_views), nr_images);
3845
for (i = 0; i < shader->info.base.num_inputs; i++) {
3846
shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i];
3847
shader->inputs[i].location = shader->info.base.input_interpolate_loc[i];
3849
switch (shader->info.base.input_interpolate[i]) {
3850
case TGSI_INTERPOLATE_CONSTANT:
3851
shader->inputs[i].interp = LP_INTERP_CONSTANT;
3853
case TGSI_INTERPOLATE_LINEAR:
3854
shader->inputs[i].interp = LP_INTERP_LINEAR;
3856
case TGSI_INTERPOLATE_PERSPECTIVE:
3857
shader->inputs[i].interp = LP_INTERP_PERSPECTIVE;
3859
case TGSI_INTERPOLATE_COLOR:
3860
shader->inputs[i].interp = LP_INTERP_COLOR;
3867
switch (shader->info.base.input_semantic_name[i]) {
3868
case TGSI_SEMANTIC_FACE:
3869
shader->inputs[i].interp = LP_INTERP_FACING;
3871
case TGSI_SEMANTIC_POSITION:
3872
/* Position was already emitted above
3874
shader->inputs[i].interp = LP_INTERP_POSITION;
3875
shader->inputs[i].src_index = 0;
3879
/* XXX this is a completely pointless index map... */
3880
shader->inputs[i].src_index = i+1;
3883
if (LP_DEBUG & DEBUG_TGSI && templ->type == PIPE_SHADER_IR_TGSI) {
3885
debug_printf("llvmpipe: Create fragment shader #%u %p:\n",
3886
shader->no, (void *) shader);
3887
tgsi_dump(templ->tokens, 0);
3888
debug_printf("usage masks:\n");
3889
for (attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) {
3890
unsigned usage_mask = shader->info.base.input_usage_mask[attrib];
3891
debug_printf(" IN[%u].%s%s%s%s\n",
3893
usage_mask & TGSI_WRITEMASK_X ? "x" : "",
3894
usage_mask & TGSI_WRITEMASK_Y ? "y" : "",
3895
usage_mask & TGSI_WRITEMASK_Z ? "z" : "",
3896
usage_mask & TGSI_WRITEMASK_W ? "w" : "");
3901
/* This will put a derived copy of the tokens into shader->base.tokens */
3902
if (templ->type == PIPE_SHADER_IR_TGSI)
3903
llvmpipe_fs_analyse(shader, templ->tokens);
3905
llvmpipe_fs_analyse_nir(shader);
3912
llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs)
3914
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3915
struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs;
3916
if (llvmpipe->fs == lp_fs)
3919
draw_bind_fragment_shader(llvmpipe->draw,
3920
(lp_fs ? lp_fs->draw_data : NULL));
3922
lp_fs_reference(llvmpipe, &llvmpipe->fs, lp_fs);
3924
/* invalidate the setup link, NEW_FS will make it update */
3925
lp_setup_set_fs_variant(llvmpipe->setup, NULL);
3926
llvmpipe->dirty |= LP_NEW_FS;
3931
* Remove shader variant from two lists: the shader's variant list
3932
* and the context's variant list.
3936
void llvmpipe_remove_shader_variant(struct llvmpipe_context *lp,
3937
struct lp_fragment_shader_variant *variant)
3939
if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3940
debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u "
3941
"v total cached %u inst %u total inst %u\n",
3942
variant->shader->no, variant->no,
3943
variant->shader->variants_created,
3944
variant->shader->variants_cached,
3945
lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs);
3948
/* remove from shader's list */
3949
remove_from_list(&variant->list_item_local);
3950
variant->shader->variants_cached--;
3952
/* remove from context's list */
3953
remove_from_list(&variant->list_item_global);
3954
lp->nr_fs_variants--;
3955
lp->nr_fs_instrs -= variant->nr_instrs;
3959
llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp,
3960
struct lp_fragment_shader_variant *variant)
3962
gallivm_destroy(variant->gallivm);
3964
lp_fs_reference(lp, &variant->shader, NULL);
3970
llvmpipe_destroy_fs(struct llvmpipe_context *llvmpipe,
3971
struct lp_fragment_shader *shader)
3973
/* Delete draw module's data */
3974
draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data);
3976
if (shader->base.ir.nir)
3977
ralloc_free(shader->base.ir.nir);
3978
assert(shader->variants_cached == 0);
3979
FREE((void *) shader->base.tokens);
3984
llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
3986
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3987
struct lp_fragment_shader *shader = fs;
3988
struct lp_fs_variant_list_item *li;
3990
/* Delete all the variants */
3991
li = first_elem(&shader->variants);
3992
while(!at_end(&shader->variants, li)) {
3993
struct lp_fs_variant_list_item *next = next_elem(li);
3994
struct lp_fragment_shader_variant *variant;
3996
llvmpipe_remove_shader_variant(llvmpipe, li->base);
3997
lp_fs_variant_reference(llvmpipe, &variant, NULL);
4001
lp_fs_reference(llvmpipe, &shader, NULL);
4005
llvmpipe_set_constant_buffer(struct pipe_context *pipe,
4006
enum pipe_shader_type shader, uint index,
4007
bool take_ownership,
4008
const struct pipe_constant_buffer *cb)
4010
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4011
struct pipe_constant_buffer *constants = &llvmpipe->constants[shader][index];
4013
assert(shader < PIPE_SHADER_TYPES);
4014
assert(index < ARRAY_SIZE(llvmpipe->constants[shader]));
4016
/* note: reference counting */
4017
util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb,
4020
/* user_buffer is only valid until the next set_constant_buffer (at most,
4021
* possibly until shader deletion), so we need to upload it now to make sure
4022
* it doesn't get updated/freed out from under us.
4024
if (constants->user_buffer) {
4025
u_upload_data(llvmpipe->pipe.const_uploader, 0, constants->buffer_size, 16,
4026
constants->user_buffer, &constants->buffer_offset,
4027
&constants->buffer);
4029
if (constants->buffer) {
4030
if (!(constants->buffer->bind & PIPE_BIND_CONSTANT_BUFFER)) {
4031
debug_printf("Illegal set constant without bind flag\n");
4032
constants->buffer->bind |= PIPE_BIND_CONSTANT_BUFFER;
4036
if (shader == PIPE_SHADER_VERTEX ||
4037
shader == PIPE_SHADER_GEOMETRY ||
4038
shader == PIPE_SHADER_TESS_CTRL ||
4039
shader == PIPE_SHADER_TESS_EVAL) {
4040
/* Pass the constants to the 'draw' module */
4041
const unsigned size = cb ? cb->buffer_size : 0;
4043
const ubyte *data = NULL;
4044
if (constants->buffer)
4045
data = (ubyte *) llvmpipe_resource_data(constants->buffer) + constants->buffer_offset;
4047
draw_set_mapped_constant_buffer(llvmpipe->draw, shader,
4050
else if (shader == PIPE_SHADER_COMPUTE)
4051
llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS;
4053
llvmpipe->dirty |= LP_NEW_FS_CONSTANTS;
4057
llvmpipe_set_shader_buffers(struct pipe_context *pipe,
4058
enum pipe_shader_type shader, unsigned start_slot,
4059
unsigned count, const struct pipe_shader_buffer *buffers,
4060
unsigned writable_bitmask)
4062
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4064
for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4065
const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL;
4067
util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer);
4069
if (buffer && buffer->buffer) {
4070
boolean read_only = !(writable_bitmask & (1 << idx));
4071
llvmpipe_flush_resource(pipe, buffer->buffer, 0, read_only, false,
4075
if (shader == PIPE_SHADER_VERTEX ||
4076
shader == PIPE_SHADER_GEOMETRY ||
4077
shader == PIPE_SHADER_TESS_CTRL ||
4078
shader == PIPE_SHADER_TESS_EVAL) {
4079
const unsigned size = buffer ? buffer->buffer_size : 0;
4080
const ubyte *data = NULL;
4081
if (buffer && buffer->buffer)
4082
data = (ubyte *) llvmpipe_resource_data(buffer->buffer);
4084
data += buffer->buffer_offset;
4085
draw_set_mapped_shader_buffer(llvmpipe->draw, shader,
4087
} else if (shader == PIPE_SHADER_COMPUTE) {
4088
llvmpipe->cs_dirty |= LP_CSNEW_SSBOS;
4089
} else if (shader == PIPE_SHADER_FRAGMENT) {
4090
llvmpipe->fs_ssbo_write_mask &= ~(((1 << count) - 1) << start_slot);
4091
llvmpipe->fs_ssbo_write_mask |= writable_bitmask << start_slot;
4092
llvmpipe->dirty |= LP_NEW_FS_SSBOS;
4098
llvmpipe_set_shader_images(struct pipe_context *pipe,
4099
enum pipe_shader_type shader, unsigned start_slot,
4100
unsigned count, unsigned unbind_num_trailing_slots,
4101
const struct pipe_image_view *images)
4103
struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4106
draw_flush(llvmpipe->draw);
4107
for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4108
const struct pipe_image_view *image = images ? &images[idx] : NULL;
4110
util_copy_image_view(&llvmpipe->images[shader][i], image);
4112
if (image && image->resource) {
4113
bool read_only = !(image->access & PIPE_IMAGE_ACCESS_WRITE);
4114
llvmpipe_flush_resource(pipe, image->resource, 0, read_only, false,
4119
llvmpipe->num_images[shader] = start_slot + count;
4120
if (shader == PIPE_SHADER_VERTEX ||
4121
shader == PIPE_SHADER_GEOMETRY ||
4122
shader == PIPE_SHADER_TESS_CTRL ||
4123
shader == PIPE_SHADER_TESS_EVAL) {
4124
draw_set_images(llvmpipe->draw,
4126
llvmpipe->images[shader],
4127
start_slot + count);
4128
} else if (shader == PIPE_SHADER_COMPUTE)
4129
llvmpipe->cs_dirty |= LP_CSNEW_IMAGES;
4131
llvmpipe->dirty |= LP_NEW_FS_IMAGES;
4133
if (unbind_num_trailing_slots) {
4134
llvmpipe_set_shader_images(pipe, shader, start_slot + count,
4135
unbind_num_trailing_slots, 0, NULL);
4140
* Return the blend factor equivalent to a destination alpha of one.
4142
static inline unsigned
4143
force_dst_alpha_one(unsigned factor, boolean clamped_zero)
4146
case PIPE_BLENDFACTOR_DST_ALPHA:
4147
return PIPE_BLENDFACTOR_ONE;
4148
case PIPE_BLENDFACTOR_INV_DST_ALPHA:
4149
return PIPE_BLENDFACTOR_ZERO;
4150
case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
4152
return PIPE_BLENDFACTOR_ZERO;
4154
return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
4162
* We need to generate several variants of the fragment pipeline to match
4163
* all the combinations of the contributing state atoms.
4165
* TODO: there is actually no reason to tie this to context state -- the
4166
* generated code could be cached globally in the screen.
4168
static struct lp_fragment_shader_variant_key *
4169
make_variant_key(struct llvmpipe_context *lp,
4170
struct lp_fragment_shader *shader,
4174
struct lp_fragment_shader_variant_key *key;
4176
key = (struct lp_fragment_shader_variant_key *)store;
4178
memset(key, 0, sizeof(*key));
4180
if (lp->framebuffer.zsbuf) {
4181
enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format;
4182
const struct util_format_description *zsbuf_desc =
4183
util_format_description(zsbuf_format);
4185
if (lp->depth_stencil->depth_enabled &&
4186
util_format_has_depth(zsbuf_desc)) {
4187
key->zsbuf_format = zsbuf_format;
4188
key->depth.enabled = lp->depth_stencil->depth_enabled;
4189
key->depth.writemask = lp->depth_stencil->depth_writemask;
4190
key->depth.func = lp->depth_stencil->depth_func;
4192
if (lp->depth_stencil->stencil[0].enabled &&
4193
util_format_has_stencil(zsbuf_desc)) {
4194
key->zsbuf_format = zsbuf_format;
4195
memcpy(&key->stencil, &lp->depth_stencil->stencil, sizeof key->stencil);
4197
if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) {
4198
key->resource_1d = TRUE;
4200
key->zsbuf_nr_samples = util_res_sample_count(lp->framebuffer.zsbuf->texture);
4204
* Propagate the depth clamp setting from the rasterizer state.
4206
key->depth_clamp = lp->rasterizer->depth_clamp;
4208
/* alpha test only applies if render buffer 0 is non-integer (or does not exist) */
4209
if (!lp->framebuffer.nr_cbufs ||
4210
!lp->framebuffer.cbufs[0] ||
4211
!util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) {
4212
key->alpha.enabled = lp->depth_stencil->alpha_enabled;
4214
if(key->alpha.enabled)
4215
key->alpha.func = lp->depth_stencil->alpha_func;
4216
/* alpha.ref_value is passed in jit_context */
4218
key->flatshade = lp->rasterizer->flatshade;
4219
key->multisample = lp->rasterizer->multisample;
4220
key->no_ms_sample_mask_out = lp->rasterizer->no_ms_sample_mask_out;
4221
if (lp->active_occlusion_queries && !lp->queries_disabled) {
4222
key->occlusion_count = TRUE;
4225
memcpy(&key->blend, lp->blend, sizeof key->blend);
4227
key->coverage_samples = 1;
4228
key->min_samples = 1;
4229
if (key->multisample) {
4230
key->coverage_samples = util_framebuffer_get_num_samples(&lp->framebuffer);
4231
key->min_samples = lp->min_samples == 1 ? 1 : key->coverage_samples;
4233
key->nr_cbufs = lp->framebuffer.nr_cbufs;
4235
if (!key->blend.independent_blend_enable) {
4236
/* we always need independent blend otherwise the fixups below won't work */
4237
for (i = 1; i < key->nr_cbufs; i++) {
4238
memcpy(&key->blend.rt[i], &key->blend.rt[0], sizeof(key->blend.rt[0]));
4240
key->blend.independent_blend_enable = 1;
4243
for (i = 0; i < lp->framebuffer.nr_cbufs; i++) {
4244
struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i];
4246
if (lp->framebuffer.cbufs[i]) {
4247
enum pipe_format format = lp->framebuffer.cbufs[i]->format;
4248
const struct util_format_description *format_desc;
4250
key->cbuf_format[i] = format;
4251
key->cbuf_nr_samples[i] = util_res_sample_count(lp->framebuffer.cbufs[i]->texture);
4254
* Figure out if this is a 1d resource. Note that OpenGL allows crazy
4255
* mixing of 2d textures with height 1 and 1d textures, so make sure
4256
* we pick 1d if any cbuf or zsbuf is 1d.
4258
if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) {
4259
key->resource_1d = TRUE;
4262
format_desc = util_format_description(format);
4263
assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
4264
format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
4267
* Mask out color channels not present in the color buffer.
4269
blend_rt->colormask &= util_format_colormask(format_desc);
4272
* Disable blend for integer formats.
4274
if (util_format_is_pure_integer(format)) {
4275
blend_rt->blend_enable = 0;
4279
* Our swizzled render tiles always have an alpha channel, but the
4280
* linear render target format often does not, so force here the dst
4283
* This is not a mere optimization. Wrong results will be produced if
4284
* the dst alpha is used, the dst format does not have alpha, and the
4285
* previous rendering was not flushed from the swizzled to linear
4286
* buffer. For example, NonPowTwo DCT.
4288
* TODO: This should be generalized to all channels for better
4289
* performance, but only alpha causes correctness issues.
4291
* Also, force rgb/alpha func/factors match, to make AoS blending
4294
if (format_desc->swizzle[3] > PIPE_SWIZZLE_W ||
4295
format_desc->swizzle[3] == format_desc->swizzle[0]) {
4296
/* Doesn't cover mixed snorm/unorm but can't render to them anyway */
4297
boolean clamped_zero = !util_format_is_float(format) &&
4298
!util_format_is_snorm(format);
4299
blend_rt->rgb_src_factor =
4300
force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero);
4301
blend_rt->rgb_dst_factor =
4302
force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero);
4303
blend_rt->alpha_func = blend_rt->rgb_func;
4304
blend_rt->alpha_src_factor = blend_rt->rgb_src_factor;
4305
blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor;
4309
/* no color buffer for this fragment output */
4310
key->cbuf_format[i] = PIPE_FORMAT_NONE;
4311
key->cbuf_nr_samples[i] = 0;
4312
blend_rt->colormask = 0x0;
4313
blend_rt->blend_enable = 0;
4317
/* This value will be the same for all the variants of a given shader:
4319
key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
4321
if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1)
4322
key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
4324
struct lp_sampler_static_state *fs_sampler;
4326
fs_sampler = lp_fs_variant_key_samplers(key);
4328
memset(fs_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler);
4330
for(i = 0; i < key->nr_samplers; ++i) {
4331
if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
4332
lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state,
4333
lp->samplers[PIPE_SHADER_FRAGMENT][i]);
4338
* XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
4339
* are dx10-style? Can't really have mixed opcodes, at least not
4340
* if we want to skip the holes here (without rescanning tgsi).
4342
if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
4343
for(i = 0; i < key->nr_sampler_views; ++i) {
4345
* Note sview may exceed what's representable by file_mask.
4346
* This will still work, the only downside is that not actually
4347
* used views may be included in the shader key.
4349
if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
4350
lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4351
lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4356
key->nr_sampler_views = key->nr_samplers;
4357
for(i = 0; i < key->nr_sampler_views; ++i) {
4358
if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
4359
lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4360
lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4365
struct lp_image_static_state *lp_image;
4366
lp_image = lp_fs_variant_key_images(key);
4367
key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
4368
for (i = 0; i < key->nr_images; ++i) {
4369
if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
4370
lp_sampler_static_texture_state_image(&lp_image[i].image_state,
4371
&lp->images[PIPE_SHADER_FRAGMENT][i]);
4375
if (shader->kind == LP_FS_KIND_AERO_MINIFICATION) {
4376
struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0);
4378
samp0->sampler_state.min_img_filter = PIPE_TEX_FILTER_NEAREST;
4379
samp0->sampler_state.mag_img_filter = PIPE_TEX_FILTER_NEAREST;
4387
* Update fragment shader state. This is called just prior to drawing
4388
* something when some fragment-related state has changed.
4391
llvmpipe_update_fs(struct llvmpipe_context *lp)
4393
struct lp_fragment_shader *shader = lp->fs;
4394
struct lp_fragment_shader_variant_key *key;
4395
struct lp_fragment_shader_variant *variant = NULL;
4396
struct lp_fs_variant_list_item *li;
4397
char store[LP_FS_MAX_VARIANT_KEY_SIZE];
4399
key = make_variant_key(lp, shader, store);
4401
/* Search the variants for one which matches the key */
4402
li = first_elem(&shader->variants);
4403
while(!at_end(&shader->variants, li)) {
4404
if(memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
4412
/* Move this variant to the head of the list to implement LRU
4413
* deletion of shader's when we have too many.
4415
move_to_head(&lp->fs_variants_list, &variant->list_item_global);
4418
/* variant not found, create it now */
4421
unsigned variants_to_cull;
4423
if (LP_DEBUG & DEBUG_FS) {
4424
debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
4427
lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0);
4430
/* First, check if we've exceeded the max number of shader variants.
4431
* If so, free 6.25% of them (the least recently used ones).
4433
variants_to_cull = lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
4435
if (variants_to_cull ||
4436
lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
4437
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
4438
debug_printf("Evicting FS: %u fs variants,\t%u total variants,"
4439
"\t%u instrs,\t%u instrs/variant\n",
4440
shader->variants_cached,
4441
lp->nr_fs_variants, lp->nr_fs_instrs,
4442
lp->nr_fs_instrs / lp->nr_fs_variants);
4446
* We need to re-check lp->nr_fs_variants because an arbitrarliy large
4447
* number of shader variants (potentially all of them) could be
4448
* pending for destruction on flush.
4451
for (i = 0; i < variants_to_cull || lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
4452
struct lp_fs_variant_list_item *item;
4453
if (is_empty_list(&lp->fs_variants_list)) {
4456
item = last_elem(&lp->fs_variants_list);
4459
llvmpipe_remove_shader_variant(lp, item->base);
4460
struct lp_fragment_shader_variant *variant = item->base;
4461
lp_fs_variant_reference(lp, &variant, NULL);
4466
* Generate the new variant.
4469
variant = generate_variant(lp, shader, key);
4472
LP_COUNT_ADD(llvm_compile_time, dt);
4473
LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
4475
/* Put the new variant into the list */
4477
insert_at_head(&shader->variants, &variant->list_item_local);
4478
insert_at_head(&lp->fs_variants_list, &variant->list_item_global);
4479
lp->nr_fs_variants++;
4480
lp->nr_fs_instrs += variant->nr_instrs;
4481
shader->variants_cached++;
4485
/* Bind this variant */
4486
lp_setup_set_fs_variant(lp->setup, variant);
4494
llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe)
4496
llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state;
4497
llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state;
4498
llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state;
4500
llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer;
4502
llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers;
4503
llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images;