« back to all changes in this revision
Viewing changes to pixman/pixman-sse2.c
- browse files at revision 4
- compare with another revision
- download diff
- download tarball
- view history from revision 4
- Committer: Bazaar Package Importer
- Author(s): Julien Cristau, Brice Goglin, Julien Cristau
- Date: 2009-02-15 23:01:14 UTC
- mfrom: (1.2.1 upstream) (2.1.9 jaunty)
- Revision ID: james.westby@ubuntu.com-20090215230114-2gqbgmk1lq0csrsm
Tags: 0.14.0-1
[ Brice Goglin ]
* Update upstream URL in debian/copyright.
* Install the upstream ChangeLog.
[ Julien Cristau ]
* New upstream release.
* Update upstream URL in debian/copyright.
* Install the upstream ChangeLog.
[ Julien Cristau ]
* New upstream release.
- files added:
- pixman-1-uninstalled.pc.in
- files removed:
- debian/libpixman-1-0.shlibs
- files modified:
- ChangeLog
added
removed
pixman/pixman-sse2.c
1
/*
2
* Copyright © 2008 Rodrigo Kumpera
3
* Copyright © 2008 André Tupinambá
4
*
5
* Permission to use, copy, modify, distribute, and sell this software and its
6
* documentation for any purpose is hereby granted without fee, provided that
7
* the above copyright notice appear in all copies and that both that
8
* copyright notice and this permission notice appear in supporting
9
* documentation, and that the name of Red Hat not be used in advertising or
10
* publicity pertaining to distribution of the software without specific,
11
* written prior permission. Red Hat makes no representations about the
12
* suitability of this software for any purpose. It is provided "as is"
13
* without express or implied warranty.
14
*
15
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
16
* SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
17
* FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
18
* SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
20
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
21
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
22
* SOFTWARE.
23
*
24
* Author: Rodrigo Kumpera (kumpera@gmail.com)
25
* André Tupinambá (andrelrt@gmail.com)
26
*
27
* Based on work by Owen Taylor and Søren Sandmann
28
*/
29
#ifdef HAVE_CONFIG_H
30
#include <config.h>
31
#endif
32
33
#include <mmintrin.h>
34
#include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
35
#include <emmintrin.h> /* for SSE2 intrinsics */
36
37
#include "pixman-sse2.h"
38
39
#ifdef USE_SSE2
40
41
/* -------------------------------------------------------------------------------------------------
42
* Locals
43
*/
44
45
static __m64 xMask0080;
46
static __m64 xMask00ff;
47
static __m64 xMask0101;
48
static __m64 xMaskAlpha;
49
50
static __m64 xMask565rgb;
51
static __m64 xMask565Unpack;
52
53
static __m128i Mask0080;
54
static __m128i Mask00ff;
55
static __m128i Mask0101;
56
static __m128i Maskffff;
57
static __m128i Maskff000000;
58
static __m128i MaskAlpha;
59
60
static __m128i Mask565r;
61
static __m128i Mask565g1, Mask565g2;
62
static __m128i Mask565b;
63
static __m128i MaskRed;
64
static __m128i MaskGreen;
65
static __m128i MaskBlue;
66
67
static __m128i Mask565FixRB;
68
static __m128i Mask565FixG;
69
70
/* -------------------------------------------------------------------------------------------------
71
* SSE2 Inlines
72
*/
73
static force_inline __m128i
74
unpack_32_1x128 (uint32_t data)
75
{
76
return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128());
77
}
78
79
static force_inline void
80
unpack_128_2x128 (__m128i data, __m128i* dataLo, __m128i* dataHi)
81
{
82
*dataLo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
83
*dataHi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
84
}
85
86
static force_inline __m128i
87
unpack565to8888 (__m128i lo)
88
{
89
__m128i r, g, b, rb, t;
90
91
r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), MaskRed);
92
g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), MaskGreen);
93
b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), MaskBlue);
94
95
rb = _mm_or_si128 (r, b);
96
t = _mm_and_si128 (rb, Mask565FixRB);
97
t = _mm_srli_epi32 (t, 5);
98
rb = _mm_or_si128 (rb, t);
99
100
t = _mm_and_si128 (g, Mask565FixG);
101
t = _mm_srli_epi32 (t, 6);
102
g = _mm_or_si128 (g, t);
103
104
return _mm_or_si128 (rb, g);
105
}
106
107
static force_inline void
108
unpack565_128_4x128 (__m128i data, __m128i* data0, __m128i* data1, __m128i* data2, __m128i* data3)
109
{
110
__m128i lo, hi;
111
112
lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
113
hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
114
115
lo = unpack565to8888 (lo);
116
hi = unpack565to8888 (hi);
117
118
unpack_128_2x128 (lo, data0, data1);
119
unpack_128_2x128 (hi, data2, data3);
120
}
121
122
static force_inline uint16_t
123
pack565_32_16 (uint32_t pixel)
124
{
125
return (uint16_t) (((pixel>>8) & 0xf800) | ((pixel>>5) & 0x07e0) | ((pixel>>3) & 0x001f));
126
}
127
128
static force_inline __m128i
129
pack_2x128_128 (__m128i lo, __m128i hi)
130
{
131
return _mm_packus_epi16 (lo, hi);
132
}
133
134
static force_inline __m128i
135
pack565_2x128_128 (__m128i lo, __m128i hi)
136
{
137
__m128i data;
138
__m128i r, g1, g2, b;
139
140
data = pack_2x128_128 ( lo, hi );
141
142
r = _mm_and_si128 (data , Mask565r);
143
g1 = _mm_and_si128 (_mm_slli_epi32 (data , 3), Mask565g1);
144
g2 = _mm_and_si128 (_mm_srli_epi32 (data , 5), Mask565g2);
145
b = _mm_and_si128 (_mm_srli_epi32 (data , 3), Mask565b);
146
147
return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
148
}
149
150
static force_inline __m128i
151
pack565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
152
{
153
return _mm_packus_epi16 (pack565_2x128_128 (*xmm0, *xmm1), pack565_2x128_128 (*xmm2, *xmm3));
154
}
155
156
static force_inline uint32_t
157
packAlpha (__m128i x)
158
{
159
return _mm_cvtsi128_si32 (_mm_packus_epi16 (_mm_packus_epi16 (_mm_srli_epi32 (x, 24),
160
_mm_setzero_si128 ()),
161
_mm_setzero_si128 ()));
162
}
163
164
static force_inline __m128i
165
expandPixel_32_1x128 (uint32_t data)
166
{
167
return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE(1, 0, 1, 0));
168
}
169
170
static force_inline __m128i
171
expandAlpha_1x128 (__m128i data)
172
{
173
return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data, _MM_SHUFFLE(3, 3, 3, 3)), _MM_SHUFFLE(3, 3, 3, 3));
174
}
175
176
static force_inline void
177
expandAlpha_2x128 (__m128i dataLo, __m128i dataHi, __m128i* alphaLo, __m128i* alphaHi)
178
{
179
__m128i lo, hi;
180
181
lo = _mm_shufflelo_epi16 (dataLo, _MM_SHUFFLE(3, 3, 3, 3));
182
hi = _mm_shufflelo_epi16 (dataHi, _MM_SHUFFLE(3, 3, 3, 3));
183
*alphaLo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE(3, 3, 3, 3));
184
*alphaHi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE(3, 3, 3, 3));
185
}
186
187
static force_inline void
188
expandAlphaRev_2x128 (__m128i dataLo, __m128i dataHi, __m128i* alphaLo, __m128i* alphaHi)
189
{
190
__m128i lo, hi;
191
192
lo = _mm_shufflelo_epi16 (dataLo, _MM_SHUFFLE(0, 0, 0, 0));
193
hi = _mm_shufflelo_epi16 (dataHi, _MM_SHUFFLE(0, 0, 0, 0));
194
*alphaLo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE(0, 0, 0, 0));
195
*alphaHi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE(0, 0, 0, 0));
196
}
197
198
static force_inline void
199
pixMultiply_2x128 (__m128i* dataLo, __m128i* dataHi, __m128i* alphaLo, __m128i* alphaHi, __m128i* retLo, __m128i* retHi)
200
{
201
__m128i lo, hi;
202
203
lo = _mm_mullo_epi16 (*dataLo, *alphaLo);
204
hi = _mm_mullo_epi16 (*dataHi, *alphaHi);
205
lo = _mm_adds_epu16 (lo, Mask0080);
206
hi = _mm_adds_epu16 (hi, Mask0080);
207
*retLo = _mm_mulhi_epu16 (lo, Mask0101);
208
*retHi = _mm_mulhi_epu16 (hi, Mask0101);
209
}
210
211
static force_inline void
212
pixAddMultiply_2x128 (__m128i* srcLo, __m128i* srcHi, __m128i* alphaDstLo, __m128i* alphaDstHi,
213
__m128i* dstLo, __m128i* dstHi, __m128i* alphaSrcLo, __m128i* alphaSrcHi,
214
__m128i* retLo, __m128i* retHi)
215
{
216
__m128i lo, hi;
217
__m128i mulLo, mulHi;
218
219
lo = _mm_mullo_epi16 (*srcLo, *alphaDstLo);
220
hi = _mm_mullo_epi16 (*srcHi, *alphaDstHi);
221
mulLo = _mm_mullo_epi16 (*dstLo, *alphaSrcLo);
222
mulHi = _mm_mullo_epi16 (*dstHi, *alphaSrcHi);
223
lo = _mm_adds_epu16 (lo, Mask0080);
224
hi = _mm_adds_epu16 (hi, Mask0080);
225
lo = _mm_adds_epu16 (lo, mulLo);
226
hi = _mm_adds_epu16 (hi, mulHi);
227
*retLo = _mm_mulhi_epu16 (lo, Mask0101);
228
*retHi = _mm_mulhi_epu16 (hi, Mask0101);
229
}
230
231
static force_inline void
232
negate_2x128 (__m128i dataLo, __m128i dataHi, __m128i* negLo, __m128i* negHi)
233
{
234
*negLo = _mm_xor_si128 (dataLo, Mask00ff);
235
*negHi = _mm_xor_si128 (dataHi, Mask00ff);
236
}
237
238
static force_inline void
239
invertColors_2x128 (__m128i dataLo, __m128i dataHi, __m128i* invLo, __m128i* invHi)
240
{
241
__m128i lo, hi;
242
243
lo = _mm_shufflelo_epi16 (dataLo, _MM_SHUFFLE(3, 0, 1, 2));
244
hi = _mm_shufflelo_epi16 (dataHi, _MM_SHUFFLE(3, 0, 1, 2));
245
*invLo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE(3, 0, 1, 2));
246
*invHi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE(3, 0, 1, 2));
247
}
248
249
static force_inline void
250
over_2x128 (__m128i* srcLo, __m128i* srcHi, __m128i* alphaLo, __m128i* alphaHi, __m128i* dstLo, __m128i* dstHi)
251
{
252
__m128i t1, t2;
253
254
negate_2x128 (*alphaLo, *alphaHi, &t1, &t2);
255
256
pixMultiply_2x128 (dstLo, dstHi, &t1, &t2, dstLo, dstHi);
257
258
*dstLo = _mm_adds_epu8 (*srcLo, *dstLo);
259
*dstHi = _mm_adds_epu8 (*srcHi, *dstHi);
260
}
261
262
static force_inline void
263
overRevNonPre_2x128 (__m128i srcLo, __m128i srcHi, __m128i* dstLo, __m128i* dstHi)
264
{
265
__m128i lo, hi;
266
__m128i alphaLo, alphaHi;
267
268
expandAlpha_2x128 (srcLo, srcHi, &alphaLo, &alphaHi);
269
270
lo = _mm_or_si128 (alphaLo, MaskAlpha);
271
hi = _mm_or_si128 (alphaHi, MaskAlpha);
272
273
invertColors_2x128 (srcLo, srcHi, &srcLo, &srcHi);
274
275
pixMultiply_2x128 (&srcLo, &srcHi, &lo, &hi, &lo, &hi);
276
277
over_2x128 (&lo, &hi, &alphaLo, &alphaHi, dstLo, dstHi);
278
}
279
280
static force_inline void
281
inOver_2x128 (__m128i* srcLo, __m128i* srcHi, __m128i* alphaLo, __m128i* alphaHi,
282
__m128i* maskLo, __m128i* maskHi, __m128i* dstLo, __m128i* dstHi)
283
{
284
__m128i sLo, sHi;
285
__m128i aLo, aHi;
286
287
pixMultiply_2x128 ( srcLo, srcHi, maskLo, maskHi, &sLo, &sHi);
288
pixMultiply_2x128 (alphaLo, alphaHi, maskLo, maskHi, &aLo, &aHi);
289
290
over_2x128 (&sLo, &sHi, &aLo, &aHi, dstLo, dstHi);
291
}
292
293
static force_inline void
294
cachePrefetch (__m128i* addr)
295
{
296
_mm_prefetch (addr, _MM_HINT_T0);
297
}
298
299
static force_inline void
300
cachePrefetchNext (__m128i* addr)
301
{
302
_mm_prefetch (addr + 4, _MM_HINT_T0); // 64 bytes ahead
303
}
304
305
/* load 4 pixels from a 16-byte boundary aligned address */
306
static force_inline __m128i
307
load128Aligned (__m128i* src)
308
{
309
return _mm_load_si128 (src);
310
}
311
312
/* load 4 pixels from a unaligned address */
313
static force_inline __m128i
314
load128Unaligned (__m128i* src)
315
{
316
return _mm_loadu_si128 (src);
317
}
318
319
/* save 4 pixels using Write Combining memory on a 16-byte boundary aligned address */
320
static force_inline void
321
save128WriteCombining (__m128i* dst, __m128i data)
322
{
323
_mm_stream_si128 (dst, data);
324
}
325
326
/* save 4 pixels on a 16-byte boundary aligned address */
327
static force_inline void
328
save128Aligned (__m128i* dst, __m128i data)
329
{
330
_mm_store_si128 (dst, data);
331
}
332
333
/* save 4 pixels on a unaligned address */
334
static force_inline void
335
save128Unaligned (__m128i* dst, __m128i data)
336
{
337
_mm_storeu_si128 (dst, data);
338
}
339
340
/* -------------------------------------------------------------------------------------------------
341
* MMX inlines
342
*/
343
344
static force_inline __m64
345
unpack_32_1x64 (uint32_t data)
346
{
347
return _mm_unpacklo_pi8 (_mm_cvtsi32_si64 (data), _mm_setzero_si64());
348
}
349
350
static force_inline __m64
351
expandAlpha_1x64 (__m64 data)
352
{
353
return _mm_shuffle_pi16 (data, _MM_SHUFFLE(3, 3, 3, 3));
354
}
355
356
static force_inline __m64
357
expandAlphaRev_1x64 (__m64 data)
358
{
359
return _mm_shuffle_pi16 (data, _MM_SHUFFLE(0, 0, 0, 0));
360
}
361
362
static force_inline __m64
363
expandPixel_8_1x64 (uint8_t data)
364
{
365
return _mm_shuffle_pi16 (unpack_32_1x64 ((uint32_t)data), _MM_SHUFFLE(0, 0, 0, 0));
366
}
367
368
static force_inline __m64
369
pixMultiply_1x64 (__m64 data, __m64 alpha)
370
{
371
return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (data, alpha),
372
xMask0080),
373
xMask0101);
374
}
375
376
static force_inline __m64
377
pixAddMultiply_1x64 (__m64* src, __m64* alphaDst, __m64* dst, __m64* alphaSrc)
378
{
379
return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (*src, *alphaDst),
380
xMask0080),
381
_mm_mullo_pi16 (*dst, *alphaSrc)),
382
xMask0101);
383
}
384
385
static force_inline __m64
386
negate_1x64 (__m64 data)
387
{
388
return _mm_xor_si64 (data, xMask00ff);
389
}
390
391
static force_inline __m64
392
invertColors_1x64 (__m64 data)
393
{
394
return _mm_shuffle_pi16 (data, _MM_SHUFFLE(3, 0, 1, 2));
395
}
396
397
static force_inline __m64
398
over_1x64 (__m64 src, __m64 alpha, __m64 dst)
399
{
400
return _mm_adds_pu8 (src, pixMultiply_1x64 (dst, negate_1x64 (alpha)));
401
}
402
403
static force_inline __m64
404
inOver_1x64 (__m64* src, __m64* alpha, __m64* mask, __m64* dst)
405
{
406
return over_1x64 (pixMultiply_1x64 (*src, *mask),
407
pixMultiply_1x64 (*alpha, *mask),
408
*dst);
409
}
410
411
static force_inline __m64
412
overRevNonPre_1x64 (__m64 src, __m64 dst)
413
{
414
__m64 alpha = expandAlpha_1x64 (src);
415
416
return over_1x64 (pixMultiply_1x64 (invertColors_1x64 (src),
417
_mm_or_si64 (alpha, xMaskAlpha)),
418
alpha,
419
dst);
420
}
421
422
static force_inline uint32_t
423
pack_1x64_32( __m64 data )
424
{
425
return _mm_cvtsi64_si32 (_mm_packs_pu16 (data, _mm_setzero_si64()));
426
}
427
428
/* Expand 16 bits positioned at @pos (0-3) of a mmx register into
429
*
430
* 00RR00GG00BB
431
*
432
* --- Expanding 565 in the low word ---
433
*
434
* m = (m << (32 - 3)) | (m << (16 - 5)) | m;
435
* m = m & (01f0003f001f);
436
* m = m * (008404100840);
437
* m = m >> 8;
438
*
439
* Note the trick here - the top word is shifted by another nibble to
440
* avoid it bumping into the middle word
441
*/
442
static force_inline __m64
443
expand565_16_1x64 (uint16_t pixel)
444
{
445
__m64 p;
446
__m64 t1, t2;
447
448
p = _mm_cvtsi32_si64 ((uint32_t) pixel);
449
450
t1 = _mm_slli_si64 (p, 36 - 11);
451
t2 = _mm_slli_si64 (p, 16 - 5);
452
453
p = _mm_or_si64 (t1, p);
454
p = _mm_or_si64 (t2, p);
455
p = _mm_and_si64 (p, xMask565rgb);
456
p = _mm_mullo_pi16 (p, xMask565Unpack);
457
458
return _mm_srli_pi16 (p, 8);
459
}
460
461
/* -------------------------------------------------------------------------------------------------
462
* Compose Core transformations
463
*/
464
static force_inline uint32_t
465
coreCombineOverUPixelsse2 (uint32_t src, uint32_t dst)
466
{
467
uint8_t a;
468
__m64 ms;
469
470
a = src >> 24;
471
472
if (a == 0xff)
473
{
474
return src;
475
}
476
else if (a)
477
{
478
ms = unpack_32_1x64 (src);
479
return pack_1x64_32 (over_1x64 (ms, expandAlpha_1x64 (ms), unpack_32_1x64 (dst)));
480
}
481
482
return dst;
483
}
484
485
static force_inline void
486
coreCombineOverUsse2 (uint32_t* pd, const uint32_t* ps, int w)
487
{
488
uint32_t pa;
489
uint32_t s, d;
490
491
__m128i xmmDstLo, xmmDstHi;
492
__m128i xmmSrcLo, xmmSrcHi;
493
__m128i xmmAlphaLo, xmmAlphaHi;
494
495
/* call prefetch hint to optimize cache load*/
496
cachePrefetch ((__m128i*)ps);
497
cachePrefetch ((__m128i*)pd);
498
499
/* Align dst on a 16-byte boundary */
500
while (w &&
501
((unsigned long)pd & 15))
502
{
503
d = *pd;
504
s = *ps++;
505
506
*pd++ = coreCombineOverUPixelsse2 (s, d);
507
w--;
508
}
509
510
/* call prefetch hint to optimize cache load*/
511
cachePrefetch ((__m128i*)ps);
512
cachePrefetch ((__m128i*)pd);
513
514
while (w >= 4)
515
{
516
/* fill cache line with next memory */
517
cachePrefetchNext ((__m128i*)ps);
518
cachePrefetchNext ((__m128i*)pd);
519
520
/* I'm loading unaligned because I'm not sure about the address alignment. */
521
xmmSrcHi = load128Unaligned ((__m128i*) ps);
522
523
/* Check the alpha channel */
524
pa = packAlpha (xmmSrcHi);
525
526
if (pa == 0xffffffff)
527
{
528
save128Aligned ((__m128i*)pd, xmmSrcHi);
529
}
530
else if (pa)
531
{
532
xmmDstHi = load128Aligned ((__m128i*) pd);
533
534
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
535
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
536
537
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
538
539
over_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDstLo, &xmmDstHi);
540
541
/* rebuid the 4 pixel data and save*/
542
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
543
}
544
545
w -= 4;
546
ps += 4;
547
pd += 4;
548
}
549
550
while (w)
551
{
552
d = *pd;
553
s = *ps++;
554
555
*pd++ = coreCombineOverUPixelsse2 (s, d);
556
w--;
557
}
558
}
559
560
static force_inline void
561
coreCombineOverReverseUsse2 (uint32_t* pd, const uint32_t* ps, int w)
562
{
563
uint32_t s, d;
564
565
__m128i xmmDstLo, xmmDstHi;
566
__m128i xmmSrcLo, xmmSrcHi;
567
__m128i xmmAlphaLo, xmmAlphaHi;
568
569
/* call prefetch hint to optimize cache load*/
570
cachePrefetch ((__m128i*)ps);
571
cachePrefetch ((__m128i*)pd);
572
573
/* Align dst on a 16-byte boundary */
574
while (w &&
575
((unsigned long)pd & 15))
576
{
577
d = *pd;
578
s = *ps++;
579
580
*pd++ = coreCombineOverUPixelsse2 (d, s);
581
w--;
582
}
583
584
/* call prefetch hint to optimize cache load*/
585
cachePrefetch ((__m128i*)ps);
586
cachePrefetch ((__m128i*)pd);
587
588
while (w >= 4)
589
{
590
/* fill cache line with next memory */
591
cachePrefetchNext ((__m128i*)ps);
592
cachePrefetchNext ((__m128i*)pd);
593
594
/* I'm loading unaligned because I'm not sure about the address alignment. */
595
xmmSrcHi = load128Unaligned ((__m128i*) ps);
596
xmmDstHi = load128Aligned ((__m128i*) pd);
597
598
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
599
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
600
601
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaLo, &xmmAlphaHi);
602
603
over_2x128 (&xmmDstLo, &xmmDstHi, &xmmAlphaLo, &xmmAlphaHi, &xmmSrcLo, &xmmSrcHi);
604
605
/* rebuid the 4 pixel data and save*/
606
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmSrcLo, xmmSrcHi));
607
608
w -= 4;
609
ps += 4;
610
pd += 4;
611
}
612
613
while (w)
614
{
615
d = *pd;
616
s = *ps++;
617
618
*pd++ = coreCombineOverUPixelsse2 (d, s);
619
w--;
620
}
621
}
622
623
static force_inline uint32_t
624
coreCombineInUPixelsse2 (uint32_t src, uint32_t dst)
625
{
626
uint32_t maska = src >> 24;
627
628
if (maska == 0)
629
{
630
return 0;
631
}
632
else if (maska != 0xff)
633
{
634
return pack_1x64_32(pixMultiply_1x64 (unpack_32_1x64 (dst), expandAlpha_1x64 (unpack_32_1x64 (src))));
635
}
636
637
return dst;
638
}
639
640
static force_inline void
641
coreCombineInUsse2 (uint32_t* pd, const uint32_t* ps, int w)
642
{
643
uint32_t s, d;
644
645
__m128i xmmSrcLo, xmmSrcHi;
646
__m128i xmmDstLo, xmmDstHi;
647
648
/* call prefetch hint to optimize cache load*/
649
cachePrefetch ((__m128i*)ps);
650
cachePrefetch ((__m128i*)pd);
651
652
while (w && ((unsigned long) pd & 15))
653
{
654
s = *ps++;
655
d = *pd;
656
657
*pd++ = coreCombineInUPixelsse2 (d, s);
658
w--;
659
}
660
661
/* call prefetch hint to optimize cache load*/
662
cachePrefetch ((__m128i*)ps);
663
cachePrefetch ((__m128i*)pd);
664
665
while (w >= 4)
666
{
667
/* fill cache line with next memory */
668
cachePrefetchNext ((__m128i*)ps);
669
cachePrefetchNext ((__m128i*)pd);
670
671
xmmDstHi = load128Aligned ((__m128i*) pd);
672
xmmSrcHi = load128Unaligned ((__m128i*) ps);
673
674
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
675
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmDstLo, &xmmDstHi);
676
677
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
678
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmDstLo, &xmmDstHi, &xmmDstLo, &xmmDstHi);
679
680
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
681
682
ps += 4;
683
pd += 4;
684
w -= 4;
685
}
686
687
while (w)
688
{
689
s = *ps++;
690
d = *pd;
691
692
*pd++ = coreCombineInUPixelsse2 (d, s);
693
w--;
694
}
695
}
696
697
static force_inline void
698
coreCombineReverseInUsse2 (uint32_t* pd, const uint32_t* ps, int w)
699
{
700
uint32_t s, d;
701
702
__m128i xmmSrcLo, xmmSrcHi;
703
__m128i xmmDstLo, xmmDstHi;
704
705
/* call prefetch hint to optimize cache load*/
706
cachePrefetch ((__m128i*)ps);
707
cachePrefetch ((__m128i*)pd);
708
709
while (w && ((unsigned long) pd & 15))
710
{
711
s = *ps++;
712
d = *pd;
713
714
*pd++ = coreCombineInUPixelsse2 (s, d);
715
w--;
716
}
717
718
/* call prefetch hint to optimize cache load*/
719
cachePrefetch ((__m128i*)ps);
720
cachePrefetch ((__m128i*)pd);
721
722
while (w >= 4)
723
{
724
/* fill cache line with next memory */
725
cachePrefetchNext ((__m128i*)ps);
726
cachePrefetchNext ((__m128i*)pd);
727
728
xmmDstHi = load128Aligned ((__m128i*) pd);
729
xmmSrcHi = load128Unaligned ((__m128i*) ps);
730
731
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
732
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
733
734
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
735
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmSrcLo, &xmmSrcHi, &xmmDstLo, &xmmDstHi);
736
737
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
738
739
ps += 4;
740
pd += 4;
741
w -= 4;
742
}
743
744
while (w)
745
{
746
s = *ps++;
747
d = *pd;
748
749
*pd++ = coreCombineInUPixelsse2 (s, d);
750
w--;
751
}
752
}
753
754
static force_inline void
755
coreCombineReverseOutUsse2 (uint32_t* pd, const uint32_t* ps, int w)
756
{
757
/* call prefetch hint to optimize cache load*/
758
cachePrefetch ((__m128i*)ps);
759
cachePrefetch ((__m128i*)pd);
760
761
while (w && ((unsigned long) pd & 15))
762
{
763
uint32_t s = *ps++;
764
uint32_t d = *pd;
765
766
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d), negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (s)))));
767
w--;
768
}
769
770
/* call prefetch hint to optimize cache load*/
771
cachePrefetch ((__m128i*)ps);
772
cachePrefetch ((__m128i*)pd);
773
774
while (w >= 4)
775
{
776
__m128i xmmSrcLo, xmmSrcHi;
777
__m128i xmmDstLo, xmmDstHi;
778
779
/* fill cache line with next memory */
780
cachePrefetchNext ((__m128i*)ps);
781
cachePrefetchNext ((__m128i*)pd);
782
783
xmmSrcHi = load128Unaligned ((__m128i*) ps);
784
xmmDstHi = load128Aligned ((__m128i*) pd);
785
786
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
787
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
788
789
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
790
negate_2x128 (xmmSrcLo, xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
791
792
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmSrcLo, &xmmSrcHi, &xmmDstLo, &xmmDstHi);
793
794
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
795
796
ps += 4;
797
pd += 4;
798
w -= 4;
799
}
800
801
while (w)
802
{
803
uint32_t s = *ps++;
804
uint32_t d = *pd;
805
806
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d), negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (s)))));
807
w--;
808
}
809
}
810
811
static force_inline void
812
coreCombineOutUsse2 (uint32_t* pd, const uint32_t* ps, int w)
813
{
814
/* call prefetch hint to optimize cache load*/
815
cachePrefetch ((__m128i*)ps);
816
cachePrefetch ((__m128i*)pd);
817
818
while (w && ((unsigned long) pd & 15))
819
{
820
uint32_t s = *ps++;
821
uint32_t d = *pd;
822
823
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s), negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (d)))));
824
w--;
825
}
826
827
/* call prefetch hint to optimize cache load*/
828
cachePrefetch ((__m128i*)ps);
829
cachePrefetch ((__m128i*)pd);
830
831
while (w >= 4)
832
{
833
__m128i xmmSrcLo, xmmSrcHi;
834
__m128i xmmDstLo, xmmDstHi;
835
836
/* fill cache line with next memory */
837
cachePrefetchNext ((__m128i*)ps);
838
cachePrefetchNext ((__m128i*)pd);
839
840
xmmSrcHi = load128Unaligned ((__m128i*) ps);
841
xmmDstHi = load128Aligned ((__m128i*) pd);
842
843
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
844
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
845
846
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmDstLo, &xmmDstHi);
847
negate_2x128 (xmmDstLo, xmmDstHi, &xmmDstLo, &xmmDstHi);
848
849
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmDstLo, &xmmDstHi, &xmmDstLo, &xmmDstHi);
850
851
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
852
853
ps += 4;
854
pd += 4;
855
w -= 4;
856
}
857
858
while (w)
859
{
860
uint32_t s = *ps++;
861
uint32_t d = *pd;
862
863
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s), negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (d)))));
864
w--;
865
}
866
}
867
868
static force_inline uint32_t
869
coreCombineAtopUPixelsse2 (uint32_t src, uint32_t dst)
870
{
871
__m64 s = unpack_32_1x64 (src);
872
__m64 d = unpack_32_1x64 (dst);
873
874
__m64 sa = negate_1x64 (expandAlpha_1x64 (s));
875
__m64 da = expandAlpha_1x64 (d);
876
877
return pack_1x64_32 (pixAddMultiply_1x64 (&s, &da, &d, &sa));
878
}
879
880
static force_inline void
881
coreCombineAtopUsse2 (uint32_t* pd, const uint32_t* ps, int w)
882
{
883
uint32_t s, d;
884
885
__m128i xmmSrcLo, xmmSrcHi;
886
__m128i xmmDstLo, xmmDstHi;
887
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
888
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
889
890
/* call prefetch hint to optimize cache load*/
891
cachePrefetch ((__m128i*)ps);
892
cachePrefetch ((__m128i*)pd);
893
894
while (w && ((unsigned long) pd & 15))
895
{
896
s = *ps++;
897
d = *pd;
898
899
*pd++ = coreCombineAtopUPixelsse2 (s, d);
900
w--;
901
}
902
903
/* call prefetch hint to optimize cache load*/
904
cachePrefetch ((__m128i*)ps);
905
cachePrefetch ((__m128i*)pd);
906
907
while (w >= 4)
908
{
909
/* fill cache line with next memory */
910
cachePrefetchNext ((__m128i*)ps);
911
cachePrefetchNext ((__m128i*)pd);
912
913
xmmSrcHi = load128Unaligned ((__m128i*) ps);
914
xmmDstHi = load128Aligned ((__m128i*) pd);
915
916
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
917
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
918
919
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
920
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
921
922
negate_2x128 (xmmAlphaSrcLo, xmmAlphaSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
923
924
pixAddMultiply_2x128 ( &xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
925
&xmmDstLo, &xmmDstHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi,
926
&xmmDstLo, &xmmDstHi );
927
928
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
929
930
ps += 4;
931
pd += 4;
932
w -= 4;
933
}
934
935
while (w)
936
{
937
s = *ps++;
938
d = *pd;
939
940
*pd++ = coreCombineAtopUPixelsse2 (s, d);
941
w--;
942
}
943
}
944
945
static force_inline uint32_t
946
coreCombineReverseAtopUPixelsse2 (uint32_t src, uint32_t dst)
947
{
948
__m64 s = unpack_32_1x64 (src);
949
__m64 d = unpack_32_1x64 (dst);
950
951
__m64 sa = expandAlpha_1x64 (s);
952
__m64 da = negate_1x64 (expandAlpha_1x64 (d));
953
954
return pack_1x64_32 (pixAddMultiply_1x64 (&s, &da, &d, &sa));
955
}
956
957
static force_inline void
958
coreCombineReverseAtopUsse2 (uint32_t* pd, const uint32_t* ps, int w)
959
{
960
uint32_t s, d;
961
962
__m128i xmmSrcLo, xmmSrcHi;
963
__m128i xmmDstLo, xmmDstHi;
964
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
965
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
966
967
/* call prefetch hint to optimize cache load*/
968
cachePrefetch ((__m128i*)ps);
969
cachePrefetch ((__m128i*)pd);
970
971
while (w && ((unsigned long) pd & 15))
972
{
973
s = *ps++;
974
d = *pd;
975
976
*pd++ = coreCombineReverseAtopUPixelsse2 (s, d);
977
w--;
978
}
979
980
/* call prefetch hint to optimize cache load*/
981
cachePrefetch ((__m128i*)ps);
982
cachePrefetch ((__m128i*)pd);
983
984
while (w >= 4)
985
{
986
/* fill cache line with next memory */
987
cachePrefetchNext ((__m128i*)ps);
988
cachePrefetchNext ((__m128i*)pd);
989
990
xmmSrcHi = load128Unaligned ((__m128i*) ps);
991
xmmDstHi = load128Aligned ((__m128i*) pd);
992
993
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
994
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
995
996
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
997
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
998
999
negate_2x128 (xmmAlphaDstLo, xmmAlphaDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1000
1001
pixAddMultiply_2x128 ( &xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
1002
&xmmDstLo, &xmmDstHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi,
1003
&xmmDstLo, &xmmDstHi );
1004
1005
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1006
1007
ps += 4;
1008
pd += 4;
1009
w -= 4;
1010
}
1011
1012
while (w)
1013
{
1014
s = *ps++;
1015
d = *pd;
1016
1017
*pd++ = coreCombineReverseAtopUPixelsse2 (s, d);
1018
w--;
1019
}
1020
}
1021
1022
static force_inline uint32_t
1023
coreCombineXorUPixelsse2 (uint32_t src, uint32_t dst)
1024
{
1025
__m64 s = unpack_32_1x64 (src);
1026
__m64 d = unpack_32_1x64 (dst);
1027
1028
__m64 negD = negate_1x64 (expandAlpha_1x64 (d));
1029
__m64 negS = negate_1x64 (expandAlpha_1x64 (s));
1030
1031
return pack_1x64_32 (pixAddMultiply_1x64 (&s, &negD, &d, &negS));
1032
}
1033
1034
static force_inline void
1035
coreCombineXorUsse2 (uint32_t* dst, const uint32_t* src, int width)
1036
{
1037
int w = width;
1038
uint32_t s, d;
1039
uint32_t* pd = dst;
1040
const uint32_t* ps = src;
1041
1042
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
1043
__m128i xmmDst, xmmDstLo, xmmDstHi;
1044
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
1045
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
1046
1047
/* call prefetch hint to optimize cache load*/
1048
cachePrefetch ((__m128i*)ps);
1049
cachePrefetch ((__m128i*)pd);
1050
1051
while (w && ((unsigned long) pd & 15))
1052
{
1053
s = *ps++;
1054
d = *pd;
1055
1056
*pd++ = coreCombineXorUPixelsse2 (s, d);
1057
w--;
1058
}
1059
1060
/* call prefetch hint to optimize cache load*/
1061
cachePrefetch ((__m128i*)ps);
1062
cachePrefetch ((__m128i*)pd);
1063
1064
while (w >= 4)
1065
{
1066
/* fill cache line with next memory */
1067
cachePrefetchNext ((__m128i*)ps);
1068
cachePrefetchNext ((__m128i*)pd);
1069
1070
xmmSrc = load128Unaligned ((__m128i*) ps);
1071
xmmDst = load128Aligned ((__m128i*) pd);
1072
1073
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
1074
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
1075
1076
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
1077
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1078
1079
negate_2x128 (xmmAlphaSrcLo, xmmAlphaSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
1080
negate_2x128 (xmmAlphaDstLo, xmmAlphaDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1081
1082
pixAddMultiply_2x128 ( &xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
1083
&xmmDstLo, &xmmDstHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi,
1084
&xmmDstLo, &xmmDstHi );
1085
1086
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1087
1088
ps += 4;
1089
pd += 4;
1090
w -= 4;
1091
}
1092
1093
while (w)
1094
{
1095
s = *ps++;
1096
d = *pd;
1097
1098
*pd++ = coreCombineXorUPixelsse2 (s, d);
1099
w--;
1100
}
1101
}
1102
1103
static force_inline void
1104
coreCombineAddUsse2 (uint32_t* dst, const uint32_t* src, int width)
1105
{
1106
int w = width;
1107
uint32_t s,d;
1108
uint32_t* pd = dst;
1109
const uint32_t* ps = src;
1110
1111
/* call prefetch hint to optimize cache load*/
1112
cachePrefetch ((__m128i*)ps);
1113
cachePrefetch ((__m128i*)pd);
1114
1115
while (w && (unsigned long)pd & 15)
1116
{
1117
s = *ps++;
1118
d = *pd;
1119
*pd++ = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1120
w--;
1121
}
1122
1123
/* call prefetch hint to optimize cache load*/
1124
cachePrefetch ((__m128i*)ps);
1125
cachePrefetch ((__m128i*)pd);
1126
1127
while (w >= 4)
1128
{
1129
/* fill cache line with next memory */
1130
cachePrefetchNext ((__m128i*)ps);
1131
cachePrefetchNext ((__m128i*)pd);
1132
1133
save128Aligned( (__m128i*)pd,
1134
_mm_adds_epu8( load128Unaligned((__m128i*)ps),
1135
load128Aligned ((__m128i*)pd)) );
1136
pd += 4;
1137
ps += 4;
1138
w -= 4;
1139
}
1140
1141
while (w--)
1142
{
1143
s = *ps++;
1144
d = *pd;
1145
*pd++ = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1146
}
1147
}
1148
1149
static force_inline uint32_t
1150
coreCombineSaturateUPixelsse2 (uint32_t src, uint32_t dst)
1151
{
1152
__m64 ms = unpack_32_1x64 (src);
1153
__m64 md = unpack_32_1x64 (dst);
1154
uint32_t sa = src >> 24;
1155
uint32_t da = ~dst >> 24;
1156
1157
if (sa > da)
1158
{
1159
ms = pixMultiply_1x64 (ms, expandAlpha_1x64 (unpack_32_1x64 (FbIntDiv(da, sa) << 24)));
1160
}
1161
1162
return pack_1x64_32 (_mm_adds_pu16 (md, ms));
1163
}
1164
1165
static force_inline void
1166
coreCombineSaturateUsse2 (uint32_t *pd, const uint32_t *ps, int w)
1167
{
1168
uint32_t s,d;
1169
1170
uint32_t packCmp;
1171
__m128i xmmSrc, xmmDst;
1172
1173
/* call prefetch hint to optimize cache load*/
1174
cachePrefetch ((__m128i*)ps);
1175
cachePrefetch ((__m128i*)pd);
1176
1177
while (w && (unsigned long)pd & 15)
1178
{
1179
s = *ps++;
1180
d = *pd;
1181
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1182
w--;
1183
}
1184
1185
/* call prefetch hint to optimize cache load*/
1186
cachePrefetch ((__m128i*)ps);
1187
cachePrefetch ((__m128i*)pd);
1188
1189
while (w >= 4)
1190
{
1191
/* fill cache line with next memory */
1192
cachePrefetchNext ((__m128i*)ps);
1193
cachePrefetchNext ((__m128i*)pd);
1194
1195
xmmDst = load128Aligned ((__m128i*)pd);
1196
xmmSrc = load128Unaligned((__m128i*)ps);
1197
1198
packCmp = _mm_movemask_epi8 (_mm_cmpgt_epi32 (_mm_srli_epi32 (xmmSrc, 24),
1199
_mm_srli_epi32 (_mm_xor_si128 (xmmDst, Maskff000000), 24)));
1200
1201
/* if some alpha src is grater than respective ~alpha dst */
1202
if (packCmp)
1203
{
1204
s = *ps++;
1205
d = *pd;
1206
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1207
1208
s = *ps++;
1209
d = *pd;
1210
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1211
1212
s = *ps++;
1213
d = *pd;
1214
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1215
1216
s = *ps++;
1217
d = *pd;
1218
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1219
}
1220
else
1221
{
1222
save128Aligned ((__m128i*)pd, _mm_adds_epu8 (xmmDst, xmmSrc));
1223
1224
pd += 4;
1225
ps += 4;
1226
}
1227
1228
w -= 4;
1229
}
1230
1231
while (w--)
1232
{
1233
s = *ps++;
1234
d = *pd;
1235
*pd++ = coreCombineSaturateUPixelsse2 (s, d);
1236
}
1237
}
1238
1239
static force_inline void
1240
coreCombineSrcCsse2 (uint32_t* pd, const uint32_t* ps, const uint32_t *pm, int w)
1241
{
1242
uint32_t s, m;
1243
1244
__m128i xmmSrcLo, xmmSrcHi;
1245
__m128i xmmMaskLo, xmmMaskHi;
1246
__m128i xmmDstLo, xmmDstHi;
1247
1248
/* call prefetch hint to optimize cache load*/
1249
cachePrefetch ((__m128i*)ps);
1250
cachePrefetch ((__m128i*)pd);
1251
cachePrefetch ((__m128i*)pm);
1252
1253
while (w && (unsigned long)pd & 15)
1254
{
1255
s = *ps++;
1256
m = *pm++;
1257
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1258
w--;
1259
}
1260
1261
/* call prefetch hint to optimize cache load*/
1262
cachePrefetch ((__m128i*)ps);
1263
cachePrefetch ((__m128i*)pd);
1264
cachePrefetch ((__m128i*)pm);
1265
1266
while (w >= 4)
1267
{
1268
/* fill cache line with next memory */
1269
cachePrefetchNext ((__m128i*)ps);
1270
cachePrefetchNext ((__m128i*)pd);
1271
cachePrefetchNext ((__m128i*)pm);
1272
1273
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1274
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1275
1276
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1277
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1278
1279
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
1280
1281
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1282
1283
ps += 4;
1284
pd += 4;
1285
pm += 4;
1286
w -= 4;
1287
}
1288
1289
while (w)
1290
{
1291
s = *ps++;
1292
m = *pm++;
1293
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1294
w--;
1295
}
1296
}
1297
1298
static force_inline uint32_t
1299
coreCombineOverCPixelsse2 (uint32_t src, uint32_t mask, uint32_t dst)
1300
{
1301
__m64 s = unpack_32_1x64 (src);
1302
__m64 expAlpha = expandAlpha_1x64 (s);
1303
__m64 unpkMask = unpack_32_1x64 (mask);
1304
__m64 unpkDst = unpack_32_1x64 (dst);
1305
1306
return pack_1x64_32 (inOver_1x64 (&s, &expAlpha, &unpkMask, &unpkDst));
1307
}
1308
1309
static force_inline void
1310
coreCombineOverCsse2 (uint32_t* pd, const uint32_t* ps, const uint32_t *pm, int w)
1311
{
1312
uint32_t s, m, d;
1313
1314
__m128i xmmAlphaLo, xmmAlphaHi;
1315
__m128i xmmSrcLo, xmmSrcHi;
1316
__m128i xmmDstLo, xmmDstHi;
1317
__m128i xmmMaskLo, xmmMaskHi;
1318
1319
/* call prefetch hint to optimize cache load*/
1320
cachePrefetch ((__m128i*)ps);
1321
cachePrefetch ((__m128i*)pd);
1322
cachePrefetch ((__m128i*)pm);
1323
1324
while (w && (unsigned long)pd & 15)
1325
{
1326
s = *ps++;
1327
m = *pm++;
1328
d = *pd;
1329
1330
*pd++ = coreCombineOverCPixelsse2 (s, m, d);
1331
w--;
1332
}
1333
1334
/* call prefetch hint to optimize cache load*/
1335
cachePrefetch ((__m128i*)ps);
1336
cachePrefetch ((__m128i*)pd);
1337
cachePrefetch ((__m128i*)pm);
1338
1339
while (w >= 4)
1340
{
1341
/* fill cache line with next memory */
1342
cachePrefetchNext ((__m128i*)ps);
1343
cachePrefetchNext ((__m128i*)pd);
1344
cachePrefetchNext ((__m128i*)pm);
1345
1346
xmmDstHi = load128Aligned ((__m128i*)pd);
1347
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1348
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1349
1350
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1351
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1352
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1353
1354
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
1355
1356
inOver_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
1357
1358
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1359
1360
ps += 4;
1361
pd += 4;
1362
pm += 4;
1363
w -= 4;
1364
}
1365
1366
while (w)
1367
{
1368
s = *ps++;
1369
m = *pm++;
1370
d = *pd;
1371
1372
*pd++ = coreCombineOverCPixelsse2 (s, m, d);
1373
w--;
1374
}
1375
}
1376
1377
static force_inline uint32_t
1378
coreCombineOverReverseCPixelsse2 (uint32_t src, uint32_t mask, uint32_t dst)
1379
{
1380
__m64 d = unpack_32_1x64 (dst);
1381
1382
return pack_1x64_32(over_1x64 (d, expandAlpha_1x64 (d), pixMultiply_1x64 (unpack_32_1x64 (src), unpack_32_1x64 (mask))));
1383
}
1384
1385
static force_inline void
1386
coreCombineOverReverseCsse2 (uint32_t* pd, const uint32_t* ps, const uint32_t *pm, int w)
1387
{
1388
uint32_t s, m, d;
1389
1390
__m128i xmmAlphaLo, xmmAlphaHi;
1391
__m128i xmmSrcLo, xmmSrcHi;
1392
__m128i xmmDstLo, xmmDstHi;
1393
__m128i xmmMaskLo, xmmMaskHi;
1394
1395
/* call prefetch hint to optimize cache load*/
1396
cachePrefetch ((__m128i*)ps);
1397
cachePrefetch ((__m128i*)pd);
1398
cachePrefetch ((__m128i*)pm);
1399
1400
while (w && (unsigned long)pd & 15)
1401
{
1402
s = *ps++;
1403
m = *pm++;
1404
d = *pd;
1405
1406
*pd++ = coreCombineOverReverseCPixelsse2 (s, m, d);
1407
w--;
1408
}
1409
1410
/* call prefetch hint to optimize cache load*/
1411
cachePrefetch ((__m128i*)ps);
1412
cachePrefetch ((__m128i*)pd);
1413
cachePrefetch ((__m128i*)pm);
1414
1415
while (w >= 4)
1416
{
1417
/* fill cache line with next memory */
1418
cachePrefetchNext ((__m128i*)ps);
1419
cachePrefetchNext ((__m128i*)pd);
1420
cachePrefetchNext ((__m128i*)pm);
1421
1422
xmmDstHi = load128Aligned ((__m128i*)pd);
1423
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1424
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1425
1426
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1427
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1428
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1429
1430
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaLo, &xmmAlphaHi);
1431
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1432
1433
over_2x128 (&xmmDstLo, &xmmDstHi, &xmmAlphaLo, &xmmAlphaHi, &xmmMaskLo, &xmmMaskHi);
1434
1435
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmMaskLo, xmmMaskHi));
1436
1437
ps += 4;
1438
pd += 4;
1439
pm += 4;
1440
w -= 4;
1441
}
1442
1443
while (w)
1444
{
1445
s = *ps++;
1446
m = *pm++;
1447
d = *pd;
1448
1449
*pd++ = coreCombineOverReverseCPixelsse2 (s, m, d);
1450
w--;
1451
}
1452
}
1453
1454
static force_inline void
1455
coreCombineInCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1456
{
1457
uint32_t s, m, d;
1458
1459
__m128i xmmAlphaLo, xmmAlphaHi;
1460
__m128i xmmSrcLo, xmmSrcHi;
1461
__m128i xmmDstLo, xmmDstHi;
1462
__m128i xmmMaskLo, xmmMaskHi;
1463
1464
/* call prefetch hint to optimize cache load*/
1465
cachePrefetch ((__m128i*)ps);
1466
cachePrefetch ((__m128i*)pd);
1467
cachePrefetch ((__m128i*)pm);
1468
1469
while (w && (unsigned long)pd & 15)
1470
{
1471
s = *ps++;
1472
m = *pm++;
1473
d = *pd;
1474
1475
*pd++ = pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1476
expandAlpha_1x64 (unpack_32_1x64 (d))));
1477
w--;
1478
}
1479
1480
/* call prefetch hint to optimize cache load*/
1481
cachePrefetch ((__m128i*)ps);
1482
cachePrefetch ((__m128i*)pd);
1483
cachePrefetch ((__m128i*)pm);
1484
1485
while (w >= 4)
1486
{
1487
/* fill cache line with next memory */
1488
cachePrefetchNext ((__m128i*)ps);
1489
cachePrefetchNext ((__m128i*)pd);
1490
cachePrefetchNext ((__m128i*)pm);
1491
1492
xmmDstHi = load128Aligned ((__m128i*)pd);
1493
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1494
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1495
1496
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1497
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1498
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1499
1500
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaLo, &xmmAlphaHi);
1501
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
1502
1503
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDstLo, &xmmDstHi);
1504
1505
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1506
1507
ps += 4;
1508
pd += 4;
1509
pm += 4;
1510
w -= 4;
1511
}
1512
1513
while (w)
1514
{
1515
s = *ps++;
1516
m = *pm++;
1517
d = *pd;
1518
1519
*pd++ = pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1520
expandAlpha_1x64 (unpack_32_1x64 (d))));
1521
w--;
1522
}
1523
}
1524
1525
static force_inline void
1526
coreCombineInReverseCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1527
{
1528
uint32_t s, m, d;
1529
1530
__m128i xmmAlphaLo, xmmAlphaHi;
1531
__m128i xmmSrcLo, xmmSrcHi;
1532
__m128i xmmDstLo, xmmDstHi;
1533
__m128i xmmMaskLo, xmmMaskHi;
1534
1535
/* call prefetch hint to optimize cache load*/
1536
cachePrefetch ((__m128i*)ps);
1537
cachePrefetch ((__m128i*)pd);
1538
cachePrefetch ((__m128i*)pm);
1539
1540
while (w && (unsigned long)pd & 15)
1541
{
1542
s = *ps++;
1543
m = *pm++;
1544
d = *pd;
1545
1546
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d),
1547
pixMultiply_1x64 (unpack_32_1x64 (m),
1548
expandAlpha_1x64 (unpack_32_1x64 (s)))));
1549
w--;
1550
}
1551
1552
/* call prefetch hint to optimize cache load*/
1553
cachePrefetch ((__m128i*)ps);
1554
cachePrefetch ((__m128i*)pd);
1555
cachePrefetch ((__m128i*)pm);
1556
1557
while (w >= 4)
1558
{
1559
/* fill cache line with next memory */
1560
cachePrefetchNext ((__m128i*)ps);
1561
cachePrefetchNext ((__m128i*)pd);
1562
cachePrefetchNext ((__m128i*)pm);
1563
1564
xmmDstHi = load128Aligned ((__m128i*)pd);
1565
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1566
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1567
1568
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1569
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1570
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1571
1572
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
1573
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmAlphaLo, &xmmAlphaHi, &xmmAlphaLo, &xmmAlphaHi);
1574
1575
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDstLo, &xmmDstHi);
1576
1577
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1578
1579
ps += 4;
1580
pd += 4;
1581
pm += 4;
1582
w -= 4;
1583
}
1584
1585
while (w)
1586
{
1587
s = *ps++;
1588
m = *pm++;
1589
d = *pd;
1590
1591
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d),
1592
pixMultiply_1x64 (unpack_32_1x64 (m),
1593
expandAlpha_1x64 (unpack_32_1x64 (s)))));
1594
w--;
1595
}
1596
}
1597
1598
static force_inline void
1599
coreCombineOutCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1600
{
1601
uint32_t s, m, d;
1602
1603
__m128i xmmAlphaLo, xmmAlphaHi;
1604
__m128i xmmSrcLo, xmmSrcHi;
1605
__m128i xmmDstLo, xmmDstHi;
1606
__m128i xmmMaskLo, xmmMaskHi;
1607
1608
/* call prefetch hint to optimize cache load*/
1609
cachePrefetch ((__m128i*)ps);
1610
cachePrefetch ((__m128i*)pd);
1611
cachePrefetch ((__m128i*)pm);
1612
1613
while (w && (unsigned long)pd & 15)
1614
{
1615
s = *ps++;
1616
m = *pm++;
1617
d = *pd;
1618
1619
*pd++ = pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1620
negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (d)))));
1621
w--;
1622
}
1623
1624
/* call prefetch hint to optimize cache load*/
1625
cachePrefetch ((__m128i*)ps);
1626
cachePrefetch ((__m128i*)pd);
1627
cachePrefetch ((__m128i*)pm);
1628
1629
while (w >= 4)
1630
{
1631
/* fill cache line with next memory */
1632
cachePrefetchNext ((__m128i*)ps);
1633
cachePrefetchNext ((__m128i*)pd);
1634
cachePrefetchNext ((__m128i*)pm);
1635
1636
xmmDstHi = load128Aligned ((__m128i*)pd);
1637
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1638
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1639
1640
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1641
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1642
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1643
1644
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaLo, &xmmAlphaHi);
1645
negate_2x128 (xmmAlphaLo, xmmAlphaHi, &xmmAlphaLo, &xmmAlphaHi);
1646
1647
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
1648
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDstLo, &xmmDstHi);
1649
1650
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1651
1652
ps += 4;
1653
pd += 4;
1654
pm += 4;
1655
w -= 4;
1656
}
1657
1658
while (w)
1659
{
1660
s = *ps++;
1661
m = *pm++;
1662
d = *pd;
1663
1664
*pd++ = pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1665
negate_1x64 (expandAlpha_1x64 (unpack_32_1x64 (d)))));
1666
w--;
1667
}
1668
}
1669
1670
static force_inline void
1671
coreCombineOutReverseCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1672
{
1673
uint32_t s, m, d;
1674
1675
__m128i xmmAlphaLo, xmmAlphaHi;
1676
__m128i xmmSrcLo, xmmSrcHi;
1677
__m128i xmmDstLo, xmmDstHi;
1678
__m128i xmmMaskLo, xmmMaskHi;
1679
1680
/* call prefetch hint to optimize cache load*/
1681
cachePrefetch ((__m128i*)ps);
1682
cachePrefetch ((__m128i*)pd);
1683
cachePrefetch ((__m128i*)pm);
1684
1685
while (w && (unsigned long)pd & 15)
1686
{
1687
s = *ps++;
1688
m = *pm++;
1689
d = *pd;
1690
1691
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d),
1692
negate_1x64 (pixMultiply_1x64 (unpack_32_1x64 (m),
1693
expandAlpha_1x64 (unpack_32_1x64 (s))))));
1694
w--;
1695
}
1696
1697
/* call prefetch hint to optimize cache load*/
1698
cachePrefetch ((__m128i*)ps);
1699
cachePrefetch ((__m128i*)pd);
1700
cachePrefetch ((__m128i*)pm);
1701
1702
while (w >= 4)
1703
{
1704
/* fill cache line with next memory */
1705
cachePrefetchNext ((__m128i*)ps);
1706
cachePrefetchNext ((__m128i*)pd);
1707
cachePrefetchNext ((__m128i*)pm);
1708
1709
xmmDstHi = load128Aligned ((__m128i*)pd);
1710
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1711
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1712
1713
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1714
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1715
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1716
1717
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
1718
1719
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmAlphaLo, &xmmAlphaHi, &xmmMaskLo, &xmmMaskHi);
1720
1721
negate_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1722
1723
pixMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
1724
1725
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1726
1727
ps += 4;
1728
pd += 4;
1729
pm += 4;
1730
w -= 4;
1731
}
1732
1733
while (w)
1734
{
1735
s = *ps++;
1736
m = *pm++;
1737
d = *pd;
1738
1739
*pd++ = pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (d),
1740
negate_1x64 (pixMultiply_1x64 (unpack_32_1x64 (m),
1741
expandAlpha_1x64 (unpack_32_1x64 (s))))));
1742
w--;
1743
}
1744
}
1745
1746
static force_inline uint32_t
1747
coreCombineAtopCPixelsse2 (uint32_t src, uint32_t mask, uint32_t dst)
1748
{
1749
__m64 m = unpack_32_1x64 (mask);
1750
__m64 s = unpack_32_1x64 (src);
1751
__m64 d = unpack_32_1x64 (dst);
1752
__m64 sa = expandAlpha_1x64 (s);
1753
__m64 da = expandAlpha_1x64 (d);
1754
1755
s = pixMultiply_1x64 (s, m);
1756
m = negate_1x64 (pixMultiply_1x64 (m, sa));
1757
1758
return pack_1x64_32 (pixAddMultiply_1x64 (&d, &m, &s, &da));
1759
}
1760
1761
static force_inline void
1762
coreCombineAtopCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1763
{
1764
uint32_t s, m, d;
1765
1766
__m128i xmmSrcLo, xmmSrcHi;
1767
__m128i xmmDstLo, xmmDstHi;
1768
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
1769
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
1770
__m128i xmmMaskLo, xmmMaskHi;
1771
1772
/* call prefetch hint to optimize cache load*/
1773
cachePrefetch ((__m128i*)ps);
1774
cachePrefetch ((__m128i*)pd);
1775
cachePrefetch ((__m128i*)pm);
1776
1777
while (w && (unsigned long)pd & 15)
1778
{
1779
s = *ps++;
1780
m = *pm++;
1781
d = *pd;
1782
1783
*pd++ = coreCombineAtopCPixelsse2 (s, m, d);
1784
w--;
1785
}
1786
1787
/* call prefetch hint to optimize cache load*/
1788
cachePrefetch ((__m128i*)ps);
1789
cachePrefetch ((__m128i*)pd);
1790
cachePrefetch ((__m128i*)pm);
1791
1792
while (w >= 4)
1793
{
1794
/* fill cache line with next memory */
1795
cachePrefetchNext ((__m128i*)ps);
1796
cachePrefetchNext ((__m128i*)pd);
1797
cachePrefetchNext ((__m128i*)pm);
1798
1799
xmmDstHi = load128Aligned ((__m128i*)pd);
1800
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1801
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1802
1803
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1804
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1805
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1806
1807
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
1808
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1809
1810
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmSrcLo, &xmmSrcHi);
1811
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi, &xmmMaskLo, &xmmMaskHi);
1812
1813
negate_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1814
1815
pixAddMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmMaskLo, &xmmMaskHi,
1816
&xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
1817
&xmmDstLo, &xmmDstHi);
1818
1819
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1820
1821
ps += 4;
1822
pd += 4;
1823
pm += 4;
1824
w -= 4;
1825
}
1826
1827
while (w)
1828
{
1829
s = *ps++;
1830
m = *pm++;
1831
d = *pd;
1832
1833
*pd++ = coreCombineAtopCPixelsse2 (s, m, d);
1834
w--;
1835
}
1836
}
1837
1838
static force_inline uint32_t
1839
coreCombineReverseAtopCPixelsse2 (uint32_t src, uint32_t mask, uint32_t dst)
1840
{
1841
__m64 m = unpack_32_1x64 (mask);
1842
__m64 s = unpack_32_1x64 (src);
1843
__m64 d = unpack_32_1x64 (dst);
1844
1845
__m64 da = negate_1x64 (expandAlpha_1x64 (d));
1846
__m64 sa = expandAlpha_1x64 (s);
1847
1848
s = pixMultiply_1x64 (s, m);
1849
m = pixMultiply_1x64 (m, sa);
1850
1851
return pack_1x64_32 (pixAddMultiply_1x64 (&d, &m, &s, &da));
1852
}
1853
1854
static force_inline void
1855
coreCombineReverseAtopCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1856
{
1857
uint32_t s, m, d;
1858
1859
__m128i xmmSrcLo, xmmSrcHi;
1860
__m128i xmmDstLo, xmmDstHi;
1861
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
1862
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
1863
__m128i xmmMaskLo, xmmMaskHi;
1864
1865
/* call prefetch hint to optimize cache load*/
1866
cachePrefetch ((__m128i*)ps);
1867
cachePrefetch ((__m128i*)pd);
1868
cachePrefetch ((__m128i*)pm);
1869
1870
while (w && (unsigned long)pd & 15)
1871
{
1872
s = *ps++;
1873
m = *pm++;
1874
d = *pd;
1875
1876
*pd++ = coreCombineReverseAtopCPixelsse2 (s, m, d);
1877
w--;
1878
}
1879
1880
/* call prefetch hint to optimize cache load*/
1881
cachePrefetch ((__m128i*)ps);
1882
cachePrefetch ((__m128i*)pd);
1883
cachePrefetch ((__m128i*)pm);
1884
1885
while (w >= 4)
1886
{
1887
/* fill cache line with next memory */
1888
cachePrefetchNext ((__m128i*)ps);
1889
cachePrefetchNext ((__m128i*)pd);
1890
cachePrefetchNext ((__m128i*)pm);
1891
1892
xmmDstHi = load128Aligned ((__m128i*)pd);
1893
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1894
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1895
1896
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1897
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1898
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1899
1900
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
1901
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1902
1903
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmSrcLo, &xmmSrcHi);
1904
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi, &xmmMaskLo, &xmmMaskHi);
1905
1906
negate_2x128 (xmmAlphaDstLo, xmmAlphaDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1907
1908
pixAddMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmMaskLo, &xmmMaskHi,
1909
&xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
1910
&xmmDstLo, &xmmDstHi);
1911
1912
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
1913
1914
ps += 4;
1915
pd += 4;
1916
pm += 4;
1917
w -= 4;
1918
}
1919
1920
while (w)
1921
{
1922
s = *ps++;
1923
m = *pm++;
1924
d = *pd;
1925
1926
*pd++ = coreCombineReverseAtopCPixelsse2 (s, m, d);
1927
w--;
1928
}
1929
}
1930
1931
static force_inline uint32_t
1932
coreCombineXorCPixelsse2 (uint32_t src, uint32_t mask, uint32_t dst)
1933
{
1934
__m64 a = unpack_32_1x64 (mask);
1935
__m64 s = unpack_32_1x64 (src);
1936
__m64 d = unpack_32_1x64 (dst);
1937
1938
__m64 alphaDst = negate_1x64 (pixMultiply_1x64 (a, expandAlpha_1x64 (s)));
1939
__m64 dest = pixMultiply_1x64 (s, a);
1940
__m64 alphaSrc = negate_1x64 (expandAlpha_1x64 (d));
1941
1942
return pack_1x64_32 (pixAddMultiply_1x64 (&d,
1943
&alphaDst,
1944
&dest,
1945
&alphaSrc));
1946
}
1947
1948
static force_inline void
1949
coreCombineXorCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
1950
{
1951
uint32_t s, m, d;
1952
1953
__m128i xmmSrcLo, xmmSrcHi;
1954
__m128i xmmDstLo, xmmDstHi;
1955
__m128i xmmAlphaSrcLo, xmmAlphaSrcHi;
1956
__m128i xmmAlphaDstLo, xmmAlphaDstHi;
1957
__m128i xmmMaskLo, xmmMaskHi;
1958
1959
/* call prefetch hint to optimize cache load*/
1960
cachePrefetch ((__m128i*)ps);
1961
cachePrefetch ((__m128i*)pd);
1962
cachePrefetch ((__m128i*)pm);
1963
1964
while (w && (unsigned long)pd & 15)
1965
{
1966
s = *ps++;
1967
m = *pm++;
1968
d = *pd;
1969
1970
*pd++ = coreCombineXorCPixelsse2 (s, m, d);
1971
w--;
1972
}
1973
1974
/* call prefetch hint to optimize cache load*/
1975
cachePrefetch ((__m128i*)ps);
1976
cachePrefetch ((__m128i*)pd);
1977
cachePrefetch ((__m128i*)pm);
1978
1979
while (w >= 4)
1980
{
1981
/* fill cache line with next memory */
1982
cachePrefetchNext ((__m128i*)ps);
1983
cachePrefetchNext ((__m128i*)pd);
1984
cachePrefetchNext ((__m128i*)pm);
1985
1986
xmmDstHi = load128Aligned ((__m128i*)pd);
1987
xmmSrcHi = load128Unaligned ((__m128i*)ps);
1988
xmmMaskHi = load128Unaligned ((__m128i*)pm);
1989
1990
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
1991
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
1992
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
1993
1994
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi);
1995
expandAlpha_2x128 (xmmDstLo, xmmDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
1996
1997
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmSrcLo, &xmmSrcHi);
1998
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmAlphaSrcLo, &xmmAlphaSrcHi, &xmmMaskLo, &xmmMaskHi);
1999
2000
negate_2x128 (xmmAlphaDstLo, xmmAlphaDstHi, &xmmAlphaDstLo, &xmmAlphaDstHi);
2001
negate_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
2002
2003
pixAddMultiply_2x128 (&xmmDstLo, &xmmDstHi, &xmmMaskLo, &xmmMaskHi,
2004
&xmmSrcLo, &xmmSrcHi, &xmmAlphaDstLo, &xmmAlphaDstHi,
2005
&xmmDstLo, &xmmDstHi);
2006
2007
save128Aligned( (__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
2008
2009
ps += 4;
2010
pd += 4;
2011
pm += 4;
2012
w -= 4;
2013
}
2014
2015
while (w)
2016
{
2017
s = *ps++;
2018
m = *pm++;
2019
d = *pd;
2020
2021
*pd++ = coreCombineXorCPixelsse2 (s, m, d);
2022
w--;
2023
}
2024
}
2025
2026
static force_inline void
2027
coreCombineAddCsse2 (uint32_t *pd, uint32_t *ps, uint32_t *pm, int w)
2028
{
2029
uint32_t s, m, d;
2030
2031
__m128i xmmSrcLo, xmmSrcHi;
2032
__m128i xmmDstLo, xmmDstHi;
2033
__m128i xmmMaskLo, xmmMaskHi;
2034
2035
/* call prefetch hint to optimize cache load*/
2036
cachePrefetch ((__m128i*)ps);
2037
cachePrefetch ((__m128i*)pd);
2038
cachePrefetch ((__m128i*)pm);
2039
2040
while (w && (unsigned long)pd & 15)
2041
{
2042
s = *ps++;
2043
m = *pm++;
2044
d = *pd;
2045
2046
*pd++ = pack_1x64_32 (_mm_adds_pu8 (pixMultiply_1x64 (unpack_32_1x64 (s),
2047
unpack_32_1x64 (m)),
2048
unpack_32_1x64 (d)));
2049
w--;
2050
}
2051
2052
/* call prefetch hint to optimize cache load*/
2053
cachePrefetch ((__m128i*)ps);
2054
cachePrefetch ((__m128i*)pd);
2055
cachePrefetch ((__m128i*)pm);
2056
2057
while (w >= 4)
2058
{
2059
/* fill cache line with next memory */
2060
cachePrefetchNext ((__m128i*)ps);
2061
cachePrefetchNext ((__m128i*)pd);
2062
cachePrefetchNext ((__m128i*)pm);
2063
2064
xmmSrcHi = load128Unaligned ((__m128i*)ps);
2065
xmmMaskHi = load128Unaligned ((__m128i*)pm);
2066
xmmDstHi = load128Aligned ((__m128i*)pd);
2067
2068
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
2069
unpack_128_2x128 (xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
2070
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
2071
2072
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmMaskLo, &xmmMaskHi, &xmmSrcLo, &xmmSrcHi);
2073
2074
save128Aligned( (__m128i*)pd, pack_2x128_128 (_mm_adds_epu8 (xmmSrcLo, xmmDstLo),
2075
_mm_adds_epu8 (xmmSrcHi, xmmDstHi)));
2076
2077
ps += 4;
2078
pd += 4;
2079
pm += 4;
2080
w -= 4;
2081
}
2082
2083
while (w)
2084
{
2085
s = *ps++;
2086
m = *pm++;
2087
d = *pd;
2088
2089
*pd++ = pack_1x64_32 (_mm_adds_pu8 (pixMultiply_1x64 (unpack_32_1x64 (s),
2090
unpack_32_1x64 (m)),
2091
unpack_32_1x64 (d)));
2092
w--;
2093
}
2094
}
2095
2096
/* -------------------------------------------------------------------------------------------------
2097
* fbComposeSetupSSE2
2098
*/
2099
static force_inline __m64
2100
createMask_16_64 (uint16_t mask)
2101
{
2102
return _mm_set1_pi16 (mask);
2103
}
2104
2105
static force_inline __m128i
2106
createMask_16_128 (uint16_t mask)
2107
{
2108
return _mm_set1_epi16 (mask);
2109
}
2110
2111
static force_inline __m64
2112
createMask_2x32_64 (uint32_t mask0, uint32_t mask1)
2113
{
2114
return _mm_set_pi32 (mask0, mask1);
2115
}
2116
2117
static force_inline __m128i
2118
createMask_2x32_128 (uint32_t mask0, uint32_t mask1)
2119
{
2120
return _mm_set_epi32 (mask0, mask1, mask0, mask1);
2121
}
2122
2123
/* SSE2 code patch for fbcompose.c */
2124
2125
static FASTCALL void
2126
sse2CombineMaskU (uint32_t *dst, const uint32_t *src, int width)
2127
{
2128
coreCombineReverseInUsse2 (dst, src, width);
2129
_mm_empty();
2130
}
2131
2132
static FASTCALL void
2133
sse2CombineOverU (uint32_t *dst, const uint32_t *src, int width)
2134
{
2135
coreCombineOverUsse2 (dst, src, width);
2136
_mm_empty();
2137
}
2138
2139
static FASTCALL void
2140
sse2CombineOverReverseU (uint32_t *dst, const uint32_t *src, int width)
2141
{
2142
coreCombineOverReverseUsse2 (dst, src, width);
2143
_mm_empty();
2144
}
2145
2146
static FASTCALL void
2147
sse2CombineInU (uint32_t *dst, const uint32_t *src, int width)
2148
{
2149
coreCombineInUsse2 (dst, src, width);
2150
_mm_empty();
2151
}
2152
2153
static FASTCALL void
2154
sse2CombineInReverseU (uint32_t *dst, const uint32_t *src, int width)
2155
{
2156
coreCombineReverseInUsse2 (dst, src, width);
2157
_mm_empty();
2158
}
2159
2160
static FASTCALL void
2161
sse2CombineOutU (uint32_t *dst, const uint32_t *src, int width)
2162
{
2163
coreCombineOutUsse2 (dst, src, width);
2164
_mm_empty();
2165
}
2166
2167
static FASTCALL void
2168
sse2CombineOutReverseU (uint32_t *dst, const uint32_t *src, int width)
2169
{
2170
coreCombineReverseOutUsse2 (dst, src, width);
2171
_mm_empty();
2172
}
2173
2174
static FASTCALL void
2175
sse2CombineAtopU (uint32_t *dst, const uint32_t *src, int width)
2176
{
2177
coreCombineAtopUsse2 (dst, src, width);
2178
_mm_empty();
2179
}
2180
2181
static FASTCALL void
2182
sse2CombineAtopReverseU (uint32_t *dst, const uint32_t *src, int width)
2183
{
2184
coreCombineReverseAtopUsse2 (dst, src, width);
2185
_mm_empty();
2186
}
2187
2188
static FASTCALL void
2189
sse2CombineXorU (uint32_t *dst, const uint32_t *src, int width)
2190
{
2191
coreCombineXorUsse2 (dst, src, width);
2192
_mm_empty();
2193
}
2194
2195
static FASTCALL void
2196
sse2CombineAddU (uint32_t *dst, const uint32_t *src, int width)
2197
{
2198
coreCombineAddUsse2 (dst, src, width);
2199
_mm_empty();
2200
}
2201
2202
static FASTCALL void
2203
sse2CombineSaturateU (uint32_t *dst, const uint32_t *src, int width)
2204
{
2205
coreCombineSaturateUsse2 (dst, src, width);
2206
_mm_empty();
2207
}
2208
2209
static FASTCALL void
2210
sse2CombineSrcC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2211
{
2212
coreCombineSrcCsse2 (dst, src, mask, width);
2213
_mm_empty();
2214
}
2215
2216
static FASTCALL void
2217
sse2CombineOverC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2218
{
2219
coreCombineOverCsse2 (dst, src, mask, width);
2220
_mm_empty();
2221
}
2222
2223
static FASTCALL void
2224
sse2CombineOverReverseC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2225
{
2226
coreCombineOverReverseCsse2 (dst, src, mask, width);
2227
_mm_empty();
2228
}
2229
2230
static FASTCALL void
2231
sse2CombineInC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2232
{
2233
coreCombineInCsse2 (dst, src, mask, width);
2234
_mm_empty();
2235
}
2236
2237
static FASTCALL void
2238
sse2CombineInReverseC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2239
{
2240
coreCombineInReverseCsse2 (dst, src, mask, width);
2241
_mm_empty();
2242
}
2243
2244
static FASTCALL void
2245
sse2CombineOutC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2246
{
2247
coreCombineOutCsse2 (dst, src, mask, width);
2248
_mm_empty();
2249
}
2250
2251
static FASTCALL void
2252
sse2CombineOutReverseC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2253
{
2254
coreCombineOutReverseCsse2 (dst, src, mask, width);
2255
_mm_empty();
2256
}
2257
2258
static FASTCALL void
2259
sse2CombineAtopC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2260
{
2261
coreCombineAtopCsse2 (dst, src, mask, width);
2262
_mm_empty();
2263
}
2264
2265
static FASTCALL void
2266
sse2CombineAtopReverseC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2267
{
2268
coreCombineReverseAtopCsse2 (dst, src, mask, width);
2269
_mm_empty();
2270
}
2271
2272
static FASTCALL void
2273
sse2CombineXorC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2274
{
2275
coreCombineXorCsse2 (dst, src, mask, width);
2276
_mm_empty();
2277
}
2278
2279
static FASTCALL void
2280
sse2CombineAddC (uint32_t *dst, uint32_t *src, uint32_t *mask, int width)
2281
{
2282
coreCombineAddCsse2 (dst, src, mask, width);
2283
_mm_empty();
2284
}
2285
2286
void
2287
fbComposeSetupSSE2(void)
2288
{
2289
static pixman_bool_t initialized = FALSE;
2290
2291
if (initialized)
2292
return;
2293
2294
/* check if we have SSE2 support and initialize accordingly */
2295
if (pixman_have_sse2())
2296
{
2297
/* SSE2 constants */
2298
Mask565r = createMask_2x32_128 (0x00f80000, 0x00f80000);
2299
Mask565g1 = createMask_2x32_128 (0x00070000, 0x00070000);
2300
Mask565g2 = createMask_2x32_128 (0x000000e0, 0x000000e0);
2301
Mask565b = createMask_2x32_128 (0x0000001f, 0x0000001f);
2302
MaskRed = createMask_2x32_128 (0x00f80000, 0x00f80000);
2303
MaskGreen = createMask_2x32_128 (0x0000fc00, 0x0000fc00);
2304
MaskBlue = createMask_2x32_128 (0x000000f8, 0x000000f8);
2305
Mask565FixRB = createMask_2x32_128 (0x00e000e0, 0x00e000e0);
2306
Mask565FixG = createMask_2x32_128 (0x0000c000, 0x0000c000);
2307
Mask0080 = createMask_16_128 (0x0080);
2308
Mask00ff = createMask_16_128 (0x00ff);
2309
Mask0101 = createMask_16_128 (0x0101);
2310
Maskffff = createMask_16_128 (0xffff);
2311
Maskff000000 = createMask_2x32_128 (0xff000000, 0xff000000);
2312
MaskAlpha = createMask_2x32_128 (0x00ff0000, 0x00000000);
2313
2314
/* MMX constants */
2315
xMask565rgb = createMask_2x32_64 (0x000001f0, 0x003f001f);
2316
xMask565Unpack = createMask_2x32_64 (0x00000084, 0x04100840);
2317
2318
xMask0080 = createMask_16_64 (0x0080);
2319
xMask00ff = createMask_16_64 (0x00ff);
2320
xMask0101 = createMask_16_64 (0x0101);
2321
xMaskAlpha = createMask_2x32_64 (0x00ff0000, 0x00000000);
2322
2323
/* SSE code patch for fbcompose.c */
2324
pixman_composeFunctions.combineU[PIXMAN_OP_OVER] = sse2CombineOverU;
2325
pixman_composeFunctions.combineU[PIXMAN_OP_OVER_REVERSE] = sse2CombineOverReverseU;
2326
pixman_composeFunctions.combineU[PIXMAN_OP_IN] = sse2CombineInU;
2327
pixman_composeFunctions.combineU[PIXMAN_OP_IN_REVERSE] = sse2CombineInReverseU;
2328
pixman_composeFunctions.combineU[PIXMAN_OP_OUT] = sse2CombineOutU;
2329
2330
pixman_composeFunctions.combineU[PIXMAN_OP_OUT_REVERSE] = sse2CombineOutReverseU;
2331
pixman_composeFunctions.combineU[PIXMAN_OP_ATOP] = sse2CombineAtopU;
2332
pixman_composeFunctions.combineU[PIXMAN_OP_ATOP_REVERSE] = sse2CombineAtopReverseU;
2333
pixman_composeFunctions.combineU[PIXMAN_OP_XOR] = sse2CombineXorU;
2334
pixman_composeFunctions.combineU[PIXMAN_OP_ADD] = sse2CombineAddU;
2335
2336
pixman_composeFunctions.combineU[PIXMAN_OP_SATURATE] = sse2CombineSaturateU;
2337
2338
pixman_composeFunctions.combineC[PIXMAN_OP_SRC] = sse2CombineSrcC;
2339
pixman_composeFunctions.combineC[PIXMAN_OP_OVER] = sse2CombineOverC;
2340
pixman_composeFunctions.combineC[PIXMAN_OP_OVER_REVERSE] = sse2CombineOverReverseC;
2341
pixman_composeFunctions.combineC[PIXMAN_OP_IN] = sse2CombineInC;
2342
pixman_composeFunctions.combineC[PIXMAN_OP_IN_REVERSE] = sse2CombineInReverseC;
2343
pixman_composeFunctions.combineC[PIXMAN_OP_OUT] = sse2CombineOutC;
2344
pixman_composeFunctions.combineC[PIXMAN_OP_OUT_REVERSE] = sse2CombineOutReverseC;
2345
pixman_composeFunctions.combineC[PIXMAN_OP_ATOP] = sse2CombineAtopC;
2346
pixman_composeFunctions.combineC[PIXMAN_OP_ATOP_REVERSE] = sse2CombineAtopReverseC;
2347
pixman_composeFunctions.combineC[PIXMAN_OP_XOR] = sse2CombineXorC;
2348
pixman_composeFunctions.combineC[PIXMAN_OP_ADD] = sse2CombineAddC;
2349
2350
pixman_composeFunctions.combineMaskU = sse2CombineMaskU;
2351
2352
_mm_empty();
2353
}
2354
2355
initialized = TRUE;
2356
}
2357
2358
2359
/* -------------------------------------------------------------------------------------------------
2360
* fbCompositeSolid_nx8888
2361
*/
2362
2363
void
2364
fbCompositeSolid_nx8888sse2 (pixman_op_t op,
2365
pixman_image_t * pSrc,
2366
pixman_image_t * pMask,
2367
pixman_image_t * pDst,
2368
int16_t xSrc,
2369
int16_t ySrc,
2370
int16_t xMask,
2371
int16_t yMask,
2372
int16_t xDst,
2373
int16_t yDst,
2374
uint16_t width,
2375
uint16_t height)
2376
{
2377
uint32_t src;
2378
uint32_t *dstLine, *dst, d;
2379
uint16_t w;
2380
int dstStride;
2381
__m128i xmmSrc, xmmAlpha;
2382
__m128i xmmDst, xmmDstLo, xmmDstHi;
2383
2384
fbComposeGetSolid(pSrc, src, pDst->bits.format);
2385
2386
if (src >> 24 == 0)
2387
return;
2388
2389
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
2390
2391
xmmSrc = expandPixel_32_1x128 (src);
2392
xmmAlpha = expandAlpha_1x128 (xmmSrc);
2393
2394
while (height--)
2395
{
2396
dst = dstLine;
2397
2398
/* call prefetch hint to optimize cache load*/
2399
cachePrefetch ((__m128i*)dst);
2400
2401
dstLine += dstStride;
2402
w = width;
2403
2404
while (w && (unsigned long)dst & 15)
2405
{
2406
d = *dst;
2407
*dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmmSrc),
2408
_mm_movepi64_pi64 (xmmAlpha),
2409
unpack_32_1x64 (d)));
2410
w--;
2411
}
2412
2413
cachePrefetch ((__m128i*)dst);
2414
2415
while (w >= 4)
2416
{
2417
/* fill cache line with next memory */
2418
cachePrefetchNext ((__m128i*)dst);
2419
2420
xmmDst = load128Aligned ((__m128i*)dst);
2421
2422
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
2423
2424
over_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmDstLo, &xmmDstHi);
2425
2426
/* rebuid the 4 pixel data and save*/
2427
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
2428
2429
w -= 4;
2430
dst += 4;
2431
}
2432
2433
while (w)
2434
{
2435
d = *dst;
2436
*dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmmSrc),
2437
_mm_movepi64_pi64 (xmmAlpha),
2438
unpack_32_1x64 (d)));
2439
w--;
2440
}
2441
2442
}
2443
_mm_empty();
2444
}
2445
2446
/* -------------------------------------------------------------------------------------------------
2447
* fbCompositeSolid_nx0565
2448
*/
2449
void
2450
fbCompositeSolid_nx0565sse2 (pixman_op_t op,
2451
pixman_image_t * pSrc,
2452
pixman_image_t * pMask,
2453
pixman_image_t * pDst,
2454
int16_t xSrc,
2455
int16_t ySrc,
2456
int16_t xMask,
2457
int16_t yMask,
2458
int16_t xDst,
2459
int16_t yDst,
2460
uint16_t width,
2461
uint16_t height)
2462
{
2463
uint32_t src;
2464
uint16_t *dstLine, *dst, d;
2465
uint16_t w;
2466
int dstStride;
2467
__m128i xmmSrc, xmmAlpha;
2468
__m128i xmmDst, xmmDst0, xmmDst1, xmmDst2, xmmDst3;
2469
2470
fbComposeGetSolid(pSrc, src, pDst->bits.format);
2471
2472
if (src >> 24 == 0)
2473
return;
2474
2475
fbComposeGetStart (pDst, xDst, yDst, uint16_t, dstStride, dstLine, 1);
2476
2477
xmmSrc = expandPixel_32_1x128 (src);
2478
xmmAlpha = expandAlpha_1x128 (xmmSrc);
2479
2480
while (height--)
2481
{
2482
dst = dstLine;
2483
2484
/* call prefetch hint to optimize cache load*/
2485
cachePrefetch ((__m128i*)dst);
2486
2487
dstLine += dstStride;
2488
w = width;
2489
2490
while (w && (unsigned long)dst & 15)
2491
{
2492
d = *dst;
2493
2494
*dst++ = pack565_32_16 (pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmmSrc),
2495
_mm_movepi64_pi64 (xmmAlpha),
2496
expand565_16_1x64 (d))));
2497
w--;
2498
}
2499
2500
/* call prefetch hint to optimize cache load*/
2501
cachePrefetch ((__m128i*)dst);
2502
2503
while (w >= 8)
2504
{
2505
/* fill cache line with next memory */
2506
cachePrefetchNext ((__m128i*)dst);
2507
2508
xmmDst = load128Aligned ((__m128i*)dst);
2509
2510
unpack565_128_4x128 (xmmDst, &xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
2511
2512
over_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmDst0, &xmmDst1);
2513
over_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmDst2, &xmmDst3);
2514
2515
xmmDst = pack565_4x128_128 (&xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
2516
save128Aligned ((__m128i*)dst, xmmDst);
2517
2518
dst += 8;
2519
w -= 8;
2520
}
2521
2522
while (w--)
2523
{
2524
d = *dst;
2525
*dst++ = pack565_32_16 (pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmmSrc),
2526
_mm_movepi64_pi64 (xmmAlpha),
2527
expand565_16_1x64 (d))));
2528
}
2529
}
2530
2531
_mm_empty();
2532
}
2533
2534
/* -------------------------------------------------------------------------------------------------
2535
* fbCompositeSolidMask_nx8888x8888C
2536
*/
2537
2538
void
2539
fbCompositeSolidMask_nx8888x8888Csse2 (pixman_op_t op,
2540
pixman_image_t * pSrc,
2541
pixman_image_t * pMask,
2542
pixman_image_t * pDst,
2543
int16_t xSrc,
2544
int16_t ySrc,
2545
int16_t xMask,
2546
int16_t yMask,
2547
int16_t xDst,
2548
int16_t yDst,
2549
uint16_t width,
2550
uint16_t height)
2551
{
2552
uint32_t src, srca;
2553
uint32_t *dstLine, d;
2554
uint32_t *maskLine, m;
2555
uint32_t packCmp;
2556
int dstStride, maskStride;
2557
2558
__m128i xmmSrc, xmmAlpha;
2559
__m128i xmmDst, xmmDstLo, xmmDstHi;
2560
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
2561
2562
__m64 mmxSrc, mmxAlpha, mmxMask, mmxDst;
2563
2564
fbComposeGetSolid(pSrc, src, pDst->bits.format);
2565
2566
srca = src >> 24;
2567
if (srca == 0)
2568
return;
2569
2570
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
2571
fbComposeGetStart (pMask, xMask, yMask, uint32_t, maskStride, maskLine, 1);
2572
2573
xmmSrc = _mm_unpacklo_epi8 (createMask_2x32_128 (src, src), _mm_setzero_si128 ());
2574
xmmAlpha = expandAlpha_1x128 (xmmSrc);
2575
mmxSrc = _mm_movepi64_pi64 (xmmSrc);
2576
mmxAlpha = _mm_movepi64_pi64 (xmmAlpha);
2577
2578
while (height--)
2579
{
2580
int w = width;
2581
uint32_t *pm = (uint32_t *)maskLine;
2582
uint32_t *pd = (uint32_t *)dstLine;
2583
2584
dstLine += dstStride;
2585
maskLine += maskStride;
2586
2587
/* call prefetch hint to optimize cache load*/
2588
cachePrefetch ((__m128i*)pd);
2589
cachePrefetch ((__m128i*)pm);
2590
2591
while (w && (unsigned long)pd & 15)
2592
{
2593
m = *pm++;
2594
2595
if (m)
2596
{
2597
d = *pd;
2598
mmxMask = unpack_32_1x64 (m);
2599
mmxDst = unpack_32_1x64 (d);
2600
2601
*pd = pack_1x64_32 (inOver_1x64 (&mmxSrc,
2602
&mmxAlpha,
2603
&mmxMask,
2604
&mmxDst));
2605
}
2606
2607
pd++;
2608
w--;
2609
}
2610
2611
/* call prefetch hint to optimize cache load*/
2612
cachePrefetch ((__m128i*)pd);
2613
cachePrefetch ((__m128i*)pm);
2614
2615
while (w >= 4)
2616
{
2617
/* fill cache line with next memory */
2618
cachePrefetchNext ((__m128i*)pd);
2619
cachePrefetchNext ((__m128i*)pm);
2620
2621
xmmMask = load128Unaligned ((__m128i*)pm);
2622
2623
packCmp = _mm_movemask_epi8 (_mm_cmpeq_epi32 (xmmMask, _mm_setzero_si128()));
2624
2625
/* if all bits in mask are zero, packCmp are equal to 0xffff */
2626
if (packCmp != 0xffff)
2627
{
2628
xmmDst = load128Aligned ((__m128i*)pd);
2629
2630
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
2631
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
2632
2633
inOver_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
2634
2635
save128Aligned ((__m128i*)pd, pack_2x128_128 (xmmDstLo, xmmDstHi));
2636
}
2637
2638
pd += 4;
2639
pm += 4;
2640
w -= 4;
2641
}
2642
2643
while (w)
2644
{
2645
m = *pm++;
2646
2647
if (m)
2648
{
2649
d = *pd;
2650
mmxMask = unpack_32_1x64 (m);
2651
mmxDst = unpack_32_1x64 (d);
2652
2653
*pd = pack_1x64_32 (inOver_1x64 (&mmxSrc,
2654
&mmxAlpha,
2655
&mmxMask,
2656
&mmxDst));
2657
}
2658
2659
pd++;
2660
w--;
2661
}
2662
}
2663
2664
_mm_empty();
2665
}
2666
2667
2668
/* -------------------------------------------------------------------------------------------------
2669
* fbCompositeSrc_8888x8x8888
2670
*/
2671
2672
void
2673
fbCompositeSrc_8888x8x8888sse2 (pixman_op_t op,
2674
pixman_image_t * pSrc,
2675
pixman_image_t * pMask,
2676
pixman_image_t * pDst,
2677
int16_t xSrc,
2678
int16_t ySrc,
2679
int16_t xMask,
2680
int16_t yMask,
2681
int16_t xDst,
2682
int16_t yDst,
2683
uint16_t width,
2684
uint16_t height)
2685
{
2686
uint32_t *dstLine, *dst;
2687
uint32_t *srcLine, *src;
2688
uint32_t mask;
2689
uint16_t w;
2690
int dstStride, srcStride;
2691
2692
__m128i xmmMask;
2693
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
2694
__m128i xmmDst, xmmDstLo, xmmDstHi;
2695
__m128i xmmAlphaLo, xmmAlphaHi;
2696
2697
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
2698
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
2699
fbComposeGetSolid (pMask, mask, pDst->bits.format);
2700
2701
xmmMask = createMask_16_128 (mask >> 24);
2702
2703
while (height--)
2704
{
2705
dst = dstLine;
2706
dstLine += dstStride;
2707
src = srcLine;
2708
srcLine += srcStride;
2709
w = width;
2710
2711
/* call prefetch hint to optimize cache load*/
2712
cachePrefetch ((__m128i*)dst);
2713
cachePrefetch ((__m128i*)src);
2714
2715
while (w && (unsigned long)dst & 15)
2716
{
2717
uint32_t s = *src++;
2718
uint32_t d = *dst;
2719
2720
__m64 ms = unpack_32_1x64 (s);
2721
__m64 alpha = expandAlpha_1x64 (ms);
2722
__m64 dest = _mm_movepi64_pi64 (xmmMask);
2723
__m64 alphaDst = unpack_32_1x64 (d);
2724
2725
*dst++ = pack_1x64_32 (inOver_1x64 (&ms,
2726
&alpha,
2727
&dest,
2728
&alphaDst));
2729
2730
w--;
2731
}
2732
2733
/* call prefetch hint to optimize cache load*/
2734
cachePrefetch ((__m128i*)dst);
2735
cachePrefetch ((__m128i*)src);
2736
2737
while (w >= 4)
2738
{
2739
/* fill cache line with next memory */
2740
cachePrefetchNext ((__m128i*)dst);
2741
cachePrefetchNext ((__m128i*)src);
2742
2743
xmmSrc = load128Unaligned ((__m128i*)src);
2744
xmmDst = load128Aligned ((__m128i*)dst);
2745
2746
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
2747
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
2748
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
2749
2750
inOver_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi, &xmmMask, &xmmMask, &xmmDstLo, &xmmDstHi);
2751
2752
save128Aligned( (__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
2753
2754
dst += 4;
2755
src += 4;
2756
w -= 4;
2757
}
2758
2759
while (w)
2760
{
2761
uint32_t s = *src++;
2762
uint32_t d = *dst;
2763
2764
__m64 ms = unpack_32_1x64 (s);
2765
__m64 alpha = expandAlpha_1x64 (ms);
2766
__m64 mask = _mm_movepi64_pi64 (xmmMask);
2767
__m64 dest = unpack_32_1x64 (d);
2768
2769
*dst++ = pack_1x64_32 (inOver_1x64 (&ms,
2770
&alpha,
2771
&mask,
2772
&dest));
2773
2774
w--;
2775
}
2776
}
2777
2778
_mm_empty();
2779
}
2780
2781
/* -------------------------------------------------------------------------------------------------
2782
* fbCompositeSrc_x888xnx8888
2783
*/
2784
void
2785
fbCompositeSrc_x888xnx8888sse2 (pixman_op_t op,
2786
pixman_image_t * pSrc,
2787
pixman_image_t * pMask,
2788
pixman_image_t * pDst,
2789
int16_t xSrc,
2790
int16_t ySrc,
2791
int16_t xMask,
2792
int16_t yMask,
2793
int16_t xDst,
2794
int16_t yDst,
2795
uint16_t width,
2796
uint16_t height)
2797
{
2798
uint32_t *dstLine, *dst;
2799
uint32_t *srcLine, *src;
2800
uint32_t mask;
2801
int dstStride, srcStride;
2802
uint16_t w;
2803
2804
__m128i xmmMask, xmmAlpha;
2805
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
2806
__m128i xmmDst, xmmDstLo, xmmDstHi;
2807
2808
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
2809
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
2810
fbComposeGetSolid (pMask, mask, pDst->bits.format);
2811
2812
xmmMask = createMask_16_128 (mask >> 24);
2813
xmmAlpha = Mask00ff;
2814
2815
while (height--)
2816
{
2817
dst = dstLine;
2818
dstLine += dstStride;
2819
src = srcLine;
2820
srcLine += srcStride;
2821
w = width;
2822
2823
/* call prefetch hint to optimize cache load*/
2824
cachePrefetch ((__m128i*)dst);
2825
cachePrefetch ((__m128i*)src);
2826
2827
while (w && (unsigned long)dst & 15)
2828
{
2829
uint32_t s = (*src++) | 0xff000000;
2830
uint32_t d = *dst;
2831
2832
__m64 src = unpack_32_1x64 (s);
2833
__m64 alpha = _mm_movepi64_pi64 (xmmAlpha);
2834
__m64 mask = _mm_movepi64_pi64 (xmmMask);
2835
__m64 dest = unpack_32_1x64 (d);
2836
2837
*dst++ = pack_1x64_32 (inOver_1x64 (&src,
2838
&alpha,
2839
&mask,
2840
&dest));
2841
2842
w--;
2843
}
2844
2845
/* call prefetch hint to optimize cache load*/
2846
cachePrefetch ((__m128i*)dst);
2847
cachePrefetch ((__m128i*)src);
2848
2849
while (w >= 4)
2850
{
2851
/* fill cache line with next memory */
2852
cachePrefetchNext ((__m128i*)dst);
2853
cachePrefetchNext ((__m128i*)src);
2854
2855
xmmSrc = _mm_or_si128 (load128Unaligned ((__m128i*)src), Maskff000000);
2856
xmmDst = load128Aligned ((__m128i*)dst);
2857
2858
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
2859
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
2860
2861
inOver_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlpha, &xmmAlpha, &xmmMask, &xmmMask, &xmmDstLo, &xmmDstHi);
2862
2863
save128Aligned( (__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
2864
2865
dst += 4;
2866
src += 4;
2867
w -= 4;
2868
2869
}
2870
2871
while (w)
2872
{
2873
uint32_t s = (*src++) | 0xff000000;
2874
uint32_t d = *dst;
2875
2876
__m64 src = unpack_32_1x64 (s);
2877
__m64 alpha = _mm_movepi64_pi64 (xmmAlpha);
2878
__m64 mask = _mm_movepi64_pi64 (xmmMask);
2879
__m64 dest = unpack_32_1x64 (d);
2880
2881
*dst++ = pack_1x64_32 (inOver_1x64 (&src,
2882
&alpha,
2883
&mask,
2884
&dest));
2885
2886
w--;
2887
}
2888
}
2889
2890
_mm_empty();
2891
}
2892
2893
/* -------------------------------------------------------------------------------------------------
2894
* fbCompositeSrc_8888x8888
2895
*/
2896
void
2897
fbCompositeSrc_8888x8888sse2 (pixman_op_t op,
2898
pixman_image_t * pSrc,
2899
pixman_image_t * pMask,
2900
pixman_image_t * pDst,
2901
int16_t xSrc,
2902
int16_t ySrc,
2903
int16_t xMask,
2904
int16_t yMask,
2905
int16_t xDst,
2906
int16_t yDst,
2907
uint16_t width,
2908
uint16_t height)
2909
{
2910
int dstStride, srcStride;
2911
uint32_t *dstLine, *dst;
2912
uint32_t *srcLine, *src;
2913
2914
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
2915
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
2916
2917
dst = dstLine;
2918
src = srcLine;
2919
2920
while (height--)
2921
{
2922
coreCombineOverUsse2 (dst, src, width);
2923
2924
dst += dstStride;
2925
src += srcStride;
2926
}
2927
_mm_empty();
2928
}
2929
2930
/* -------------------------------------------------------------------------------------------------
2931
* fbCompositeSrc_8888x0565
2932
*/
2933
static force_inline uint16_t
2934
fbCompositeSrc_8888x0565pixel (uint32_t src, uint16_t dst)
2935
{
2936
__m64 ms;
2937
2938
ms = unpack_32_1x64 (src);
2939
return pack565_32_16( pack_1x64_32 (over_1x64 (ms,
2940
expandAlpha_1x64 (ms),
2941
expand565_16_1x64 (dst))));
2942
}
2943
2944
void
2945
fbCompositeSrc_8888x0565sse2 (pixman_op_t op,
2946
pixman_image_t * pSrc,
2947
pixman_image_t * pMask,
2948
pixman_image_t * pDst,
2949
int16_t xSrc,
2950
int16_t ySrc,
2951
int16_t xMask,
2952
int16_t yMask,
2953
int16_t xDst,
2954
int16_t yDst,
2955
uint16_t width,
2956
uint16_t height)
2957
{
2958
uint16_t *dstLine, *dst, d;
2959
uint32_t *srcLine, *src, s;
2960
int dstStride, srcStride;
2961
uint16_t w;
2962
2963
__m128i xmmAlphaLo, xmmAlphaHi;
2964
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
2965
__m128i xmmDst, xmmDst0, xmmDst1, xmmDst2, xmmDst3;
2966
2967
fbComposeGetStart (pDst, xDst, yDst, uint16_t, dstStride, dstLine, 1);
2968
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
2969
2970
#if 0
2971
/* FIXME
2972
*
2973
* I copy the code from MMX one and keep the fixme.
2974
* If it's a problem there, probably is a problem here.
2975
*/
2976
assert (pSrc->pDrawable == pMask->pDrawable);
2977
#endif
2978
2979
while (height--)
2980
{
2981
dst = dstLine;
2982
src = srcLine;
2983
2984
/* call prefetch hint to optimize cache load*/
2985
cachePrefetch ((__m128i*)src);
2986
cachePrefetch ((__m128i*)dst);
2987
2988
dstLine += dstStride;
2989
srcLine += srcStride;
2990
w = width;
2991
2992
/* Align dst on a 16-byte boundary */
2993
while (w &&
2994
((unsigned long)dst & 15))
2995
{
2996
s = *src++;
2997
d = *dst;
2998
2999
*dst++ = fbCompositeSrc_8888x0565pixel (s, d);
3000
w--;
3001
}
3002
3003
/* call prefetch hint to optimize cache load*/
3004
cachePrefetch ((__m128i*)src);
3005
cachePrefetch ((__m128i*)dst);
3006
3007
/* It's a 8 pixel loop */
3008
while (w >= 8)
3009
{
3010
/* fill cache line with next memory */
3011
cachePrefetchNext ((__m128i*)src);
3012
cachePrefetchNext ((__m128i*)dst);
3013
3014
/* I'm loading unaligned because I'm not sure about the address alignment. */
3015
xmmSrc = load128Unaligned ((__m128i*) src);
3016
xmmDst = load128Aligned ((__m128i*) dst);
3017
3018
/* Unpacking */
3019
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
3020
unpack565_128_4x128 (xmmDst, &xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
3021
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
3022
3023
/* I'm loading next 4 pixels from memory before to optimze the memory read. */
3024
xmmSrc = load128Unaligned ((__m128i*) (src+4));
3025
3026
over_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDst0, &xmmDst1);
3027
3028
/* Unpacking */
3029
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
3030
expandAlpha_2x128 (xmmSrcLo, xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi);
3031
3032
over_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmAlphaLo, &xmmAlphaHi, &xmmDst2, &xmmDst3);
3033
3034
save128Aligned ((__m128i*)dst, pack565_4x128_128 (&xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3));
3035
3036
w -= 8;
3037
dst += 8;
3038
src += 8;
3039
}
3040
3041
while (w--)
3042
{
3043
s = *src++;
3044
d = *dst;
3045
3046
*dst++ = fbCompositeSrc_8888x0565pixel (s, d);
3047
}
3048
}
3049
3050
_mm_empty();
3051
}
3052
3053
/* -------------------------------------------------------------------------------------------------
3054
* fbCompositeSolidMask_nx8x8888
3055
*/
3056
3057
void
3058
fbCompositeSolidMask_nx8x8888sse2 (pixman_op_t op,
3059
pixman_image_t * pSrc,
3060
pixman_image_t * pMask,
3061
pixman_image_t * pDst,
3062
int16_t xSrc,
3063
int16_t ySrc,
3064
int16_t xMask,
3065
int16_t yMask,
3066
int16_t xDst,
3067
int16_t yDst,
3068
uint16_t width,
3069
uint16_t height)
3070
{
3071
uint32_t src, srca;
3072
uint32_t *dstLine, *dst;
3073
uint8_t *maskLine, *mask;
3074
int dstStride, maskStride;
3075
uint16_t w;
3076
uint32_t m, d;
3077
3078
__m128i xmmSrc, xmmAlpha, xmmDef;
3079
__m128i xmmDst, xmmDstLo, xmmDstHi;
3080
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
3081
3082
__m64 mmxSrc, mmxAlpha, mmxMask, mmxDest;
3083
3084
fbComposeGetSolid(pSrc, src, pDst->bits.format);
3085
3086
srca = src >> 24;
3087
if (srca == 0)
3088
return;
3089
3090
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
3091
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
3092
3093
xmmDef = createMask_2x32_128 (src, src);
3094
xmmSrc = expandPixel_32_1x128 (src);
3095
xmmAlpha = expandAlpha_1x128 (xmmSrc);
3096
mmxSrc = _mm_movepi64_pi64 (xmmSrc);
3097
mmxAlpha = _mm_movepi64_pi64 (xmmAlpha);
3098
3099
while (height--)
3100
{
3101
dst = dstLine;
3102
dstLine += dstStride;
3103
mask = maskLine;
3104
maskLine += maskStride;
3105
w = width;
3106
3107
/* call prefetch hint to optimize cache load*/
3108
cachePrefetch ((__m128i*)mask);
3109
cachePrefetch ((__m128i*)dst);
3110
3111
while (w && (unsigned long)dst & 15)
3112
{
3113
uint8_t m = *mask++;
3114
3115
if (m)
3116
{
3117
d = *dst;
3118
mmxMask = expandPixel_8_1x64 (m);
3119
mmxDest = unpack_32_1x64 (d);
3120
3121
*dst = pack_1x64_32 (inOver_1x64 (&mmxSrc,
3122
&mmxAlpha,
3123
&mmxMask,
3124
&mmxDest));
3125
}
3126
3127
w--;
3128
dst++;
3129
}
3130
3131
/* call prefetch hint to optimize cache load*/
3132
cachePrefetch ((__m128i*)mask);
3133
cachePrefetch ((__m128i*)dst);
3134
3135
while (w >= 4)
3136
{
3137
/* fill cache line with next memory */
3138
cachePrefetchNext ((__m128i*)mask);
3139
cachePrefetchNext ((__m128i*)dst);
3140
3141
m = *((uint32_t*)mask);
3142
3143
if (srca == 0xff && m == 0xffffffff)
3144
{
3145
save128Aligned ((__m128i*)dst, xmmDef);
3146
}
3147
else if (m)
3148
{
3149
xmmDst = load128Aligned ((__m128i*) dst);
3150
xmmMask = unpack_32_1x128 (m);
3151
xmmMask = _mm_unpacklo_epi8 (xmmMask, _mm_setzero_si128());
3152
3153
/* Unpacking */
3154
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
3155
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3156
3157
expandAlphaRev_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
3158
3159
inOver_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi);
3160
3161
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
3162
}
3163
3164
w -= 4;
3165
dst += 4;
3166
mask += 4;
3167
}
3168
3169
while (w)
3170
{
3171
uint8_t m = *mask++;
3172
3173
if (m)
3174
{
3175
d = *dst;
3176
mmxMask = expandPixel_8_1x64 (m);
3177
mmxDest = unpack_32_1x64 (d);
3178
3179
*dst = pack_1x64_32 (inOver_1x64 (&mmxSrc,
3180
&mmxAlpha,
3181
&mmxMask,
3182
&mmxDest));
3183
}
3184
3185
w--;
3186
dst++;
3187
}
3188
}
3189
3190
_mm_empty();
3191
}
3192
3193
/* -------------------------------------------------------------------------------------------------
3194
* fbCompositeSolidMask_nx8x8888
3195
*/
3196
3197
pixman_bool_t
3198
pixmanFillsse2 (uint32_t *bits,
3199
int stride,
3200
int bpp,
3201
int x,
3202
int y,
3203
int width,
3204
int height,
3205
uint32_t data)
3206
{
3207
uint32_t byte_width;
3208
uint8_t *byte_line;
3209
3210
__m128i xmmDef;
3211
3212
if (bpp == 16 && (data >> 16 != (data & 0xffff)))
3213
return FALSE;
3214
3215
if (bpp != 16 && bpp != 32)
3216
return FALSE;
3217
3218
if (bpp == 16)
3219
{
3220
stride = stride * (int) sizeof (uint32_t) / 2;
3221
byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
3222
byte_width = 2 * width;
3223
stride *= 2;
3224
}
3225
else
3226
{
3227
stride = stride * (int) sizeof (uint32_t) / 4;
3228
byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
3229
byte_width = 4 * width;
3230
stride *= 4;
3231
}
3232
3233
cachePrefetch ((__m128i*)byte_line);
3234
xmmDef = createMask_2x32_128 (data, data);
3235
3236
while (height--)
3237
{
3238
int w;
3239
uint8_t *d = byte_line;
3240
byte_line += stride;
3241
w = byte_width;
3242
3243
3244
cachePrefetchNext ((__m128i*)d);
3245
3246
while (w >= 2 && ((unsigned long)d & 3))
3247
{
3248
*(uint16_t *)d = data;
3249
w -= 2;
3250
d += 2;
3251
}
3252
3253
while (w >= 4 && ((unsigned long)d & 15))
3254
{
3255
*(uint32_t *)d = data;
3256
3257
w -= 4;
3258
d += 4;
3259
}
3260
3261
cachePrefetchNext ((__m128i*)d);
3262
3263
while (w >= 128)
3264
{
3265
cachePrefetch (((__m128i*)d) + 12);
3266
3267
save128Aligned ((__m128i*)(d), xmmDef);
3268
save128Aligned ((__m128i*)(d+16), xmmDef);
3269
save128Aligned ((__m128i*)(d+32), xmmDef);
3270
save128Aligned ((__m128i*)(d+48), xmmDef);
3271
save128Aligned ((__m128i*)(d+64), xmmDef);
3272
save128Aligned ((__m128i*)(d+80), xmmDef);
3273
save128Aligned ((__m128i*)(d+96), xmmDef);
3274
save128Aligned ((__m128i*)(d+112), xmmDef);
3275
3276
d += 128;
3277
w -= 128;
3278
}
3279
3280
if (w >= 64)
3281
{
3282
cachePrefetch (((__m128i*)d) + 8);
3283
3284
save128Aligned ((__m128i*)(d), xmmDef);
3285
save128Aligned ((__m128i*)(d+16), xmmDef);
3286
save128Aligned ((__m128i*)(d+32), xmmDef);
3287
save128Aligned ((__m128i*)(d+48), xmmDef);
3288
3289
d += 64;
3290
w -= 64;
3291
}
3292
3293
cachePrefetchNext ((__m128i*)d);
3294
3295
if (w >= 32)
3296
{
3297
save128Aligned ((__m128i*)(d), xmmDef);
3298
save128Aligned ((__m128i*)(d+16), xmmDef);
3299
3300
d += 32;
3301
w -= 32;
3302
}
3303
3304
if (w >= 16)
3305
{
3306
save128Aligned ((__m128i*)(d), xmmDef);
3307
3308
d += 16;
3309
w -= 16;
3310
}
3311
3312
cachePrefetchNext ((__m128i*)d);
3313
3314
while (w >= 4)
3315
{
3316
*(uint32_t *)d = data;
3317
3318
w -= 4;
3319
d += 4;
3320
}
3321
3322
if (w >= 2)
3323
{
3324
*(uint16_t *)d = data;
3325
w -= 2;
3326
d += 2;
3327
}
3328
}
3329
3330
_mm_empty();
3331
return TRUE;
3332
}
3333
3334
void
3335
fbCompositeSolidMaskSrc_nx8x8888sse2 (pixman_op_t op,
3336
pixman_image_t * pSrc,
3337
pixman_image_t * pMask,
3338
pixman_image_t * pDst,
3339
int16_t xSrc,
3340
int16_t ySrc,
3341
int16_t xMask,
3342
int16_t yMask,
3343
int16_t xDst,
3344
int16_t yDst,
3345
uint16_t width,
3346
uint16_t height)
3347
{
3348
uint32_t src, srca;
3349
uint32_t *dstLine, *dst;
3350
uint8_t *maskLine, *mask;
3351
int dstStride, maskStride;
3352
uint16_t w;
3353
uint32_t m;
3354
3355
__m128i xmmSrc, xmmDef;
3356
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
3357
3358
fbComposeGetSolid(pSrc, src, pDst->bits.format);
3359
3360
srca = src >> 24;
3361
if (srca == 0)
3362
{
3363
pixmanFillsse2 (pDst->bits.bits, pDst->bits.rowstride,
3364
PIXMAN_FORMAT_BPP (pDst->bits.format),
3365
xDst, yDst, width, height, 0);
3366
return;
3367
}
3368
3369
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
3370
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
3371
3372
xmmDef = createMask_2x32_128 (src, src);
3373
xmmSrc = expandPixel_32_1x128 (src);
3374
3375
while (height--)
3376
{
3377
dst = dstLine;
3378
dstLine += dstStride;
3379
mask = maskLine;
3380
maskLine += maskStride;
3381
w = width;
3382
3383
/* call prefetch hint to optimize cache load*/
3384
cachePrefetch ((__m128i*)mask);
3385
cachePrefetch ((__m128i*)dst);
3386
3387
while (w && (unsigned long)dst & 15)
3388
{
3389
uint8_t m = *mask++;
3390
3391
if (m)
3392
{
3393
*dst = pack_1x64_32 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmSrc), expandPixel_8_1x64 (m)));
3394
}
3395
else
3396
{
3397
*dst = 0;
3398
}
3399
3400
w--;
3401
dst++;
3402
}
3403
3404
/* call prefetch hint to optimize cache load*/
3405
cachePrefetch ((__m128i*)mask);
3406
cachePrefetch ((__m128i*)dst);
3407
3408
while (w >= 4)
3409
{
3410
/* fill cache line with next memory */
3411
cachePrefetchNext ((__m128i*)mask);
3412
cachePrefetchNext ((__m128i*)dst);
3413
3414
m = *((uint32_t*)mask);
3415
3416
if (srca == 0xff && m == 0xffffffff)
3417
{
3418
save128Aligned ((__m128i*)dst, xmmDef);
3419
}
3420
else if (m)
3421
{
3422
xmmMask = unpack_32_1x128 (m);
3423
xmmMask = _mm_unpacklo_epi8 (xmmMask, _mm_setzero_si128());
3424
3425
/* Unpacking */
3426
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3427
3428
expandAlphaRev_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
3429
3430
pixMultiply_2x128 (&xmmSrc, &xmmSrc, &xmmMaskLo, &xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
3431
3432
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmMaskLo, xmmMaskHi));
3433
}
3434
else
3435
{
3436
save128Aligned ((__m128i*)dst, _mm_setzero_si128());
3437
}
3438
3439
w -= 4;
3440
dst += 4;
3441
mask += 4;
3442
}
3443
3444
while (w)
3445
{
3446
uint8_t m = *mask++;
3447
3448
if (m)
3449
{
3450
*dst = pack_1x64_32 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmSrc), expandPixel_8_1x64 (m)));
3451
}
3452
else
3453
{
3454
*dst = 0;
3455
}
3456
3457
w--;
3458
dst++;
3459
}
3460
}
3461
3462
_mm_empty();
3463
}
3464
3465
/* -------------------------------------------------------------------------------------------------
3466
* fbCompositeSolidMask_nx8x0565
3467
*/
3468
3469
void
3470
fbCompositeSolidMask_nx8x0565sse2 (pixman_op_t op,
3471
pixman_image_t * pSrc,
3472
pixman_image_t * pMask,
3473
pixman_image_t * pDst,
3474
int16_t xSrc,
3475
int16_t ySrc,
3476
int16_t xMask,
3477
int16_t yMask,
3478
int16_t xDst,
3479
int16_t yDst,
3480
uint16_t width,
3481
uint16_t height)
3482
{
3483
uint32_t src, srca;
3484
uint16_t *dstLine, *dst, d;
3485
uint8_t *maskLine, *mask;
3486
int dstStride, maskStride;
3487
uint16_t w;
3488
uint32_t m;
3489
__m64 mmxSrc, mmxAlpha, mmxMask, mmxDest;
3490
3491
__m128i xmmSrc, xmmAlpha;
3492
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
3493
__m128i xmmDst, xmmDst0, xmmDst1, xmmDst2, xmmDst3;
3494
3495
fbComposeGetSolid(pSrc, src, pDst->bits.format);
3496
3497
srca = src >> 24;
3498
if (srca == 0)
3499
return;
3500
3501
fbComposeGetStart (pDst, xDst, yDst, uint16_t, dstStride, dstLine, 1);
3502
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
3503
3504
xmmSrc = expandPixel_32_1x128 (src);
3505
xmmAlpha = expandAlpha_1x128 (xmmSrc);
3506
mmxSrc = _mm_movepi64_pi64 (xmmSrc);
3507
mmxAlpha = _mm_movepi64_pi64 (xmmAlpha);
3508
3509
while (height--)
3510
{
3511
dst = dstLine;
3512
dstLine += dstStride;
3513
mask = maskLine;
3514
maskLine += maskStride;
3515
w = width;
3516
3517
/* call prefetch hint to optimize cache load*/
3518
cachePrefetch ((__m128i*)mask);
3519
cachePrefetch ((__m128i*)dst);
3520
3521
while (w && (unsigned long)dst & 15)
3522
{
3523
m = *mask++;
3524
3525
if (m)
3526
{
3527
d = *dst;
3528
mmxMask = expandAlphaRev_1x64 (unpack_32_1x64 (m));
3529
mmxDest = expand565_16_1x64 (d);
3530
3531
*dst = pack565_32_16 (pack_1x64_32 (inOver_1x64 (&mmxSrc,
3532
&mmxAlpha,
3533
&mmxMask,
3534
&mmxDest)));
3535
}
3536
3537
w--;
3538
dst++;
3539
}
3540
3541
/* call prefetch hint to optimize cache load*/
3542
cachePrefetch ((__m128i*)mask);
3543
cachePrefetch ((__m128i*)dst);
3544
3545
while (w >= 8)
3546
{
3547
/* fill cache line with next memory */
3548
cachePrefetchNext ((__m128i*)mask);
3549
cachePrefetchNext ((__m128i*)dst);
3550
3551
xmmDst = load128Aligned ((__m128i*) dst);
3552
unpack565_128_4x128 (xmmDst, &xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
3553
3554
m = *((uint32_t*)mask);
3555
mask += 4;
3556
3557
if (m)
3558
{
3559
xmmMask = unpack_32_1x128 (m);
3560
xmmMask = _mm_unpacklo_epi8 (xmmMask, _mm_setzero_si128());
3561
3562
/* Unpacking */
3563
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3564
3565
expandAlphaRev_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
3566
inOver_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDst0, &xmmDst1);
3567
}
3568
3569
m = *((uint32_t*)mask);
3570
mask += 4;
3571
3572
if (m)
3573
{
3574
xmmMask = unpack_32_1x128 (m);
3575
xmmMask = _mm_unpacklo_epi8 (xmmMask, _mm_setzero_si128());
3576
3577
/* Unpacking */
3578
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3579
3580
expandAlphaRev_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
3581
inOver_2x128 (&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDst2, &xmmDst3);
3582
}
3583
3584
save128Aligned ((__m128i*)dst, pack565_4x128_128 (&xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3));
3585
3586
w -= 8;
3587
dst += 8;
3588
}
3589
3590
while (w)
3591
{
3592
m = *mask++;
3593
3594
if (m)
3595
{
3596
d = *dst;
3597
mmxMask = expandAlphaRev_1x64 (unpack_32_1x64 (m));
3598
mmxDest = expand565_16_1x64 (d);
3599
3600
*dst = pack565_32_16 (pack_1x64_32 (inOver_1x64 (&mmxSrc,
3601
&mmxAlpha,
3602
&mmxMask,
3603
&mmxDest)));
3604
}
3605
3606
w--;
3607
dst++;
3608
}
3609
}
3610
3611
_mm_empty();
3612
}
3613
3614
/* -------------------------------------------------------------------------------------------------
3615
* fbCompositeSrc_8888RevNPx0565
3616
*/
3617
3618
void
3619
fbCompositeSrc_8888RevNPx0565sse2 (pixman_op_t op,
3620
pixman_image_t * pSrc,
3621
pixman_image_t * pMask,
3622
pixman_image_t * pDst,
3623
int16_t xSrc,
3624
int16_t ySrc,
3625
int16_t xMask,
3626
int16_t yMask,
3627
int16_t xDst,
3628
int16_t yDst,
3629
uint16_t width,
3630
uint16_t height)
3631
{
3632
uint16_t *dstLine, *dst, d;
3633
uint32_t *srcLine, *src, s;
3634
int dstStride, srcStride;
3635
uint16_t w;
3636
uint32_t packCmp;
3637
3638
__m64 ms;
3639
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
3640
__m128i xmmDst, xmmDst0, xmmDst1, xmmDst2, xmmDst3;
3641
3642
fbComposeGetStart (pDst, xDst, yDst, uint16_t, dstStride, dstLine, 1);
3643
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
3644
3645
#if 0
3646
/* FIXME
3647
*
3648
* I copy the code from MMX one and keep the fixme.
3649
* If it's a problem there, probably is a problem here.
3650
*/
3651
assert (pSrc->pDrawable == pMask->pDrawable);
3652
#endif
3653
3654
while (height--)
3655
{
3656
dst = dstLine;
3657
dstLine += dstStride;
3658
src = srcLine;
3659
srcLine += srcStride;
3660
w = width;
3661
3662
/* call prefetch hint to optimize cache load*/
3663
cachePrefetch ((__m128i*)src);
3664
cachePrefetch ((__m128i*)dst);
3665
3666
while (w && (unsigned long)dst & 15)
3667
{
3668
s = *src++;
3669
d = *dst;
3670
3671
ms = unpack_32_1x64 (s);
3672
3673
*dst++ = pack565_32_16 (pack_1x64_32 (overRevNonPre_1x64(ms, expand565_16_1x64 (d))));
3674
w--;
3675
}
3676
3677
/* call prefetch hint to optimize cache load*/
3678
cachePrefetch ((__m128i*)src);
3679
cachePrefetch ((__m128i*)dst);
3680
3681
while (w >= 8)
3682
{
3683
/* fill cache line with next memory */
3684
cachePrefetchNext ((__m128i*)src);
3685
cachePrefetchNext ((__m128i*)dst);
3686
3687
/* First round */
3688
xmmSrc = load128Unaligned((__m128i*)src);
3689
xmmDst = load128Aligned ((__m128i*)dst);
3690
3691
packCmp = packAlpha (xmmSrc);
3692
3693
unpack565_128_4x128 (xmmDst, &xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
3694
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
3695
3696
/* preload next round*/
3697
xmmSrc = load128Unaligned((__m128i*)(src+4));
3698
/* preload next round*/
3699
3700
if (packCmp == 0xffffffff)
3701
{
3702
invertColors_2x128 (xmmSrcLo, xmmSrcHi, &xmmDst0, &xmmDst1);
3703
}
3704
else if (packCmp)
3705
{
3706
overRevNonPre_2x128 (xmmSrcLo, xmmSrcHi, &xmmDst0, &xmmDst1);
3707
}
3708
3709
/* Second round */
3710
packCmp = packAlpha (xmmSrc);
3711
3712
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
3713
3714
if (packCmp == 0xffffffff)
3715
{
3716
invertColors_2x128 (xmmSrcLo, xmmSrcHi, &xmmDst2, &xmmDst3);
3717
}
3718
else if (packCmp)
3719
{
3720
overRevNonPre_2x128 (xmmSrcLo, xmmSrcHi, &xmmDst2, &xmmDst3);
3721
}
3722
3723
save128Aligned ((__m128i*)dst, pack565_4x128_128 (&xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3));
3724
3725
w -= 8;
3726
src += 8;
3727
dst += 8;
3728
}
3729
3730
while (w)
3731
{
3732
s = *src++;
3733
d = *dst;
3734
3735
ms = unpack_32_1x64 (s);
3736
3737
*dst++ = pack565_32_16 (pack_1x64_32 (overRevNonPre_1x64(ms, expand565_16_1x64 (d))));
3738
w--;
3739
}
3740
}
3741
3742
_mm_empty();
3743
}
3744
3745
/* "8888RevNP" is GdkPixbuf's format: ABGR, non premultiplied */
3746
3747
/* -------------------------------------------------------------------------------------------------
3748
* fbCompositeSrc_8888RevNPx8888
3749
*/
3750
3751
void
3752
fbCompositeSrc_8888RevNPx8888sse2 (pixman_op_t op,
3753
pixman_image_t * pSrc,
3754
pixman_image_t * pMask,
3755
pixman_image_t * pDst,
3756
int16_t xSrc,
3757
int16_t ySrc,
3758
int16_t xMask,
3759
int16_t yMask,
3760
int16_t xDst,
3761
int16_t yDst,
3762
uint16_t width,
3763
uint16_t height)
3764
{
3765
uint32_t *dstLine, *dst, d;
3766
uint32_t *srcLine, *src, s;
3767
int dstStride, srcStride;
3768
uint16_t w;
3769
uint32_t packCmp;
3770
3771
__m128i xmmSrcLo, xmmSrcHi;
3772
__m128i xmmDstLo, xmmDstHi;
3773
3774
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
3775
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
3776
3777
#if 0
3778
/* FIXME
3779
*
3780
* I copy the code from MMX one and keep the fixme.
3781
* If it's a problem there, probably is a problem here.
3782
*/
3783
assert (pSrc->pDrawable == pMask->pDrawable);
3784
#endif
3785
3786
while (height--)
3787
{
3788
dst = dstLine;
3789
dstLine += dstStride;
3790
src = srcLine;
3791
srcLine += srcStride;
3792
w = width;
3793
3794
/* call prefetch hint to optimize cache load*/
3795
cachePrefetch ((__m128i*)src);
3796
cachePrefetch ((__m128i*)dst);
3797
3798
while (w && (unsigned long)dst & 15)
3799
{
3800
s = *src++;
3801
d = *dst;
3802
3803
*dst++ = pack_1x64_32 (overRevNonPre_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
3804
3805
w--;
3806
}
3807
3808
/* call prefetch hint to optimize cache load*/
3809
cachePrefetch ((__m128i*)src);
3810
cachePrefetch ((__m128i*)dst);
3811
3812
while (w >= 4)
3813
{
3814
/* fill cache line with next memory */
3815
cachePrefetchNext ((__m128i*)src);
3816
cachePrefetchNext ((__m128i*)dst);
3817
3818
xmmSrcHi = load128Unaligned((__m128i*)src);
3819
3820
packCmp = packAlpha (xmmSrcHi);
3821
3822
unpack_128_2x128 (xmmSrcHi, &xmmSrcLo, &xmmSrcHi);
3823
3824
if (packCmp == 0xffffffff)
3825
{
3826
invertColors_2x128( xmmSrcLo, xmmSrcHi, &xmmDstLo, &xmmDstHi);
3827
3828
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
3829
}
3830
else if (packCmp)
3831
{
3832
xmmDstHi = load128Aligned ((__m128i*)dst);
3833
3834
unpack_128_2x128 (xmmDstHi, &xmmDstLo, &xmmDstHi);
3835
3836
overRevNonPre_2x128 (xmmSrcLo, xmmSrcHi, &xmmDstLo, &xmmDstHi);
3837
3838
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
3839
}
3840
3841
w -= 4;
3842
dst += 4;
3843
src += 4;
3844
}
3845
3846
while (w)
3847
{
3848
s = *src++;
3849
d = *dst;
3850
3851
*dst++ = pack_1x64_32 (overRevNonPre_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
3852
3853
w--;
3854
}
3855
}
3856
3857
_mm_empty();
3858
}
3859
3860
/* -------------------------------------------------------------------------------------------------
3861
* fbCompositeSolidMask_nx8888x0565C
3862
*/
3863
3864
void
3865
fbCompositeSolidMask_nx8888x0565Csse2 (pixman_op_t op,
3866
pixman_image_t * pSrc,
3867
pixman_image_t * pMask,
3868
pixman_image_t * pDst,
3869
int16_t xSrc,
3870
int16_t ySrc,
3871
int16_t xMask,
3872
int16_t yMask,
3873
int16_t xDst,
3874
int16_t yDst,
3875
uint16_t width,
3876
uint16_t height)
3877
{
3878
uint32_t src, srca;
3879
uint16_t *dstLine, *dst, d;
3880
uint32_t *maskLine, *mask, m;
3881
int dstStride, maskStride;
3882
int w;
3883
uint32_t packCmp;
3884
3885
__m128i xmmSrc, xmmAlpha;
3886
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
3887
__m128i xmmDst, xmmDst0, xmmDst1, xmmDst2, xmmDst3;
3888
3889
__m64 mmxSrc, mmxAlpha, mmxMask, mmxDest;
3890
3891
fbComposeGetSolid(pSrc, src, pDst->bits.format);
3892
3893
srca = src >> 24;
3894
if (srca == 0)
3895
return;
3896
3897
fbComposeGetStart (pDst, xDst, yDst, uint16_t, dstStride, dstLine, 1);
3898
fbComposeGetStart (pMask, xMask, yMask, uint32_t, maskStride, maskLine, 1);
3899
3900
xmmSrc = expandPixel_32_1x128 (src);
3901
xmmAlpha = expandAlpha_1x128 (xmmSrc);
3902
mmxSrc = _mm_movepi64_pi64 (xmmSrc);
3903
mmxAlpha = _mm_movepi64_pi64 (xmmAlpha);
3904
3905
while (height--)
3906
{
3907
w = width;
3908
mask = maskLine;
3909
dst = dstLine;
3910
maskLine += maskStride;
3911
dstLine += dstStride;
3912
3913
/* call prefetch hint to optimize cache load*/
3914
cachePrefetch ((__m128i*)mask);
3915
cachePrefetch ((__m128i*)dst);
3916
3917
while (w && ((unsigned long)dst & 15))
3918
{
3919
m = *(uint32_t *) mask;
3920
3921
if (m)
3922
{
3923
d = *dst;
3924
mmxMask = unpack_32_1x64 (m);
3925
mmxDest = expand565_16_1x64 (d);
3926
3927
*dst = pack565_32_16 (pack_1x64_32 (inOver_1x64 (&mmxSrc,
3928
&mmxAlpha,
3929
&mmxMask,
3930
&mmxDest)));
3931
}
3932
3933
w--;
3934
dst++;
3935
mask++;
3936
}
3937
3938
/* call prefetch hint to optimize cache load*/
3939
cachePrefetch ((__m128i*)mask);
3940
cachePrefetch ((__m128i*)dst);
3941
3942
while (w >= 8)
3943
{
3944
/* fill cache line with next memory */
3945
cachePrefetchNext ((__m128i*)mask);
3946
cachePrefetchNext ((__m128i*)dst);
3947
3948
/* First round */
3949
xmmMask = load128Unaligned((__m128i*)mask);
3950
xmmDst = load128Aligned((__m128i*)dst);
3951
3952
packCmp = _mm_movemask_epi8 (_mm_cmpeq_epi32 (xmmMask, _mm_setzero_si128()));
3953
3954
unpack565_128_4x128 (xmmDst, &xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3);
3955
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3956
3957
/* preload next round*/
3958
xmmMask = load128Unaligned((__m128i*)(mask+4));
3959
/* preload next round*/
3960
3961
if (packCmp != 0xffff)
3962
{
3963
inOver_2x128(&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDst0, &xmmDst1);
3964
}
3965
3966
/* Second round */
3967
packCmp = _mm_movemask_epi8 (_mm_cmpeq_epi32 (xmmMask, _mm_setzero_si128()));
3968
3969
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
3970
3971
if (packCmp != 0xffff)
3972
{
3973
inOver_2x128(&xmmSrc, &xmmSrc, &xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmDst2, &xmmDst3);
3974
}
3975
3976
save128Aligned ((__m128i*)dst, pack565_4x128_128 (&xmmDst0, &xmmDst1, &xmmDst2, &xmmDst3));
3977
3978
w -= 8;
3979
dst += 8;
3980
mask += 8;
3981
}
3982
3983
while (w)
3984
{
3985
m = *(uint32_t *) mask;
3986
3987
if (m)
3988
{
3989
d = *dst;
3990
mmxMask = unpack_32_1x64 (m);
3991
mmxDest = expand565_16_1x64 (d);
3992
3993
*dst = pack565_32_16 (pack_1x64_32 (inOver_1x64 (&mmxSrc,
3994
&mmxAlpha,
3995
&mmxMask,
3996
&mmxDest)));
3997
}
3998
3999
w--;
4000
dst++;
4001
mask++;
4002
}
4003
}
4004
4005
_mm_empty ();
4006
}
4007
4008
/* -------------------------------------------------------------------------------------------------
4009
* fbCompositeIn_nx8x8
4010
*/
4011
4012
void
4013
fbCompositeIn_nx8x8sse2 (pixman_op_t op,
4014
pixman_image_t * pSrc,
4015
pixman_image_t * pMask,
4016
pixman_image_t * pDst,
4017
int16_t xSrc,
4018
int16_t ySrc,
4019
int16_t xMask,
4020
int16_t yMask,
4021
int16_t xDst,
4022
int16_t yDst,
4023
uint16_t width,
4024
uint16_t height)
4025
{
4026
uint8_t *dstLine, *dst;
4027
uint8_t *maskLine, *mask;
4028
int dstStride, maskStride;
4029
uint16_t w, d, m;
4030
uint32_t src;
4031
uint8_t sa;
4032
4033
__m128i xmmAlpha;
4034
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
4035
__m128i xmmDst, xmmDstLo, xmmDstHi;
4036
4037
fbComposeGetStart (pDst, xDst, yDst, uint8_t, dstStride, dstLine, 1);
4038
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
4039
4040
fbComposeGetSolid(pSrc, src, pDst->bits.format);
4041
4042
sa = src >> 24;
4043
if (sa == 0)
4044
return;
4045
4046
xmmAlpha = expandAlpha_1x128 (expandPixel_32_1x128 (src));
4047
4048
while (height--)
4049
{
4050
dst = dstLine;
4051
dstLine += dstStride;
4052
mask = maskLine;
4053
maskLine += maskStride;
4054
w = width;
4055
4056
/* call prefetch hint to optimize cache load*/
4057
cachePrefetch ((__m128i*)mask);
4058
cachePrefetch ((__m128i*)dst);
4059
4060
while (w && ((unsigned long)dst & 15))
4061
{
4062
m = (uint32_t) *mask++;
4063
d = (uint32_t) *dst;
4064
4065
*dst++ = (uint8_t) pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmAlpha), unpack_32_1x64 (m)),
4066
unpack_32_1x64 (d)));
4067
w--;
4068
}
4069
4070
/* call prefetch hint to optimize cache load*/
4071
cachePrefetch ((__m128i*)mask);
4072
cachePrefetch ((__m128i*)dst);
4073
4074
while (w >= 16)
4075
{
4076
/* fill cache line with next memory */
4077
cachePrefetchNext ((__m128i*)mask);
4078
cachePrefetchNext ((__m128i*)dst);
4079
4080
xmmMask = load128Unaligned((__m128i*)mask);
4081
xmmDst = load128Aligned((__m128i*)dst);
4082
4083
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
4084
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
4085
4086
pixMultiply_2x128 (&xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
4087
pixMultiply_2x128 (&xmmMaskLo, &xmmMaskHi, &xmmDstLo, &xmmDstHi, &xmmDstLo, &xmmDstHi);
4088
4089
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
4090
4091
mask += 16;
4092
dst += 16;
4093
w -= 16;
4094
}
4095
4096
while (w)
4097
{
4098
m = (uint32_t) *mask++;
4099
d = (uint32_t) *dst;
4100
4101
*dst++ = (uint8_t) pack_1x64_32 (pixMultiply_1x64 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmAlpha), unpack_32_1x64 (m)),
4102
unpack_32_1x64 (d)));
4103
w--;
4104
}
4105
}
4106
4107
_mm_empty();
4108
}
4109
4110
/* -------------------------------------------------------------------------------------------------
4111
* fbCompositeIn_8x8
4112
*/
4113
4114
void
4115
fbCompositeIn_8x8sse2 (pixman_op_t op,
4116
pixman_image_t * pSrc,
4117
pixman_image_t * pMask,
4118
pixman_image_t * pDst,
4119
int16_t xSrc,
4120
int16_t ySrc,
4121
int16_t xMask,
4122
int16_t yMask,
4123
int16_t xDst,
4124
int16_t yDst,
4125
uint16_t width,
4126
uint16_t height)
4127
{
4128
uint8_t *dstLine, *dst;
4129
uint8_t *srcLine, *src;
4130
int srcStride, dstStride;
4131
uint16_t w;
4132
uint32_t s, d;
4133
4134
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
4135
__m128i xmmDst, xmmDstLo, xmmDstHi;
4136
4137
fbComposeGetStart (pDst, xDst, yDst, uint8_t, dstStride, dstLine, 1);
4138
fbComposeGetStart (pSrc, xSrc, ySrc, uint8_t, srcStride, srcLine, 1);
4139
4140
while (height--)
4141
{
4142
dst = dstLine;
4143
dstLine += dstStride;
4144
src = srcLine;
4145
srcLine += srcStride;
4146
w = width;
4147
4148
/* call prefetch hint to optimize cache load*/
4149
cachePrefetch ((__m128i*)src);
4150
cachePrefetch ((__m128i*)dst);
4151
4152
while (w && ((unsigned long)dst & 15))
4153
{
4154
s = (uint32_t) *src++;
4155
d = (uint32_t) *dst;
4156
4157
*dst++ = (uint8_t) pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s),unpack_32_1x64 (d)));
4158
w--;
4159
}
4160
4161
/* call prefetch hint to optimize cache load*/
4162
cachePrefetch ((__m128i*)src);
4163
cachePrefetch ((__m128i*)dst);
4164
4165
while (w >= 16)
4166
{
4167
/* fill cache line with next memory */
4168
cachePrefetchNext ((__m128i*)src);
4169
cachePrefetchNext ((__m128i*)dst);
4170
4171
xmmSrc = load128Unaligned((__m128i*)src);
4172
xmmDst = load128Aligned((__m128i*)dst);
4173
4174
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
4175
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
4176
4177
pixMultiply_2x128 (&xmmSrcLo, &xmmSrcHi, &xmmDstLo, &xmmDstHi, &xmmDstLo, &xmmDstHi);
4178
4179
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
4180
4181
src += 16;
4182
dst += 16;
4183
w -= 16;
4184
}
4185
4186
while (w)
4187
{
4188
s = (uint32_t) *src++;
4189
d = (uint32_t) *dst;
4190
4191
*dst++ = (uint8_t) pack_1x64_32 (pixMultiply_1x64 (unpack_32_1x64 (s),unpack_32_1x64 (d)));
4192
w--;
4193
}
4194
}
4195
4196
_mm_empty ();
4197
}
4198
4199
/* -------------------------------------------------------------------------------------------------
4200
* fbCompositeSrcAdd_8888x8x8
4201
*/
4202
4203
void
4204
fbCompositeSrcAdd_8888x8x8sse2 (pixman_op_t op,
4205
pixman_image_t * pSrc,
4206
pixman_image_t * pMask,
4207
pixman_image_t * pDst,
4208
int16_t xSrc,
4209
int16_t ySrc,
4210
int16_t xMask,
4211
int16_t yMask,
4212
int16_t xDst,
4213
int16_t yDst,
4214
uint16_t width,
4215
uint16_t height)
4216
{
4217
uint8_t *dstLine, *dst;
4218
uint8_t *maskLine, *mask;
4219
int dstStride, maskStride;
4220
uint16_t w;
4221
uint32_t src;
4222
uint8_t sa;
4223
uint32_t m, d;
4224
4225
__m128i xmmAlpha;
4226
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
4227
__m128i xmmDst, xmmDstLo, xmmDstHi;
4228
4229
fbComposeGetStart (pDst, xDst, yDst, uint8_t, dstStride, dstLine, 1);
4230
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
4231
4232
fbComposeGetSolid(pSrc, src, pDst->bits.format);
4233
4234
sa = src >> 24;
4235
if (sa == 0)
4236
return;
4237
4238
xmmAlpha = expandAlpha_1x128 (expandPixel_32_1x128 (src));
4239
4240
while (height--)
4241
{
4242
dst = dstLine;
4243
dstLine += dstStride;
4244
mask = maskLine;
4245
maskLine += maskStride;
4246
w = width;
4247
4248
/* call prefetch hint to optimize cache load*/
4249
cachePrefetch ((__m128i*)mask);
4250
cachePrefetch ((__m128i*)dst);
4251
4252
while (w && ((unsigned long)dst & 15))
4253
{
4254
m = (uint32_t) *mask++;
4255
d = (uint32_t) *dst;
4256
4257
*dst++ = (uint8_t) pack_1x64_32 (_mm_adds_pu16 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmAlpha), unpack_32_1x64 (m)),
4258
unpack_32_1x64 (d)));
4259
w--;
4260
}
4261
4262
/* call prefetch hint to optimize cache load*/
4263
cachePrefetch ((__m128i*)mask);
4264
cachePrefetch ((__m128i*)dst);
4265
4266
while (w >= 16)
4267
{
4268
/* fill cache line with next memory */
4269
cachePrefetchNext ((__m128i*)mask);
4270
cachePrefetchNext ((__m128i*)dst);
4271
4272
xmmMask = load128Unaligned((__m128i*)mask);
4273
xmmDst = load128Aligned((__m128i*)dst);
4274
4275
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
4276
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
4277
4278
pixMultiply_2x128 (&xmmAlpha, &xmmAlpha, &xmmMaskLo, &xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
4279
4280
xmmDstLo = _mm_adds_epu16 (xmmMaskLo, xmmDstLo);
4281
xmmDstHi = _mm_adds_epu16 (xmmMaskHi, xmmDstHi);
4282
4283
save128Aligned ((__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
4284
4285
mask += 16;
4286
dst += 16;
4287
w -= 16;
4288
}
4289
4290
while (w)
4291
{
4292
m = (uint32_t) *mask++;
4293
d = (uint32_t) *dst;
4294
4295
*dst++ = (uint8_t) pack_1x64_32 (_mm_adds_pu16 (pixMultiply_1x64 (_mm_movepi64_pi64 (xmmAlpha), unpack_32_1x64 (m)),
4296
unpack_32_1x64 (d)));
4297
w--;
4298
}
4299
}
4300
4301
_mm_empty();
4302
}
4303
4304
/* -------------------------------------------------------------------------------------------------
4305
* fbCompositeSrcAdd_8000x8000
4306
*/
4307
4308
void
4309
fbCompositeSrcAdd_8000x8000sse2 (pixman_op_t op,
4310
pixman_image_t * pSrc,
4311
pixman_image_t * pMask,
4312
pixman_image_t * pDst,
4313
int16_t xSrc,
4314
int16_t ySrc,
4315
int16_t xMask,
4316
int16_t yMask,
4317
int16_t xDst,
4318
int16_t yDst,
4319
uint16_t width,
4320
uint16_t height)
4321
{
4322
uint8_t *dstLine, *dst;
4323
uint8_t *srcLine, *src;
4324
int dstStride, srcStride;
4325
uint16_t w;
4326
uint16_t t;
4327
4328
fbComposeGetStart (pSrc, xSrc, ySrc, uint8_t, srcStride, srcLine, 1);
4329
fbComposeGetStart (pDst, xDst, yDst, uint8_t, dstStride, dstLine, 1);
4330
4331
while (height--)
4332
{
4333
dst = dstLine;
4334
src = srcLine;
4335
4336
/* call prefetch hint to optimize cache load*/
4337
cachePrefetch ((__m128i*)src);
4338
cachePrefetch ((__m128i*)dst);
4339
4340
dstLine += dstStride;
4341
srcLine += srcStride;
4342
w = width;
4343
4344
/* Small head */
4345
while (w && (unsigned long)dst & 3)
4346
{
4347
t = (*dst) + (*src++);
4348
*dst++ = t | (0 - (t >> 8));
4349
w--;
4350
}
4351
4352
coreCombineAddUsse2 ((uint32_t*)dst, (uint32_t*)src, w >> 2);
4353
4354
/* Small tail */
4355
dst += w & 0xfffc;
4356
src += w & 0xfffc;
4357
4358
w &= 3;
4359
4360
while (w)
4361
{
4362
t = (*dst) + (*src++);
4363
*dst++ = t | (0 - (t >> 8));
4364
w--;
4365
}
4366
}
4367
4368
_mm_empty();
4369
}
4370
4371
/* -------------------------------------------------------------------------------------------------
4372
* fbCompositeSrcAdd_8888x8888
4373
*/
4374
void
4375
fbCompositeSrcAdd_8888x8888sse2 (pixman_op_t op,
4376
pixman_image_t * pSrc,
4377
pixman_image_t * pMask,
4378
pixman_image_t * pDst,
4379
int16_t xSrc,
4380
int16_t ySrc,
4381
int16_t xMask,
4382
int16_t yMask,
4383
int16_t xDst,
4384
int16_t yDst,
4385
uint16_t width,
4386
uint16_t height)
4387
{
4388
uint32_t *dstLine, *dst;
4389
uint32_t *srcLine, *src;
4390
int dstStride, srcStride;
4391
4392
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
4393
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
4394
4395
while (height--)
4396
{
4397
dst = dstLine;
4398
dstLine += dstStride;
4399
src = srcLine;
4400
srcLine += srcStride;
4401
4402
coreCombineAddUsse2 (dst, src, width);
4403
}
4404
4405
_mm_empty();
4406
}
4407
4408
/* -------------------------------------------------------------------------------------------------
4409
* fbCompositeCopyAreasse2
4410
*/
4411
4412
pixman_bool_t
4413
pixmanBltsse2 (uint32_t *src_bits,
4414
uint32_t *dst_bits,
4415
int src_stride,
4416
int dst_stride,
4417
int src_bpp,
4418
int dst_bpp,
4419
int src_x, int src_y,
4420
int dst_x, int dst_y,
4421
int width, int height)
4422
{
4423
uint8_t * src_bytes;
4424
uint8_t * dst_bytes;
4425
int byte_width;
4426
4427
if (src_bpp != dst_bpp)
4428
return FALSE;
4429
4430
if (src_bpp == 16)
4431
{
4432
src_stride = src_stride * (int) sizeof (uint32_t) / 2;
4433
dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
4434
src_bytes = (uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
4435
dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
4436
byte_width = 2 * width;
4437
src_stride *= 2;
4438
dst_stride *= 2;
4439
}
4440
else if (src_bpp == 32)
4441
{
4442
src_stride = src_stride * (int) sizeof (uint32_t) / 4;
4443
dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
4444
src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
4445
dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
4446
byte_width = 4 * width;
4447
src_stride *= 4;
4448
dst_stride *= 4;
4449
}
4450
else
4451
{
4452
return FALSE;
4453
}
4454
4455
cachePrefetch ((__m128i*)src_bytes);
4456
cachePrefetch ((__m128i*)dst_bytes);
4457
4458
while (height--)
4459
{
4460
int w;
4461
uint8_t *s = src_bytes;
4462
uint8_t *d = dst_bytes;
4463
src_bytes += src_stride;
4464
dst_bytes += dst_stride;
4465
w = byte_width;
4466
4467
cachePrefetchNext ((__m128i*)s);
4468
cachePrefetchNext ((__m128i*)d);
4469
4470
while (w >= 2 && ((unsigned long)d & 3))
4471
{
4472
*(uint16_t *)d = *(uint16_t *)s;
4473
w -= 2;
4474
s += 2;
4475
d += 2;
4476
}
4477
4478
while (w >= 4 && ((unsigned long)d & 15))
4479
{
4480
*(uint32_t *)d = *(uint32_t *)s;
4481
4482
w -= 4;
4483
s += 4;
4484
d += 4;
4485
}
4486
4487
cachePrefetchNext ((__m128i*)s);
4488
cachePrefetchNext ((__m128i*)d);
4489
4490
while (w >= 64)
4491
{
4492
__m128i xmm0, xmm1, xmm2, xmm3;
4493
4494
/* 128 bytes ahead */
4495
cachePrefetch (((__m128i*)s) + 8);
4496
cachePrefetch (((__m128i*)d) + 8);
4497
4498
xmm0 = load128Unaligned ((__m128i*)(s));
4499
xmm1 = load128Unaligned ((__m128i*)(s+16));
4500
xmm2 = load128Unaligned ((__m128i*)(s+32));
4501
xmm3 = load128Unaligned ((__m128i*)(s+48));
4502
4503
save128Aligned ((__m128i*)(d), xmm0);
4504
save128Aligned ((__m128i*)(d+16), xmm1);
4505
save128Aligned ((__m128i*)(d+32), xmm2);
4506
save128Aligned ((__m128i*)(d+48), xmm3);
4507
4508
s += 64;
4509
d += 64;
4510
w -= 64;
4511
}
4512
4513
cachePrefetchNext ((__m128i*)s);
4514
cachePrefetchNext ((__m128i*)d);
4515
4516
while (w >= 16)
4517
{
4518
save128Aligned ((__m128i*)d, load128Unaligned ((__m128i*)s) );
4519
4520
w -= 16;
4521
d += 16;
4522
s += 16;
4523
}
4524
4525
cachePrefetchNext ((__m128i*)s);
4526
cachePrefetchNext ((__m128i*)d);
4527
4528
while (w >= 4)
4529
{
4530
*(uint32_t *)d = *(uint32_t *)s;
4531
4532
w -= 4;
4533
s += 4;
4534
d += 4;
4535
}
4536
4537
if (w >= 2)
4538
{
4539
*(uint16_t *)d = *(uint16_t *)s;
4540
w -= 2;
4541
s += 2;
4542
d += 2;
4543
}
4544
}
4545
4546
_mm_empty();
4547
4548
return TRUE;
4549
}
4550
4551
void
4552
fbCompositeCopyAreasse2 (pixman_op_t op,
4553
pixman_image_t * pSrc,
4554
pixman_image_t * pMask,
4555
pixman_image_t * pDst,
4556
int16_t xSrc,
4557
int16_t ySrc,
4558
int16_t xMask,
4559
int16_t yMask,
4560
int16_t xDst,
4561
int16_t yDst,
4562
uint16_t width,
4563
uint16_t height)
4564
{
4565
pixmanBltsse2 (pSrc->bits.bits,
4566
pDst->bits.bits,
4567
pSrc->bits.rowstride,
4568
pDst->bits.rowstride,
4569
PIXMAN_FORMAT_BPP (pSrc->bits.format),
4570
PIXMAN_FORMAT_BPP (pDst->bits.format),
4571
xSrc, ySrc, xDst, yDst, width, height);
4572
}
4573
4574
#if 0
4575
/* This code are buggy in MMX version, now the bug was translated to SSE2 version */
4576
void
4577
fbCompositeOver_x888x8x8888sse2 (pixman_op_t op,
4578
pixman_image_t * pSrc,
4579
pixman_image_t * pMask,
4580
pixman_image_t * pDst,
4581
int16_t xSrc,
4582
int16_t ySrc,
4583
int16_t xMask,
4584
int16_t yMask,
4585
int16_t xDst,
4586
int16_t yDst,
4587
uint16_t width,
4588
uint16_t height)
4589
{
4590
uint32_t *src, *srcLine, s;
4591
uint32_t *dst, *dstLine, d;
4592
uint8_t *mask, *maskLine;
4593
uint32_t m;
4594
int srcStride, maskStride, dstStride;
4595
uint16_t w;
4596
4597
__m128i xmmSrc, xmmSrcLo, xmmSrcHi;
4598
__m128i xmmDst, xmmDstLo, xmmDstHi;
4599
__m128i xmmMask, xmmMaskLo, xmmMaskHi;
4600
4601
fbComposeGetStart (pDst, xDst, yDst, uint32_t, dstStride, dstLine, 1);
4602
fbComposeGetStart (pMask, xMask, yMask, uint8_t, maskStride, maskLine, 1);
4603
fbComposeGetStart (pSrc, xSrc, ySrc, uint32_t, srcStride, srcLine, 1);
4604
4605
while (height--)
4606
{
4607
src = srcLine;
4608
srcLine += srcStride;
4609
dst = dstLine;
4610
dstLine += dstStride;
4611
mask = maskLine;
4612
maskLine += maskStride;
4613
4614
w = width;
4615
4616
/* call prefetch hint to optimize cache load*/
4617
cachePrefetch ((__m128i*)src);
4618
cachePrefetch ((__m128i*)dst);
4619
cachePrefetch ((__m128i*)mask);
4620
4621
while (w && (unsigned long)dst & 15)
4622
{
4623
s = 0xff000000 | *src++;
4624
m = (uint32_t) *mask++;
4625
d = *dst;
4626
4627
__m64 ms = unpack_32_1x64 (s);
4628
4629
if (m != 0xff)
4630
{
4631
ms = inOver_1x64 (ms,
4632
xMask00ff,
4633
expandAlphaRev_1x64 (unpack_32_1x64 (m)),
4634
unpack_32_1x64 (d));
4635
}
4636
4637
*dst++ = pack_1x64_32 (ms);
4638
w--;
4639
}
4640
4641
/* call prefetch hint to optimize cache load*/
4642
cachePrefetch ((__m128i*)src);
4643
cachePrefetch ((__m128i*)dst);
4644
cachePrefetch ((__m128i*)mask);
4645
4646
while (w >= 4)
4647
{
4648
/* fill cache line with next memory */
4649
cachePrefetchNext ((__m128i*)src);
4650
cachePrefetchNext ((__m128i*)dst);
4651
cachePrefetchNext ((__m128i*)mask);
4652
4653
m = *(uint32_t*) mask;
4654
xmmSrc = _mm_or_si128 (load128Unaligned ((__m128i*)src), Maskff000000);
4655
4656
if (m == 0xffffffff)
4657
{
4658
save128Aligned ((__m128i*)dst, xmmSrc);
4659
}
4660
else
4661
{
4662
xmmDst = load128Aligned ((__m128i*)dst);
4663
4664
xmmMask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
4665
4666
unpack_128_2x128 (xmmSrc, &xmmSrcLo, &xmmSrcHi);
4667
unpack_128_2x128 (xmmMask, &xmmMaskLo, &xmmMaskHi);
4668
unpack_128_2x128 (xmmDst, &xmmDstLo, &xmmDstHi);
4669
4670
expandAlphaRev_2x128 (xmmMaskLo, xmmMaskHi, &xmmMaskLo, &xmmMaskHi);
4671
4672
inOver_2x128 (xmmSrcLo, xmmSrcHi, Mask00ff, Mask00ff, xmmMaskLo, xmmMaskHi, &xmmDstLo, &xmmDstHi);
4673
4674
save128Aligned( (__m128i*)dst, pack_2x128_128 (xmmDstLo, xmmDstHi));
4675
}
4676
4677
src += 4;
4678
dst += 4;
4679
mask += 4;
4680
w -= 4;
4681
}
4682
4683
while (w)
4684
{
4685
m = (uint32_t) *mask++;
4686
4687
if (m)
4688
{
4689
s = 0xff000000 | *src;
4690
4691
if (m == 0xff)
4692
{
4693
*dst = s;
4694
}
4695
else
4696
{
4697
d = *dst;
4698
4699
*dst = pack_1x64_32 (inOver_1x64 (unpack_32_1x64 (s),
4700
xMask00ff,
4701
expandAlphaRev_1x64 (unpack_32_1x64 (m)),
4702
unpack_32_1x64 (d)));
4703
}
4704
4705
}
4706
4707
src++;
4708
dst++;
4709
w--;
4710
}
4711
}
4712
4713
_mm_empty();
4714
}
4715
#endif /* #if 0 */
4716
4717
#endif /* USE_SSE2 */
Loggerhead is a web-based interface for Breezy
Version: 2.0.1