1
/* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file
3
* For Intel x86 CPU (Pentium-MMX or later) and GNU C compiler.
5
* See http://www.intel.com/drg/pentiumII/appnotes/916/916.htm
6
* and http://www.intel.com/drg/pentiumII/appnotes/923/923.htm
7
* for Intel's performance analysis of the MMX vs. non-MMX code.
9
* libpng version 1.2.8 - December 3, 2004
10
* For conditions of distribution and use, see copyright notice in png.h
11
* Copyright (c) 1998-2004 Glenn Randers-Pehrson
12
* Copyright (c) 1998, Intel Corporation
14
* Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998.
15
* Interface to libpng contributed by Gilles Vollant, 1999.
16
* GNU C port by Greg Roelofs, 1999-2001.
18
* Lines 2350-4300 converted in place with intel2gas 1.3.1:
20
* intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c
22
* and then cleaned up by hand. See http://hermes.terminal.at/intel2gas/ .
24
* NOTE: A sufficiently recent version of GNU as (or as.exe under DOS/Windows)
25
* is required to assemble the newer MMX instructions such as movq.
28
* ftp://ftp.simtel.net/pub/simtelnet/gnu/djgpp/v2gnu/bnu281b.zip
30
* (or a later version in the same directory). For Linux, check your
31
* distribution's web site(s) or try these links:
33
* http://rufus.w3.org/linux/RPM/binutils.html
34
* http://www.debian.org/Packages/stable/devel/binutils.html
35
* ftp://ftp.slackware.com/pub/linux/slackware/slackware/slakware/d1/
38
* For other platforms, see the main GNU site:
40
* ftp://ftp.gnu.org/pub/gnu/binutils/
42
* Version 2.5.2l.15 is definitely too old...
46
* TEMPORARY PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs)
47
* =====================================
50
* - fixed sign error in post-MMX cleanup code (16- & 32-bit cases)
53
* - additional optimizations (possible or definite):
54
* x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested]
55
* - write MMX code for 48-bit case (pixel_bytes == 6)
56
* - figure out what's up with 24-bit case (pixel_bytes == 3):
57
* why subtract 8 from width_mmx in the pass 4/5 case?
58
* (only width_mmx case) (near line 1606)
59
* x [DONE] replace pixel_bytes within each block with the true
60
* constant value (or are compilers smart enough to do that?)
61
* - rewrite all MMX interlacing code so it's aligned with
62
* the *beginning* of the row buffer, not the end. This
63
* would not only allow one to eliminate half of the memory
64
* writes for odd passes (that is, pass == odd), it may also
65
* eliminate some unaligned-data-access exceptions (assuming
66
* there's a penalty for not aligning 64-bit accesses on
67
* 64-bit boundaries). The only catch is that the "leftover"
68
* pixel(s) at the end of the row would have to be saved,
69
* but there are enough unused MMX registers in every case,
70
* so this is not a problem. A further benefit is that the
71
* post-MMX cleanup code (C code) in at least some of the
72
* cases could be done within the assembler block.
73
* x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing,
74
* inconsistent, and don't match the MMX Programmer's Reference
75
* Manual conventions anyway. They should be changed to
76
* "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that
77
* was lowest in memory (e.g., corresponding to a left pixel)
78
* and b7 is the byte that was highest (e.g., a right pixel).
81
* - Brennan's Guide notwithstanding, gcc under Linux does *not*
82
* want globals prefixed by underscores when referencing them--
83
* i.e., if the variable is const4, then refer to it as const4,
84
* not _const4. This seems to be a djgpp-specific requirement.
85
* Also, such variables apparently *must* be declared outside
86
* of functions; neither static nor automatic variables work if
87
* defined within the scope of a single function, but both
88
* static and truly global (multi-module) variables work fine.
91
* - fixed png_combine_row() non-MMX replication bug (odd passes only?)
92
* - switched from string-concatenation-with-macros to cleaner method of
93
* renaming global variables for djgpp--i.e., always use prefixes in
94
* inlined assembler code (== strings) and conditionally rename the
95
* variables, not the other way around. Hence _const4, _mask8_0, etc.
98
* - fixed mmxsupport()/png_do_read_interlace() first-row bug
99
* This one was severely weird: even though mmxsupport() doesn't touch
100
* ebx (where "row" pointer was stored), it nevertheless managed to zero
101
* the register (even in static/non-fPIC code--see below), which in turn
102
* caused png_do_read_interlace() to return prematurely on the first row of
103
* interlaced images (i.e., without expanding the interlaced pixels).
104
* Inspection of the generated assembly code didn't turn up any clues,
105
* although it did point at a minor optimization (i.e., get rid of
106
* mmx_supported_local variable and just use eax). Possibly the CPUID
107
* instruction is more destructive than it looks? (Not yet checked.)
108
* - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly
109
* listings... Apparently register spillage has to do with ebx, since
110
* it's used to index the global offset table. Commenting it out of the
111
* input-reg lists in png_combine_row() eliminated compiler barfage, so
112
* ifdef'd with __PIC__ macro: if defined, use a global for unmask
115
* - verified CPUID clobberage: 12-char string constant ("GenuineIntel",
116
* "AuthenticAMD", etc.) placed in ebx:ecx:edx. Still need to polish.
119
* - made "diff" variable (now "_dif") global to simplify conversion of
120
* filtering routines (running out of regs, sigh). "diff" is still used
121
* in interlacing routines, however.
122
* - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX
123
* macro determines which is used); original not yet tested.
126
* - when compiling with gcc, be sure to use -fomit-frame-pointer
129
* - fixed a register-name typo in png_do_read_interlace(), default (MMX) case,
130
* pass == 4 or 5, that caused visible corruption of interlaced images
133
* - Various problems were reported with gcc 2.95.2 in the Cygwin environment,
134
* many of the form "forbidden register 0 (ax) was spilled for class AREG."
135
* This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and
136
* Chuck Wilson supplied a patch involving dummy output registers. See
137
* http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624
138
* for the original (anonymous) SourceForge bug report.
141
* - Chuck Wilson passed along these remaining gcc 2.95.2 errors:
142
* pnggccrd.c: In function `png_combine_row':
143
* pnggccrd.c:525: more than 10 operands in `asm'
144
* pnggccrd.c:669: more than 10 operands in `asm'
145
* pnggccrd.c:828: more than 10 operands in `asm'
146
* pnggccrd.c:994: more than 10 operands in `asm'
147
* pnggccrd.c:1177: more than 10 operands in `asm'
148
* They are all the same problem and can be worked around by using the
149
* global _unmask variable unconditionally, not just in the -fPIC case.
150
* Reportedly earlier versions of gcc also have the problem with more than
151
* 10 operands; they just don't report it. Much strangeness ensues, etc.
154
* - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted
155
* MMX routine); began converting png_read_filter_row_mmx_sub()
156
* - to finish remaining sections:
157
* - clean up indentation and comments
158
* - preload local variables
159
* - add output and input regs (order of former determines numerical
161
* - avoid all usage of ebx (including bx, bh, bl) register [20000823]
162
* - remove "$" from addressing of Shift and Mask variables [20000823]
165
* - global union vars causing segfaults in png_read_filter_row_mmx_sub()?
168
* - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with
169
* shared-library (-fPIC) version! Code works just fine as part of static
170
* library. Damn damn damn damn damn, should have tested that sooner.
171
* ebx is getting clobbered again (explicitly this time); need to save it
172
* on stack or rewrite asm code to avoid using it altogether. Blargh!
175
* - first section was trickiest; all remaining sections have ebx -> edx now.
176
* (-fPIC works again.) Also added missing underscores to various Shift*
177
* and *Mask* globals and got rid of leading "$" signs.
180
* - added visual separators to help navigate microscopic printed copies
181
* (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working
182
* on png_read_filter_row_mmx_avg()
185
* - finished png_read_filter_row_mmx_avg(): only Paeth left! (930 lines...)
186
* What the hell, did png_read_filter_row_mmx_paeth(), too. Comments not
187
* cleaned up/shortened in either routine, but functionality is complete
188
* and seems to be working fine.
191
* - ahhh, figured out last(?) bit of gcc/gas asm-fu: if register is listed
192
* as an input reg (with dummy output variables, etc.), then it *cannot*
193
* also appear in the clobber list or gcc 2.95.2 will barf. The solution
194
* is simple enough...
197
* - bug in png_read_filter_row_mmx_avg(): 16-bit grayscale not handled
198
* correctly (but 48-bit RGB just fine)
201
* - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors:
202
* - "_ShiftBpp.use = 24;" should have been "_ShiftBpp.use = 16;"
203
* - "_ShiftRem.use = 40;" should have been "_ShiftRem.use = 48;"
204
* - "psllq _ShiftRem, %%mm2" should have been "psrlq _ShiftRem, %%mm2"
207
* - added new png_init_mmx_flags() function (here only because it needs to
208
* call mmxsupport(), which should probably become global png_mmxsupport());
209
* modified other MMX routines to run conditionally (png_ptr->asm_flags)
212
* - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported,
213
* and made it public; moved png_init_mmx_flags() to png.c as internal func
216
* - removed dependency on png_read_filter_row_c() (C code already duplicated
217
* within MMX version of png_read_filter_row()) so no longer necessary to
218
* compile it into pngrutil.o
221
* - fixed buffer-overrun bug in png_combine_row() C code (non-MMX)
224
* - eliminated incorrect use of width_mmx in pixel_bytes == 8 case
227
* - more tinkering with clobber list at lines 4529 and 5033, to get
228
* it to compile on gcc-3.4.
231
* - test png_do_read_interlace() 64-bit case (pixel_bytes == 8)
232
* - write MMX code for 48-bit case (pixel_bytes == 6)
233
* - figure out what's up with 24-bit case (pixel_bytes == 3):
234
* why subtract 8 from width_mmx in the pass 4/5 case?
235
* (only width_mmx case) (near line 1606)
236
* - rewrite all MMX interlacing code so it's aligned with beginning
237
* of the row buffer, not the end (see 19991007 for details)
238
* x pick one version of mmxsupport() and get rid of the other
239
* - add error messages to any remaining bogus default cases
240
* - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed)
241
* x add support for runtime enable/disable/query of various MMX routines
247
#if defined(PNG_USE_PNGGCCRD)
249
int PNGAPI png_mmx_support(void);
251
#ifdef PNG_USE_LOCAL_ARRAYS
252
static const int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0};
253
static const int FARDATA png_pass_inc[7] = {8, 8, 4, 4, 2, 2, 1};
254
static const int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1};
257
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
258
/* djgpp, Win32, and Cygwin add their own underscores to global variables,
259
* so define them without: */
260
#if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__)
261
# define _mmx_supported mmx_supported
262
# define _const4 const4
263
# define _const6 const6
264
# define _mask8_0 mask8_0
265
# define _mask16_1 mask16_1
266
# define _mask16_0 mask16_0
267
# define _mask24_2 mask24_2
268
# define _mask24_1 mask24_1
269
# define _mask24_0 mask24_0
270
# define _mask32_3 mask32_3
271
# define _mask32_2 mask32_2
272
# define _mask32_1 mask32_1
273
# define _mask32_0 mask32_0
274
# define _mask48_5 mask48_5
275
# define _mask48_4 mask48_4
276
# define _mask48_3 mask48_3
277
# define _mask48_2 mask48_2
278
# define _mask48_1 mask48_1
279
# define _mask48_0 mask48_0
280
# define _LBCarryMask LBCarryMask
281
# define _HBClearMask HBClearMask
282
# define _ActiveMask ActiveMask
283
# define _ActiveMask2 ActiveMask2
284
# define _ActiveMaskEnd ActiveMaskEnd
285
# define _ShiftBpp ShiftBpp
286
# define _ShiftRem ShiftRem
287
#ifdef PNG_THREAD_UNSAFE_OK
288
# define _unmask unmask
289
# define _FullLength FullLength
290
# define _MMXLength MMXLength
292
# define _patemp patemp
293
# define _pbtemp pbtemp
294
# define _pctemp pctemp
299
/* These constants are used in the inlined MMX assembly code.
300
Ignore gcc's "At top level: defined but not used" warnings. */
302
/* GRR 20000706: originally _unmask was needed only when compiling with -fPIC,
303
* since that case uses the %ebx register for indexing the Global Offset Table
304
* and there were no other registers available. But gcc 2.95 and later emit
305
* "more than 10 operands in `asm'" errors when %ebx is used to preload unmask
306
* in the non-PIC case, so we'll just use the global unconditionally now.
308
#ifdef PNG_THREAD_UNSAFE_OK
312
static unsigned long long _mask8_0 = 0x0102040810204080LL;
314
static unsigned long long _mask16_1 = 0x0101020204040808LL;
315
static unsigned long long _mask16_0 = 0x1010202040408080LL;
317
static unsigned long long _mask24_2 = 0x0101010202020404LL;
318
static unsigned long long _mask24_1 = 0x0408080810101020LL;
319
static unsigned long long _mask24_0 = 0x2020404040808080LL;
321
static unsigned long long _mask32_3 = 0x0101010102020202LL;
322
static unsigned long long _mask32_2 = 0x0404040408080808LL;
323
static unsigned long long _mask32_1 = 0x1010101020202020LL;
324
static unsigned long long _mask32_0 = 0x4040404080808080LL;
326
static unsigned long long _mask48_5 = 0x0101010101010202LL;
327
static unsigned long long _mask48_4 = 0x0202020204040404LL;
328
static unsigned long long _mask48_3 = 0x0404080808080808LL;
329
static unsigned long long _mask48_2 = 0x1010101010102020LL;
330
static unsigned long long _mask48_1 = 0x2020202040404040LL;
331
static unsigned long long _mask48_0 = 0x4040808080808080LL;
333
static unsigned long long _const4 = 0x0000000000FFFFFFLL;
334
//static unsigned long long _const5 = 0x000000FFFFFF0000LL; // NOT USED
335
static unsigned long long _const6 = 0x00000000000000FFLL;
337
// These are used in the row-filter routines and should/would be local
338
// variables if not for gcc addressing limitations.
339
// WARNING: Their presence probably defeats the thread safety of libpng.
341
#ifdef PNG_THREAD_UNSAFE_OK
342
static png_uint_32 _FullLength;
343
static png_uint_32 _MMXLength;
345
static int _patemp; // temp variables for Paeth routine
351
png_squelch_warnings(void)
353
#ifdef PNG_THREAD_UNSAFE_OK
358
_MMXLength = _MMXLength;
363
_mask16_1 = _mask16_1;
364
_mask16_0 = _mask16_0;
365
_mask24_2 = _mask24_2;
366
_mask24_1 = _mask24_1;
367
_mask24_0 = _mask24_0;
368
_mask32_3 = _mask32_3;
369
_mask32_2 = _mask32_2;
370
_mask32_1 = _mask32_1;
371
_mask32_0 = _mask32_0;
372
_mask48_5 = _mask48_5;
373
_mask48_4 = _mask48_4;
374
_mask48_3 = _mask48_3;
375
_mask48_2 = _mask48_2;
376
_mask48_1 = _mask48_1;
377
_mask48_0 = _mask48_0;
379
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
382
static int _mmx_supported = 2;
384
/*===========================================================================*/
386
/* P N G _ C O M B I N E _ R O W */
388
/*===========================================================================*/
390
#if defined(PNG_HAVE_ASSEMBLER_COMBINE_ROW)
393
#define BPP3 3 /* bytes per pixel (a.k.a. pixel_bytes) */
395
#define BPP6 6 /* (defined only to help avoid cut-and-paste errors) */
398
/* Combines the row recently read in with the previous row.
399
This routine takes care of alpha and transparency if requested.
400
This routine also handles the two methods of progressive display
401
of interlaced images, depending on the mask value.
402
The mask value describes which pixels are to be combined with
403
the row. The pattern always repeats every 8 pixels, so just 8
404
bits are needed. A one indicates the pixel is to be combined; a
405
zero indicates the pixel is to be skipped. This is in addition
406
to any alpha or transparency value associated with the pixel.
407
If you want all pixels to be combined, pass 0xff (255) in mask. */
409
/* Use this routine for the x86 platform - it uses a faster MMX routine
410
if the machine supports MMX. */
413
png_combine_row(png_structp png_ptr, png_bytep row, int mask)
415
png_debug(1, "in png_combine_row (pnggccrd.c)\n");
417
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
418
if (_mmx_supported == 2) {
419
#if !defined(PNG_1_0_X)
420
/* this should have happened in png_init_mmx_flags() already */
421
png_warning(png_ptr, "asm_flags may not have been initialized");
429
png_debug(2,"mask == 0xff: doing single png_memcpy()\n");
430
png_memcpy(row, png_ptr->row_buf + 1,
431
(png_size_t)PNG_ROWBYTES(png_ptr->row_info.pixel_depth,png_ptr->width));
433
else /* (png_combine_row() is never called with mask == 0) */
435
switch (png_ptr->row_info.pixel_depth)
437
case 1: /* png_ptr->row_info.pixel_depth */
441
int s_inc, s_start, s_end;
446
sp = png_ptr->row_buf + 1;
449
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
450
if (png_ptr->transformations & PNG_PACKSWAP)
466
for (i = 0; i < png_ptr->width; i++)
472
value = (*sp >> shift) & 0x1;
473
*dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff);
474
*dp |= (png_byte)(value << shift);
494
case 2: /* png_ptr->row_info.pixel_depth */
498
int s_start, s_end, s_inc;
504
sp = png_ptr->row_buf + 1;
507
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
508
if (png_ptr->transformations & PNG_PACKSWAP)
524
for (i = 0; i < png_ptr->width; i++)
528
value = (*sp >> shift) & 0x3;
529
*dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff);
530
*dp |= (png_byte)(value << shift);
549
case 4: /* png_ptr->row_info.pixel_depth */
553
int s_start, s_end, s_inc;
559
sp = png_ptr->row_buf + 1;
562
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
563
if (png_ptr->transformations & PNG_PACKSWAP)
578
for (i = 0; i < png_ptr->width; i++)
582
value = (*sp >> shift) & 0xf;
583
*dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff);
584
*dp |= (png_byte)(value << shift);
603
case 8: /* png_ptr->row_info.pixel_depth */
608
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
609
#if !defined(PNG_1_0_X)
610
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
611
/* && _mmx_supported */ )
618
int dummy_value_a; // fix 'forbidden register spilled' error
623
_unmask = ~mask; // global variable for -fPIC version
624
srcptr = png_ptr->row_buf + 1;
626
len = png_ptr->width &~7; // reduce to multiple of 8
627
diff = (int) (png_ptr->width & 7); // amount lost
629
__asm__ __volatile__ (
630
"movd _unmask, %%mm7 \n\t" // load bit pattern
631
"psubb %%mm6, %%mm6 \n\t" // zero mm6
632
"punpcklbw %%mm7, %%mm7 \n\t"
633
"punpcklwd %%mm7, %%mm7 \n\t"
634
"punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
636
"movq _mask8_0, %%mm0 \n\t"
637
"pand %%mm7, %%mm0 \n\t" // nonzero if keep byte
638
"pcmpeqb %%mm6, %%mm0 \n\t" // zeros->1s, v versa
640
// preload "movl len, %%ecx \n\t" // load length of line
641
// preload "movl srcptr, %%esi \n\t" // load source
642
// preload "movl dstptr, %%edi \n\t" // load dest
644
"cmpl $0, %%ecx \n\t" // len == 0 ?
645
"je mainloop8end \n\t"
648
"movq (%%esi), %%mm4 \n\t" // *srcptr
649
"pand %%mm0, %%mm4 \n\t"
650
"movq %%mm0, %%mm6 \n\t"
651
"pandn (%%edi), %%mm6 \n\t" // *dstptr
652
"por %%mm6, %%mm4 \n\t"
653
"movq %%mm4, (%%edi) \n\t"
654
"addl $8, %%esi \n\t" // inc by 8 bytes processed
655
"addl $8, %%edi \n\t"
656
"subl $8, %%ecx \n\t" // dec by 8 pixels processed
660
// preload "movl diff, %%ecx \n\t" // (diff is in eax)
661
"movl %%eax, %%ecx \n\t"
662
"cmpl $0, %%ecx \n\t"
664
// preload "movl mask, %%edx \n\t"
665
"sall $24, %%edx \n\t" // make low byte, high byte
668
"sall %%edx \n\t" // move high bit to CF
669
"jnc skip8 \n\t" // if CF = 0
670
"movb (%%esi), %%al \n\t"
671
"movb %%al, (%%edi) \n\t"
677
"jnz secondloop8 \n\t"
682
: "=a" (dummy_value_a), // output regs (dummy)
683
"=d" (dummy_value_d),
684
"=c" (dummy_value_c),
685
"=S" (dummy_value_S),
688
: "3" (srcptr), // esi // input regs
691
// was (unmask) "b" RESERVED // ebx // Global Offset Table idx
695
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
696
: "%mm0", "%mm4", "%mm6", "%mm7" // clobber list
700
else /* mmx _not supported - Use modified C routine */
701
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
703
register png_uint_32 i;
704
png_uint_32 initial_val = png_pass_start[png_ptr->pass];
705
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
706
register int stride = png_pass_inc[png_ptr->pass];
707
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
708
register int rep_bytes = png_pass_width[png_ptr->pass];
709
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
710
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
711
int diff = (int) (png_ptr->width & 7); /* amount lost */
712
register png_uint_32 final_val = len; /* GRR bugfix */
714
srcptr = png_ptr->row_buf + 1 + initial_val;
715
dstptr = row + initial_val;
717
for (i = initial_val; i < final_val; i += stride)
719
png_memcpy(dstptr, srcptr, rep_bytes);
723
if (diff) /* number of leftover pixels: 3 for pngtest */
725
final_val+=diff /* *BPP1 */ ;
726
for (; i < final_val; i += stride)
728
if (rep_bytes > (int)(final_val-i))
729
rep_bytes = (int)(final_val-i);
730
png_memcpy(dstptr, srcptr, rep_bytes);
736
} /* end of else (_mmx_supported) */
741
case 16: /* png_ptr->row_info.pixel_depth */
746
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
747
#if !defined(PNG_1_0_X)
748
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
749
/* && _mmx_supported */ )
756
int dummy_value_a; // fix 'forbidden register spilled' error
761
_unmask = ~mask; // global variable for -fPIC version
762
srcptr = png_ptr->row_buf + 1;
764
len = png_ptr->width &~7; // reduce to multiple of 8
765
diff = (int) (png_ptr->width & 7); // amount lost //
767
__asm__ __volatile__ (
768
"movd _unmask, %%mm7 \n\t" // load bit pattern
769
"psubb %%mm6, %%mm6 \n\t" // zero mm6
770
"punpcklbw %%mm7, %%mm7 \n\t"
771
"punpcklwd %%mm7, %%mm7 \n\t"
772
"punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
774
"movq _mask16_0, %%mm0 \n\t"
775
"movq _mask16_1, %%mm1 \n\t"
777
"pand %%mm7, %%mm0 \n\t"
778
"pand %%mm7, %%mm1 \n\t"
780
"pcmpeqb %%mm6, %%mm0 \n\t"
781
"pcmpeqb %%mm6, %%mm1 \n\t"
783
// preload "movl len, %%ecx \n\t" // load length of line
784
// preload "movl srcptr, %%esi \n\t" // load source
785
// preload "movl dstptr, %%edi \n\t" // load dest
787
"cmpl $0, %%ecx \n\t"
788
"jz mainloop16end \n\t"
791
"movq (%%esi), %%mm4 \n\t"
792
"pand %%mm0, %%mm4 \n\t"
793
"movq %%mm0, %%mm6 \n\t"
794
"movq (%%edi), %%mm7 \n\t"
795
"pandn %%mm7, %%mm6 \n\t"
796
"por %%mm6, %%mm4 \n\t"
797
"movq %%mm4, (%%edi) \n\t"
799
"movq 8(%%esi), %%mm5 \n\t"
800
"pand %%mm1, %%mm5 \n\t"
801
"movq %%mm1, %%mm7 \n\t"
802
"movq 8(%%edi), %%mm6 \n\t"
803
"pandn %%mm6, %%mm7 \n\t"
804
"por %%mm7, %%mm5 \n\t"
805
"movq %%mm5, 8(%%edi) \n\t"
807
"addl $16, %%esi \n\t" // inc by 16 bytes processed
808
"addl $16, %%edi \n\t"
809
"subl $8, %%ecx \n\t" // dec by 8 pixels processed
812
"mainloop16end: \n\t"
813
// preload "movl diff, %%ecx \n\t" // (diff is in eax)
814
"movl %%eax, %%ecx \n\t"
815
"cmpl $0, %%ecx \n\t"
817
// preload "movl mask, %%edx \n\t"
818
"sall $24, %%edx \n\t" // make low byte, high byte
821
"sall %%edx \n\t" // move high bit to CF
822
"jnc skip16 \n\t" // if CF = 0
823
"movw (%%esi), %%ax \n\t"
824
"movw %%ax, (%%edi) \n\t"
827
"addl $2, %%esi \n\t"
828
"addl $2, %%edi \n\t"
830
"jnz secondloop16 \n\t"
835
: "=a" (dummy_value_a), // output regs (dummy)
836
"=c" (dummy_value_c),
837
"=d" (dummy_value_d),
838
"=S" (dummy_value_S),
841
: "0" (diff), // eax // input regs
842
// was (unmask) " " RESERVED // ebx // Global Offset Table idx
848
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
849
: "%mm0", "%mm1", "%mm4" // clobber list
850
, "%mm5", "%mm6", "%mm7"
854
else /* mmx _not supported - Use modified C routine */
855
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
857
register png_uint_32 i;
858
png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass];
859
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
860
register int stride = BPP2 * png_pass_inc[png_ptr->pass];
861
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
862
register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass];
863
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
864
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
865
int diff = (int) (png_ptr->width & 7); /* amount lost */
866
register png_uint_32 final_val = BPP2 * len; /* GRR bugfix */
868
srcptr = png_ptr->row_buf + 1 + initial_val;
869
dstptr = row + initial_val;
871
for (i = initial_val; i < final_val; i += stride)
873
png_memcpy(dstptr, srcptr, rep_bytes);
877
if (diff) /* number of leftover pixels: 3 for pngtest */
879
final_val+=diff*BPP2;
880
for (; i < final_val; i += stride)
882
if (rep_bytes > (int)(final_val-i))
883
rep_bytes = (int)(final_val-i);
884
png_memcpy(dstptr, srcptr, rep_bytes);
889
} /* end of else (_mmx_supported) */
894
case 24: /* png_ptr->row_info.pixel_depth */
899
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
900
#if !defined(PNG_1_0_X)
901
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
902
/* && _mmx_supported */ )
909
int dummy_value_a; // fix 'forbidden register spilled' error
914
_unmask = ~mask; // global variable for -fPIC version
915
srcptr = png_ptr->row_buf + 1;
917
len = png_ptr->width &~7; // reduce to multiple of 8
918
diff = (int) (png_ptr->width & 7); // amount lost //
920
__asm__ __volatile__ (
921
"movd _unmask, %%mm7 \n\t" // load bit pattern
922
"psubb %%mm6, %%mm6 \n\t" // zero mm6
923
"punpcklbw %%mm7, %%mm7 \n\t"
924
"punpcklwd %%mm7, %%mm7 \n\t"
925
"punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
927
"movq _mask24_0, %%mm0 \n\t"
928
"movq _mask24_1, %%mm1 \n\t"
929
"movq _mask24_2, %%mm2 \n\t"
931
"pand %%mm7, %%mm0 \n\t"
932
"pand %%mm7, %%mm1 \n\t"
933
"pand %%mm7, %%mm2 \n\t"
935
"pcmpeqb %%mm6, %%mm0 \n\t"
936
"pcmpeqb %%mm6, %%mm1 \n\t"
937
"pcmpeqb %%mm6, %%mm2 \n\t"
939
// preload "movl len, %%ecx \n\t" // load length of line
940
// preload "movl srcptr, %%esi \n\t" // load source
941
// preload "movl dstptr, %%edi \n\t" // load dest
943
"cmpl $0, %%ecx \n\t"
944
"jz mainloop24end \n\t"
947
"movq (%%esi), %%mm4 \n\t"
948
"pand %%mm0, %%mm4 \n\t"
949
"movq %%mm0, %%mm6 \n\t"
950
"movq (%%edi), %%mm7 \n\t"
951
"pandn %%mm7, %%mm6 \n\t"
952
"por %%mm6, %%mm4 \n\t"
953
"movq %%mm4, (%%edi) \n\t"
955
"movq 8(%%esi), %%mm5 \n\t"
956
"pand %%mm1, %%mm5 \n\t"
957
"movq %%mm1, %%mm7 \n\t"
958
"movq 8(%%edi), %%mm6 \n\t"
959
"pandn %%mm6, %%mm7 \n\t"
960
"por %%mm7, %%mm5 \n\t"
961
"movq %%mm5, 8(%%edi) \n\t"
963
"movq 16(%%esi), %%mm6 \n\t"
964
"pand %%mm2, %%mm6 \n\t"
965
"movq %%mm2, %%mm4 \n\t"
966
"movq 16(%%edi), %%mm7 \n\t"
967
"pandn %%mm7, %%mm4 \n\t"
968
"por %%mm4, %%mm6 \n\t"
969
"movq %%mm6, 16(%%edi) \n\t"
971
"addl $24, %%esi \n\t" // inc by 24 bytes processed
972
"addl $24, %%edi \n\t"
973
"subl $8, %%ecx \n\t" // dec by 8 pixels processed
977
"mainloop24end: \n\t"
978
// preload "movl diff, %%ecx \n\t" // (diff is in eax)
979
"movl %%eax, %%ecx \n\t"
980
"cmpl $0, %%ecx \n\t"
982
// preload "movl mask, %%edx \n\t"
983
"sall $24, %%edx \n\t" // make low byte, high byte
986
"sall %%edx \n\t" // move high bit to CF
987
"jnc skip24 \n\t" // if CF = 0
988
"movw (%%esi), %%ax \n\t"
989
"movw %%ax, (%%edi) \n\t"
990
"xorl %%eax, %%eax \n\t"
991
"movb 2(%%esi), %%al \n\t"
992
"movb %%al, 2(%%edi) \n\t"
995
"addl $3, %%esi \n\t"
996
"addl $3, %%edi \n\t"
998
"jnz secondloop24 \n\t"
1003
: "=a" (dummy_value_a), // output regs (dummy)
1004
"=d" (dummy_value_d),
1005
"=c" (dummy_value_c),
1006
"=S" (dummy_value_S),
1007
"=D" (dummy_value_D)
1009
: "3" (srcptr), // esi // input regs
1010
"4" (dstptr), // edi
1012
// was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1016
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1017
: "%mm0", "%mm1", "%mm2" // clobber list
1018
, "%mm4", "%mm5", "%mm6", "%mm7"
1022
else /* mmx _not supported - Use modified C routine */
1023
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1025
register png_uint_32 i;
1026
png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass];
1027
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1028
register int stride = BPP3 * png_pass_inc[png_ptr->pass];
1029
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1030
register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass];
1031
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1032
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1033
int diff = (int) (png_ptr->width & 7); /* amount lost */
1034
register png_uint_32 final_val = BPP3 * len; /* GRR bugfix */
1036
srcptr = png_ptr->row_buf + 1 + initial_val;
1037
dstptr = row + initial_val;
1039
for (i = initial_val; i < final_val; i += stride)
1041
png_memcpy(dstptr, srcptr, rep_bytes);
1045
if (diff) /* number of leftover pixels: 3 for pngtest */
1047
final_val+=diff*BPP3;
1048
for (; i < final_val; i += stride)
1050
if (rep_bytes > (int)(final_val-i))
1051
rep_bytes = (int)(final_val-i);
1052
png_memcpy(dstptr, srcptr, rep_bytes);
1057
} /* end of else (_mmx_supported) */
1062
case 32: /* png_ptr->row_info.pixel_depth */
1067
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1068
#if !defined(PNG_1_0_X)
1069
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1070
/* && _mmx_supported */ )
1077
int dummy_value_a; // fix 'forbidden register spilled' error
1082
_unmask = ~mask; // global variable for -fPIC version
1083
srcptr = png_ptr->row_buf + 1;
1085
len = png_ptr->width &~7; // reduce to multiple of 8
1086
diff = (int) (png_ptr->width & 7); // amount lost //
1088
__asm__ __volatile__ (
1089
"movd _unmask, %%mm7 \n\t" // load bit pattern
1090
"psubb %%mm6, %%mm6 \n\t" // zero mm6
1091
"punpcklbw %%mm7, %%mm7 \n\t"
1092
"punpcklwd %%mm7, %%mm7 \n\t"
1093
"punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1095
"movq _mask32_0, %%mm0 \n\t"
1096
"movq _mask32_1, %%mm1 \n\t"
1097
"movq _mask32_2, %%mm2 \n\t"
1098
"movq _mask32_3, %%mm3 \n\t"
1100
"pand %%mm7, %%mm0 \n\t"
1101
"pand %%mm7, %%mm1 \n\t"
1102
"pand %%mm7, %%mm2 \n\t"
1103
"pand %%mm7, %%mm3 \n\t"
1105
"pcmpeqb %%mm6, %%mm0 \n\t"
1106
"pcmpeqb %%mm6, %%mm1 \n\t"
1107
"pcmpeqb %%mm6, %%mm2 \n\t"
1108
"pcmpeqb %%mm6, %%mm3 \n\t"
1110
// preload "movl len, %%ecx \n\t" // load length of line
1111
// preload "movl srcptr, %%esi \n\t" // load source
1112
// preload "movl dstptr, %%edi \n\t" // load dest
1114
"cmpl $0, %%ecx \n\t" // lcr
1115
"jz mainloop32end \n\t"
1118
"movq (%%esi), %%mm4 \n\t"
1119
"pand %%mm0, %%mm4 \n\t"
1120
"movq %%mm0, %%mm6 \n\t"
1121
"movq (%%edi), %%mm7 \n\t"
1122
"pandn %%mm7, %%mm6 \n\t"
1123
"por %%mm6, %%mm4 \n\t"
1124
"movq %%mm4, (%%edi) \n\t"
1126
"movq 8(%%esi), %%mm5 \n\t"
1127
"pand %%mm1, %%mm5 \n\t"
1128
"movq %%mm1, %%mm7 \n\t"
1129
"movq 8(%%edi), %%mm6 \n\t"
1130
"pandn %%mm6, %%mm7 \n\t"
1131
"por %%mm7, %%mm5 \n\t"
1132
"movq %%mm5, 8(%%edi) \n\t"
1134
"movq 16(%%esi), %%mm6 \n\t"
1135
"pand %%mm2, %%mm6 \n\t"
1136
"movq %%mm2, %%mm4 \n\t"
1137
"movq 16(%%edi), %%mm7 \n\t"
1138
"pandn %%mm7, %%mm4 \n\t"
1139
"por %%mm4, %%mm6 \n\t"
1140
"movq %%mm6, 16(%%edi) \n\t"
1142
"movq 24(%%esi), %%mm7 \n\t"
1143
"pand %%mm3, %%mm7 \n\t"
1144
"movq %%mm3, %%mm5 \n\t"
1145
"movq 24(%%edi), %%mm4 \n\t"
1146
"pandn %%mm4, %%mm5 \n\t"
1147
"por %%mm5, %%mm7 \n\t"
1148
"movq %%mm7, 24(%%edi) \n\t"
1150
"addl $32, %%esi \n\t" // inc by 32 bytes processed
1151
"addl $32, %%edi \n\t"
1152
"subl $8, %%ecx \n\t" // dec by 8 pixels processed
1153
"ja mainloop32 \n\t"
1155
"mainloop32end: \n\t"
1156
// preload "movl diff, %%ecx \n\t" // (diff is in eax)
1157
"movl %%eax, %%ecx \n\t"
1158
"cmpl $0, %%ecx \n\t"
1160
// preload "movl mask, %%edx \n\t"
1161
"sall $24, %%edx \n\t" // low byte => high byte
1163
"secondloop32: \n\t"
1164
"sall %%edx \n\t" // move high bit to CF
1165
"jnc skip32 \n\t" // if CF = 0
1166
"movl (%%esi), %%eax \n\t"
1167
"movl %%eax, (%%edi) \n\t"
1170
"addl $4, %%esi \n\t"
1171
"addl $4, %%edi \n\t"
1173
"jnz secondloop32 \n\t"
1178
: "=a" (dummy_value_a), // output regs (dummy)
1179
"=d" (dummy_value_d),
1180
"=c" (dummy_value_c),
1181
"=S" (dummy_value_S),
1182
"=D" (dummy_value_D)
1184
: "3" (srcptr), // esi // input regs
1185
"4" (dstptr), // edi
1187
// was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1191
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1192
: "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
1193
, "%mm4", "%mm5", "%mm6", "%mm7"
1197
else /* mmx _not supported - Use modified C routine */
1198
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1200
register png_uint_32 i;
1201
png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass];
1202
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1203
register int stride = BPP4 * png_pass_inc[png_ptr->pass];
1204
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1205
register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass];
1206
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1207
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1208
int diff = (int) (png_ptr->width & 7); /* amount lost */
1209
register png_uint_32 final_val = BPP4 * len; /* GRR bugfix */
1211
srcptr = png_ptr->row_buf + 1 + initial_val;
1212
dstptr = row + initial_val;
1214
for (i = initial_val; i < final_val; i += stride)
1216
png_memcpy(dstptr, srcptr, rep_bytes);
1220
if (diff) /* number of leftover pixels: 3 for pngtest */
1222
final_val+=diff*BPP4;
1223
for (; i < final_val; i += stride)
1225
if (rep_bytes > (int)(final_val-i))
1226
rep_bytes = (int)(final_val-i);
1227
png_memcpy(dstptr, srcptr, rep_bytes);
1232
} /* end of else (_mmx_supported) */
1237
case 48: /* png_ptr->row_info.pixel_depth */
1242
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1243
#if !defined(PNG_1_0_X)
1244
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1245
/* && _mmx_supported */ )
1252
int dummy_value_a; // fix 'forbidden register spilled' error
1257
_unmask = ~mask; // global variable for -fPIC version
1258
srcptr = png_ptr->row_buf + 1;
1260
len = png_ptr->width &~7; // reduce to multiple of 8
1261
diff = (int) (png_ptr->width & 7); // amount lost //
1263
__asm__ __volatile__ (
1264
"movd _unmask, %%mm7 \n\t" // load bit pattern
1265
"psubb %%mm6, %%mm6 \n\t" // zero mm6
1266
"punpcklbw %%mm7, %%mm7 \n\t"
1267
"punpcklwd %%mm7, %%mm7 \n\t"
1268
"punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1270
"movq _mask48_0, %%mm0 \n\t"
1271
"movq _mask48_1, %%mm1 \n\t"
1272
"movq _mask48_2, %%mm2 \n\t"
1273
"movq _mask48_3, %%mm3 \n\t"
1274
"movq _mask48_4, %%mm4 \n\t"
1275
"movq _mask48_5, %%mm5 \n\t"
1277
"pand %%mm7, %%mm0 \n\t"
1278
"pand %%mm7, %%mm1 \n\t"
1279
"pand %%mm7, %%mm2 \n\t"
1280
"pand %%mm7, %%mm3 \n\t"
1281
"pand %%mm7, %%mm4 \n\t"
1282
"pand %%mm7, %%mm5 \n\t"
1284
"pcmpeqb %%mm6, %%mm0 \n\t"
1285
"pcmpeqb %%mm6, %%mm1 \n\t"
1286
"pcmpeqb %%mm6, %%mm2 \n\t"
1287
"pcmpeqb %%mm6, %%mm3 \n\t"
1288
"pcmpeqb %%mm6, %%mm4 \n\t"
1289
"pcmpeqb %%mm6, %%mm5 \n\t"
1291
// preload "movl len, %%ecx \n\t" // load length of line
1292
// preload "movl srcptr, %%esi \n\t" // load source
1293
// preload "movl dstptr, %%edi \n\t" // load dest
1295
"cmpl $0, %%ecx \n\t"
1296
"jz mainloop48end \n\t"
1299
"movq (%%esi), %%mm7 \n\t"
1300
"pand %%mm0, %%mm7 \n\t"
1301
"movq %%mm0, %%mm6 \n\t"
1302
"pandn (%%edi), %%mm6 \n\t"
1303
"por %%mm6, %%mm7 \n\t"
1304
"movq %%mm7, (%%edi) \n\t"
1306
"movq 8(%%esi), %%mm6 \n\t"
1307
"pand %%mm1, %%mm6 \n\t"
1308
"movq %%mm1, %%mm7 \n\t"
1309
"pandn 8(%%edi), %%mm7 \n\t"
1310
"por %%mm7, %%mm6 \n\t"
1311
"movq %%mm6, 8(%%edi) \n\t"
1313
"movq 16(%%esi), %%mm6 \n\t"
1314
"pand %%mm2, %%mm6 \n\t"
1315
"movq %%mm2, %%mm7 \n\t"
1316
"pandn 16(%%edi), %%mm7 \n\t"
1317
"por %%mm7, %%mm6 \n\t"
1318
"movq %%mm6, 16(%%edi) \n\t"
1320
"movq 24(%%esi), %%mm7 \n\t"
1321
"pand %%mm3, %%mm7 \n\t"
1322
"movq %%mm3, %%mm6 \n\t"
1323
"pandn 24(%%edi), %%mm6 \n\t"
1324
"por %%mm6, %%mm7 \n\t"
1325
"movq %%mm7, 24(%%edi) \n\t"
1327
"movq 32(%%esi), %%mm6 \n\t"
1328
"pand %%mm4, %%mm6 \n\t"
1329
"movq %%mm4, %%mm7 \n\t"
1330
"pandn 32(%%edi), %%mm7 \n\t"
1331
"por %%mm7, %%mm6 \n\t"
1332
"movq %%mm6, 32(%%edi) \n\t"
1334
"movq 40(%%esi), %%mm7 \n\t"
1335
"pand %%mm5, %%mm7 \n\t"
1336
"movq %%mm5, %%mm6 \n\t"
1337
"pandn 40(%%edi), %%mm6 \n\t"
1338
"por %%mm6, %%mm7 \n\t"
1339
"movq %%mm7, 40(%%edi) \n\t"
1341
"addl $48, %%esi \n\t" // inc by 48 bytes processed
1342
"addl $48, %%edi \n\t"
1343
"subl $8, %%ecx \n\t" // dec by 8 pixels processed
1345
"ja mainloop48 \n\t"
1347
"mainloop48end: \n\t"
1348
// preload "movl diff, %%ecx \n\t" // (diff is in eax)
1349
"movl %%eax, %%ecx \n\t"
1350
"cmpl $0, %%ecx \n\t"
1352
// preload "movl mask, %%edx \n\t"
1353
"sall $24, %%edx \n\t" // make low byte, high byte
1355
"secondloop48: \n\t"
1356
"sall %%edx \n\t" // move high bit to CF
1357
"jnc skip48 \n\t" // if CF = 0
1358
"movl (%%esi), %%eax \n\t"
1359
"movl %%eax, (%%edi) \n\t"
1362
"addl $4, %%esi \n\t"
1363
"addl $4, %%edi \n\t"
1365
"jnz secondloop48 \n\t"
1370
: "=a" (dummy_value_a), // output regs (dummy)
1371
"=d" (dummy_value_d),
1372
"=c" (dummy_value_c),
1373
"=S" (dummy_value_S),
1374
"=D" (dummy_value_D)
1376
: "3" (srcptr), // esi // input regs
1377
"4" (dstptr), // edi
1379
// was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1383
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1384
: "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
1385
, "%mm4", "%mm5", "%mm6", "%mm7"
1389
else /* mmx _not supported - Use modified C routine */
1390
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1392
register png_uint_32 i;
1393
png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass];
1394
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1395
register int stride = BPP6 * png_pass_inc[png_ptr->pass];
1396
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1397
register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass];
1398
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1399
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1400
int diff = (int) (png_ptr->width & 7); /* amount lost */
1401
register png_uint_32 final_val = BPP6 * len; /* GRR bugfix */
1403
srcptr = png_ptr->row_buf + 1 + initial_val;
1404
dstptr = row + initial_val;
1406
for (i = initial_val; i < final_val; i += stride)
1408
png_memcpy(dstptr, srcptr, rep_bytes);
1412
if (diff) /* number of leftover pixels: 3 for pngtest */
1414
final_val+=diff*BPP6;
1415
for (; i < final_val; i += stride)
1417
if (rep_bytes > (int)(final_val-i))
1418
rep_bytes = (int)(final_val-i);
1419
png_memcpy(dstptr, srcptr, rep_bytes);
1424
} /* end of else (_mmx_supported) */
1429
case 64: /* png_ptr->row_info.pixel_depth */
1433
register png_uint_32 i;
1434
png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass];
1435
/* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1436
register int stride = BPP8 * png_pass_inc[png_ptr->pass];
1437
/* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1438
register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass];
1439
/* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1440
png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1441
int diff = (int) (png_ptr->width & 7); /* amount lost */
1442
register png_uint_32 final_val = BPP8 * len; /* GRR bugfix */
1444
srcptr = png_ptr->row_buf + 1 + initial_val;
1445
dstptr = row + initial_val;
1447
for (i = initial_val; i < final_val; i += stride)
1449
png_memcpy(dstptr, srcptr, rep_bytes);
1453
if (diff) /* number of leftover pixels: 3 for pngtest */
1455
final_val+=diff*BPP8;
1456
for (; i < final_val; i += stride)
1458
if (rep_bytes > (int)(final_val-i))
1459
rep_bytes = (int)(final_val-i);
1460
png_memcpy(dstptr, srcptr, rep_bytes);
1469
default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */
1471
/* this should never happen */
1472
png_warning(png_ptr, "Invalid row_info.pixel_depth in pnggccrd");
1475
} /* end switch (png_ptr->row_info.pixel_depth) */
1477
} /* end if (non-trivial mask) */
1479
} /* end png_combine_row() */
1481
#endif /* PNG_HAVE_ASSEMBLER_COMBINE_ROW */
1486
/*===========================================================================*/
1488
/* P N G _ D O _ R E A D _ I N T E R L A C E */
1490
/*===========================================================================*/
1492
#if defined(PNG_READ_INTERLACING_SUPPORTED)
1493
#if defined(PNG_HAVE_ASSEMBLER_READ_INTERLACE)
1495
/* png_do_read_interlace() is called after any 16-bit to 8-bit conversion
1496
* has taken place. [GRR: what other steps come before and/or after?]
1500
png_do_read_interlace(png_structp png_ptr)
1502
png_row_infop row_info = &(png_ptr->row_info);
1503
png_bytep row = png_ptr->row_buf + 1;
1504
int pass = png_ptr->pass;
1505
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
1506
png_uint_32 transformations = png_ptr->transformations;
1509
png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n");
1511
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1512
if (_mmx_supported == 2) {
1513
#if !defined(PNG_1_0_X)
1514
/* this should have happened in png_init_mmx_flags() already */
1515
png_warning(png_ptr, "asm_flags may not have been initialized");
1521
if (row != NULL && row_info != NULL)
1523
png_uint_32 final_width;
1525
final_width = row_info->width * png_pass_inc[pass];
1527
switch (row_info->pixel_depth)
1533
int s_start, s_end, s_inc;
1538
sp = row + (png_size_t)((row_info->width - 1) >> 3);
1539
dp = row + (png_size_t)((final_width - 1) >> 3);
1540
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
1541
if (transformations & PNG_PACKSWAP)
1543
sshift = (int)((row_info->width + 7) & 7);
1544
dshift = (int)((final_width + 7) & 7);
1552
sshift = 7 - (int)((row_info->width + 7) & 7);
1553
dshift = 7 - (int)((final_width + 7) & 7);
1559
for (i = row_info->width; i; i--)
1561
v = (png_byte)((*sp >> sshift) & 0x1);
1562
for (j = 0; j < png_pass_inc[pass]; j++)
1564
*dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff);
1565
*dp |= (png_byte)(v << dshift);
1566
if (dshift == s_end)
1574
if (sshift == s_end)
1589
int s_start, s_end, s_inc;
1592
sp = row + (png_size_t)((row_info->width - 1) >> 2);
1593
dp = row + (png_size_t)((final_width - 1) >> 2);
1594
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
1595
if (transformations & PNG_PACKSWAP)
1597
sshift = (png_size_t)(((row_info->width + 3) & 3) << 1);
1598
dshift = (png_size_t)(((final_width + 3) & 3) << 1);
1606
sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1);
1607
dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1);
1613
for (i = row_info->width; i; i--)
1618
v = (png_byte)((*sp >> sshift) & 0x3);
1619
for (j = 0; j < png_pass_inc[pass]; j++)
1621
*dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff);
1622
*dp |= (png_byte)(v << dshift);
1623
if (dshift == s_end)
1631
if (sshift == s_end)
1646
int s_start, s_end, s_inc;
1649
sp = row + (png_size_t)((row_info->width - 1) >> 1);
1650
dp = row + (png_size_t)((final_width - 1) >> 1);
1651
#if defined(PNG_READ_PACKSWAP_SUPPORTED)
1652
if (transformations & PNG_PACKSWAP)
1654
sshift = (png_size_t)(((row_info->width + 1) & 1) << 2);
1655
dshift = (png_size_t)(((final_width + 1) & 1) << 2);
1663
sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2);
1664
dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2);
1670
for (i = row_info->width; i; i--)
1675
v = (png_byte)((*sp >> sshift) & 0xf);
1676
for (j = 0; j < png_pass_inc[pass]; j++)
1678
*dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff);
1679
*dp |= (png_byte)(v << dshift);
1680
if (dshift == s_end)
1688
if (sshift == s_end)
1699
/*====================================================================*/
1701
default: /* 8-bit or larger (this is where the routine is modified) */
1704
// static unsigned long long _const4 = 0x0000000000FFFFFFLL; no good
1705
// static unsigned long long const4 = 0x0000000000FFFFFFLL; no good
1706
// unsigned long long _const4 = 0x0000000000FFFFFFLL; no good
1707
// unsigned long long const4 = 0x0000000000FFFFFFLL; no good
1711
png_size_t pixel_bytes;
1712
int width = (int)row_info->width;
1714
pixel_bytes = (row_info->pixel_depth >> 3);
1716
/* point sptr at the last pixel in the pre-expanded row: */
1717
sptr = row + (width - 1) * pixel_bytes;
1719
/* point dp at the last pixel position in the expanded row: */
1720
dp = row + (final_width - 1) * pixel_bytes;
1722
/* New code by Nirav Chhatrapati - Intel Corporation */
1724
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1725
#if !defined(PNG_1_0_X)
1726
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE)
1727
/* && _mmx_supported */ )
1732
//--------------------------------------------------------------
1733
if (pixel_bytes == 3)
1735
if (((pass == 0) || (pass == 1)) && width)
1737
int dummy_value_c; // fix 'forbidden register spilled'
1741
__asm__ __volatile__ (
1742
"subl $21, %%edi \n\t"
1743
// (png_pass_inc[pass] - 1)*pixel_bytes
1745
".loop3_pass0: \n\t"
1746
"movd (%%esi), %%mm0 \n\t" // x x x x x 2 1 0
1747
"pand _const4, %%mm0 \n\t" // z z z z z 2 1 0
1748
"movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0
1749
"psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z
1750
"movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z
1751
"psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z
1752
"psrlq $8, %%mm1 \n\t" // z z z z z z 2 1
1753
"por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z
1754
"por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1
1755
"movq %%mm0, %%mm3 \n\t" // 2 1 0 2 1 0 2 1
1756
"psllq $16, %%mm0 \n\t" // 0 2 1 0 2 1 z z
1757
"movq %%mm3, %%mm4 \n\t" // 2 1 0 2 1 0 2 1
1758
"punpckhdq %%mm0, %%mm3 \n\t" // 0 2 1 0 2 1 0 2
1759
"movq %%mm4, 16(%%edi) \n\t"
1760
"psrlq $32, %%mm0 \n\t" // z z z z 0 2 1 0
1761
"movq %%mm3, 8(%%edi) \n\t"
1762
"punpckldq %%mm4, %%mm0 \n\t" // 1 0 2 1 0 2 1 0
1763
"subl $3, %%esi \n\t"
1764
"movq %%mm0, (%%edi) \n\t"
1765
"subl $24, %%edi \n\t"
1767
"jnz .loop3_pass0 \n\t"
1770
: "=c" (dummy_value_c), // output regs (dummy)
1771
"=S" (dummy_value_S),
1772
"=D" (dummy_value_D)
1774
: "1" (sptr), // esi // input regs
1777
"rim" (_const4) // %1(?) (0x0000000000FFFFFFLL)
1779
#if 0 /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1780
: "%mm0", "%mm1", "%mm2" // clobber list
1785
else if (((pass == 2) || (pass == 3)) && width)
1787
int dummy_value_c; // fix 'forbidden register spilled'
1791
__asm__ __volatile__ (
1792
"subl $9, %%edi \n\t"
1793
// (png_pass_inc[pass] - 1)*pixel_bytes
1795
".loop3_pass2: \n\t"
1796
"movd (%%esi), %%mm0 \n\t" // x x x x x 2 1 0
1797
"pand _const4, %%mm0 \n\t" // z z z z z 2 1 0
1798
"movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0
1799
"psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z
1800
"movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z
1801
"psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z
1802
"psrlq $8, %%mm1 \n\t" // z z z z z z 2 1
1803
"por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z
1804
"por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1
1805
"movq %%mm0, 4(%%edi) \n\t"
1806
"psrlq $16, %%mm0 \n\t" // z z 2 1 0 2 1 0
1807
"subl $3, %%esi \n\t"
1808
"movd %%mm0, (%%edi) \n\t"
1809
"subl $12, %%edi \n\t"
1811
"jnz .loop3_pass2 \n\t"
1814
: "=c" (dummy_value_c), // output regs (dummy)
1815
"=S" (dummy_value_S),
1816
"=D" (dummy_value_D)
1818
: "1" (sptr), // esi // input regs
1821
"rim" (_const4) // (0x0000000000FFFFFFLL)
1823
#if 0 /* %mm0, ..., %mm2 not supported by gcc 2.7.2.3 or egcs 1.1 */
1824
: "%mm0", "%mm1", "%mm2" // clobber list
1828
else if (width) /* && ((pass == 4) || (pass == 5)) */
1830
int width_mmx = ((width >> 1) << 1) - 8; // GRR: huh?
1833
width -= width_mmx; // 8 or 9 pix, 24 or 27 bytes
1836
// png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1};
1837
// sptr points at last pixel in pre-expanded row
1838
// dp points at last pixel position in expanded row
1839
int dummy_value_c; // fix 'forbidden register spilled'
1843
__asm__ __volatile__ (
1844
"subl $3, %%esi \n\t"
1845
"subl $9, %%edi \n\t"
1846
// (png_pass_inc[pass] + 1)*pixel_bytes
1848
".loop3_pass4: \n\t"
1849
"movq (%%esi), %%mm0 \n\t" // x x 5 4 3 2 1 0
1850
"movq %%mm0, %%mm1 \n\t" // x x 5 4 3 2 1 0
1851
"movq %%mm0, %%mm2 \n\t" // x x 5 4 3 2 1 0
1852
"psllq $24, %%mm0 \n\t" // 4 3 2 1 0 z z z
1853
"pand _const4, %%mm1 \n\t" // z z z z z 2 1 0
1854
"psrlq $24, %%mm2 \n\t" // z z z x x 5 4 3
1855
"por %%mm1, %%mm0 \n\t" // 4 3 2 1 0 2 1 0
1856
"movq %%mm2, %%mm3 \n\t" // z z z x x 5 4 3
1857
"psllq $8, %%mm2 \n\t" // z z x x 5 4 3 z
1858
"movq %%mm0, (%%edi) \n\t"
1859
"psrlq $16, %%mm3 \n\t" // z z z z z x x 5
1860
"pand _const6, %%mm3 \n\t" // z z z z z z z 5
1861
"por %%mm3, %%mm2 \n\t" // z z x x 5 4 3 5
1862
"subl $6, %%esi \n\t"
1863
"movd %%mm2, 8(%%edi) \n\t"
1864
"subl $12, %%edi \n\t"
1865
"subl $2, %%ecx \n\t"
1866
"jnz .loop3_pass4 \n\t"
1869
: "=c" (dummy_value_c), // output regs (dummy)
1870
"=S" (dummy_value_S),
1871
"=D" (dummy_value_D)
1873
: "1" (sptr), // esi // input regs
1875
"0" (width_mmx), // ecx
1876
"rim" (_const4), // 0x0000000000FFFFFFLL
1877
"rim" (_const6) // 0x00000000000000FFLL
1879
#if 0 /* %mm0, ..., %mm3 not supported by gcc 2.7.2.3 or egcs 1.1 */
1880
: "%mm0", "%mm1" // clobber list
1886
sptr -= width_mmx*3;
1888
for (i = width; i; i--)
1893
png_memcpy(v, sptr, 3);
1894
for (j = 0; j < png_pass_inc[pass]; j++)
1896
png_memcpy(dp, v, 3);
1902
} /* end of pixel_bytes == 3 */
1904
//--------------------------------------------------------------
1905
else if (pixel_bytes == 1)
1907
if (((pass == 0) || (pass == 1)) && width)
1909
int width_mmx = ((width >> 2) << 2);
1910
width -= width_mmx; // 0-3 pixels => 0-3 bytes
1913
int dummy_value_c; // fix 'forbidden register spilled'
1917
__asm__ __volatile__ (
1918
"subl $3, %%esi \n\t"
1919
"subl $31, %%edi \n\t"
1921
".loop1_pass0: \n\t"
1922
"movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0
1923
"movq %%mm0, %%mm1 \n\t" // x x x x 3 2 1 0
1924
"punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
1925
"movq %%mm0, %%mm2 \n\t" // 3 3 2 2 1 1 0 0
1926
"punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0
1927
"movq %%mm0, %%mm3 \n\t" // 1 1 1 1 0 0 0 0
1928
"punpckldq %%mm0, %%mm0 \n\t" // 0 0 0 0 0 0 0 0
1929
"punpckhdq %%mm3, %%mm3 \n\t" // 1 1 1 1 1 1 1 1
1930
"movq %%mm0, (%%edi) \n\t"
1931
"punpckhwd %%mm2, %%mm2 \n\t" // 3 3 3 3 2 2 2 2
1932
"movq %%mm3, 8(%%edi) \n\t"
1933
"movq %%mm2, %%mm4 \n\t" // 3 3 3 3 2 2 2 2
1934
"punpckldq %%mm2, %%mm2 \n\t" // 2 2 2 2 2 2 2 2
1935
"punpckhdq %%mm4, %%mm4 \n\t" // 3 3 3 3 3 3 3 3
1936
"movq %%mm2, 16(%%edi) \n\t"
1937
"subl $4, %%esi \n\t"
1938
"movq %%mm4, 24(%%edi) \n\t"
1939
"subl $32, %%edi \n\t"
1940
"subl $4, %%ecx \n\t"
1941
"jnz .loop1_pass0 \n\t"
1944
: "=c" (dummy_value_c), // output regs (dummy)
1945
"=S" (dummy_value_S),
1946
"=D" (dummy_value_D)
1948
: "1" (sptr), // esi // input regs
1950
"0" (width_mmx) // ecx
1952
#if 0 /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1953
: "%mm0", "%mm1", "%mm2" // clobber list
1961
for (i = width; i; i--)
1965
/* I simplified this part in version 1.0.4e
1966
* here and in several other instances where
1967
* pixel_bytes == 1 -- GR-P
1972
* png_memcpy(v, sptr, pixel_bytes);
1973
* for (j = 0; j < png_pass_inc[pass]; j++)
1975
* png_memcpy(dp, v, pixel_bytes);
1976
* dp -= pixel_bytes;
1978
* sptr -= pixel_bytes;
1980
* Replacement code is in the next three lines:
1983
for (j = 0; j < png_pass_inc[pass]; j++)
1990
else if (((pass == 2) || (pass == 3)) && width)
1992
int width_mmx = ((width >> 2) << 2);
1993
width -= width_mmx; // 0-3 pixels => 0-3 bytes
1996
int dummy_value_c; // fix 'forbidden register spilled'
2000
__asm__ __volatile__ (
2001
"subl $3, %%esi \n\t"
2002
"subl $15, %%edi \n\t"
2004
".loop1_pass2: \n\t"
2005
"movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0
2006
"punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
2007
"movq %%mm0, %%mm1 \n\t" // 3 3 2 2 1 1 0 0
2008
"punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0
2009
"punpckhwd %%mm1, %%mm1 \n\t" // 3 3 3 3 2 2 2 2
2010
"movq %%mm0, (%%edi) \n\t"
2011
"subl $4, %%esi \n\t"
2012
"movq %%mm1, 8(%%edi) \n\t"
2013
"subl $16, %%edi \n\t"
2014
"subl $4, %%ecx \n\t"
2015
"jnz .loop1_pass2 \n\t"
2018
: "=c" (dummy_value_c), // output regs (dummy)
2019
"=S" (dummy_value_S),
2020
"=D" (dummy_value_D)
2022
: "1" (sptr), // esi // input regs
2024
"0" (width_mmx) // ecx
2026
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2027
: "%mm0", "%mm1" // clobber list
2034
for (i = width; i; i--)
2038
for (j = 0; j < png_pass_inc[pass]; j++)
2045
else if (width) /* && ((pass == 4) || (pass == 5)) */
2047
int width_mmx = ((width >> 3) << 3);
2048
width -= width_mmx; // 0-3 pixels => 0-3 bytes
2051
int dummy_value_c; // fix 'forbidden register spilled'
2055
__asm__ __volatile__ (
2056
"subl $7, %%esi \n\t"
2057
"subl $15, %%edi \n\t"
2059
".loop1_pass4: \n\t"
2060
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2061
"movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2062
"punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
2063
"punpckhbw %%mm1, %%mm1 \n\t" // 7 7 6 6 5 5 4 4
2064
"movq %%mm1, 8(%%edi) \n\t"
2065
"subl $8, %%esi \n\t"
2066
"movq %%mm0, (%%edi) \n\t"
2067
"subl $16, %%edi \n\t"
2068
"subl $8, %%ecx \n\t"
2069
"jnz .loop1_pass4 \n\t"
2072
: "=c" (dummy_value_c), // output regs (none)
2073
"=S" (dummy_value_S),
2074
"=D" (dummy_value_D)
2076
: "1" (sptr), // esi // input regs
2078
"0" (width_mmx) // ecx
2080
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2081
: "%mm0", "%mm1" // clobber list
2088
for (i = width; i; i--)
2092
for (j = 0; j < png_pass_inc[pass]; j++)
2099
} /* end of pixel_bytes == 1 */
2101
//--------------------------------------------------------------
2102
else if (pixel_bytes == 2)
2104
if (((pass == 0) || (pass == 1)) && width)
2106
int width_mmx = ((width >> 1) << 1);
2107
width -= width_mmx; // 0,1 pixels => 0,2 bytes
2110
int dummy_value_c; // fix 'forbidden register spilled'
2114
__asm__ __volatile__ (
2115
"subl $2, %%esi \n\t"
2116
"subl $30, %%edi \n\t"
2118
".loop2_pass0: \n\t"
2119
"movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0
2120
"punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2121
"movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0
2122
"punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0
2123
"punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2
2124
"movq %%mm0, (%%edi) \n\t"
2125
"movq %%mm0, 8(%%edi) \n\t"
2126
"movq %%mm1, 16(%%edi) \n\t"
2127
"subl $4, %%esi \n\t"
2128
"movq %%mm1, 24(%%edi) \n\t"
2129
"subl $32, %%edi \n\t"
2130
"subl $2, %%ecx \n\t"
2131
"jnz .loop2_pass0 \n\t"
2134
: "=c" (dummy_value_c), // output regs (dummy)
2135
"=S" (dummy_value_S),
2136
"=D" (dummy_value_D)
2138
: "1" (sptr), // esi // input regs
2140
"0" (width_mmx) // ecx
2142
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2143
: "%mm0", "%mm1" // clobber list
2148
sptr -= (width_mmx*2 - 2); // sign fixed
2149
dp -= (width_mmx*16 - 2); // sign fixed
2150
for (i = width; i; i--)
2155
png_memcpy(v, sptr, 2);
2156
for (j = 0; j < png_pass_inc[pass]; j++)
2159
png_memcpy(dp, v, 2);
2163
else if (((pass == 2) || (pass == 3)) && width)
2165
int width_mmx = ((width >> 1) << 1) ;
2166
width -= width_mmx; // 0,1 pixels => 0,2 bytes
2169
int dummy_value_c; // fix 'forbidden register spilled'
2173
__asm__ __volatile__ (
2174
"subl $2, %%esi \n\t"
2175
"subl $14, %%edi \n\t"
2177
".loop2_pass2: \n\t"
2178
"movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0
2179
"punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2180
"movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0
2181
"punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0
2182
"punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2
2183
"movq %%mm0, (%%edi) \n\t"
2184
"subl $4, %%esi \n\t"
2185
"movq %%mm1, 8(%%edi) \n\t"
2186
"subl $16, %%edi \n\t"
2187
"subl $2, %%ecx \n\t"
2188
"jnz .loop2_pass2 \n\t"
2191
: "=c" (dummy_value_c), // output regs (dummy)
2192
"=S" (dummy_value_S),
2193
"=D" (dummy_value_D)
2195
: "1" (sptr), // esi // input regs
2197
"0" (width_mmx) // ecx
2199
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2200
: "%mm0", "%mm1" // clobber list
2205
sptr -= (width_mmx*2 - 2); // sign fixed
2206
dp -= (width_mmx*8 - 2); // sign fixed
2207
for (i = width; i; i--)
2212
png_memcpy(v, sptr, 2);
2213
for (j = 0; j < png_pass_inc[pass]; j++)
2216
png_memcpy(dp, v, 2);
2220
else if (width) // pass == 4 or 5
2222
int width_mmx = ((width >> 1) << 1) ;
2223
width -= width_mmx; // 0,1 pixels => 0,2 bytes
2226
int dummy_value_c; // fix 'forbidden register spilled'
2230
__asm__ __volatile__ (
2231
"subl $2, %%esi \n\t"
2232
"subl $6, %%edi \n\t"
2234
".loop2_pass4: \n\t"
2235
"movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0
2236
"punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2237
"subl $4, %%esi \n\t"
2238
"movq %%mm0, (%%edi) \n\t"
2239
"subl $8, %%edi \n\t"
2240
"subl $2, %%ecx \n\t"
2241
"jnz .loop2_pass4 \n\t"
2244
: "=c" (dummy_value_c), // output regs (dummy)
2245
"=S" (dummy_value_S),
2246
"=D" (dummy_value_D)
2248
: "1" (sptr), // esi // input regs
2250
"0" (width_mmx) // ecx
2252
#if 0 /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2253
: "%mm0" // clobber list
2258
sptr -= (width_mmx*2 - 2); // sign fixed
2259
dp -= (width_mmx*4 - 2); // sign fixed
2260
for (i = width; i; i--)
2265
png_memcpy(v, sptr, 2);
2266
for (j = 0; j < png_pass_inc[pass]; j++)
2269
png_memcpy(dp, v, 2);
2273
} /* end of pixel_bytes == 2 */
2275
//--------------------------------------------------------------
2276
else if (pixel_bytes == 4)
2278
if (((pass == 0) || (pass == 1)) && width)
2280
int width_mmx = ((width >> 1) << 1);
2281
width -= width_mmx; // 0,1 pixels => 0,4 bytes
2284
int dummy_value_c; // fix 'forbidden register spilled'
2288
__asm__ __volatile__ (
2289
"subl $4, %%esi \n\t"
2290
"subl $60, %%edi \n\t"
2292
".loop4_pass0: \n\t"
2293
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2294
"movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2295
"punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2296
"punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2297
"movq %%mm0, (%%edi) \n\t"
2298
"movq %%mm0, 8(%%edi) \n\t"
2299
"movq %%mm0, 16(%%edi) \n\t"
2300
"movq %%mm0, 24(%%edi) \n\t"
2301
"movq %%mm1, 32(%%edi) \n\t"
2302
"movq %%mm1, 40(%%edi) \n\t"
2303
"movq %%mm1, 48(%%edi) \n\t"
2304
"subl $8, %%esi \n\t"
2305
"movq %%mm1, 56(%%edi) \n\t"
2306
"subl $64, %%edi \n\t"
2307
"subl $2, %%ecx \n\t"
2308
"jnz .loop4_pass0 \n\t"
2311
: "=c" (dummy_value_c), // output regs (dummy)
2312
"=S" (dummy_value_S),
2313
"=D" (dummy_value_D)
2315
: "1" (sptr), // esi // input regs
2317
"0" (width_mmx) // ecx
2319
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2320
: "%mm0", "%mm1" // clobber list
2325
sptr -= (width_mmx*4 - 4); // sign fixed
2326
dp -= (width_mmx*32 - 4); // sign fixed
2327
for (i = width; i; i--)
2332
png_memcpy(v, sptr, 4);
2333
for (j = 0; j < png_pass_inc[pass]; j++)
2336
png_memcpy(dp, v, 4);
2340
else if (((pass == 2) || (pass == 3)) && width)
2342
int width_mmx = ((width >> 1) << 1);
2343
width -= width_mmx; // 0,1 pixels => 0,4 bytes
2346
int dummy_value_c; // fix 'forbidden register spilled'
2350
__asm__ __volatile__ (
2351
"subl $4, %%esi \n\t"
2352
"subl $28, %%edi \n\t"
2354
".loop4_pass2: \n\t"
2355
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2356
"movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2357
"punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2358
"punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2359
"movq %%mm0, (%%edi) \n\t"
2360
"movq %%mm0, 8(%%edi) \n\t"
2361
"movq %%mm1, 16(%%edi) \n\t"
2362
"movq %%mm1, 24(%%edi) \n\t"
2363
"subl $8, %%esi \n\t"
2364
"subl $32, %%edi \n\t"
2365
"subl $2, %%ecx \n\t"
2366
"jnz .loop4_pass2 \n\t"
2369
: "=c" (dummy_value_c), // output regs (dummy)
2370
"=S" (dummy_value_S),
2371
"=D" (dummy_value_D)
2373
: "1" (sptr), // esi // input regs
2375
"0" (width_mmx) // ecx
2377
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2378
: "%mm0", "%mm1" // clobber list
2383
sptr -= (width_mmx*4 - 4); // sign fixed
2384
dp -= (width_mmx*16 - 4); // sign fixed
2385
for (i = width; i; i--)
2390
png_memcpy(v, sptr, 4);
2391
for (j = 0; j < png_pass_inc[pass]; j++)
2394
png_memcpy(dp, v, 4);
2398
else if (width) // pass == 4 or 5
2400
int width_mmx = ((width >> 1) << 1) ;
2401
width -= width_mmx; // 0,1 pixels => 0,4 bytes
2404
int dummy_value_c; // fix 'forbidden register spilled'
2408
__asm__ __volatile__ (
2409
"subl $4, %%esi \n\t"
2410
"subl $12, %%edi \n\t"
2412
".loop4_pass4: \n\t"
2413
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2414
"movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2415
"punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2416
"punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2417
"movq %%mm0, (%%edi) \n\t"
2418
"subl $8, %%esi \n\t"
2419
"movq %%mm1, 8(%%edi) \n\t"
2420
"subl $16, %%edi \n\t"
2421
"subl $2, %%ecx \n\t"
2422
"jnz .loop4_pass4 \n\t"
2425
: "=c" (dummy_value_c), // output regs (dummy)
2426
"=S" (dummy_value_S),
2427
"=D" (dummy_value_D)
2429
: "1" (sptr), // esi // input regs
2431
"0" (width_mmx) // ecx
2433
#if 0 /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2434
: "%mm0", "%mm1" // clobber list
2439
sptr -= (width_mmx*4 - 4); // sign fixed
2440
dp -= (width_mmx*8 - 4); // sign fixed
2441
for (i = width; i; i--)
2446
png_memcpy(v, sptr, 4);
2447
for (j = 0; j < png_pass_inc[pass]; j++)
2450
png_memcpy(dp, v, 4);
2454
} /* end of pixel_bytes == 4 */
2456
//--------------------------------------------------------------
2457
else if (pixel_bytes == 8)
2459
// GRR TEST: should work, but needs testing (special 64-bit version of rpng2?)
2460
// GRR NOTE: no need to combine passes here!
2461
if (((pass == 0) || (pass == 1)) && width)
2463
int dummy_value_c; // fix 'forbidden register spilled'
2467
// source is 8-byte RRGGBBAA
2468
// dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ...
2469
__asm__ __volatile__ (
2470
"subl $56, %%edi \n\t" // start of last block
2472
".loop8_pass0: \n\t"
2473
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2474
"movq %%mm0, (%%edi) \n\t"
2475
"movq %%mm0, 8(%%edi) \n\t"
2476
"movq %%mm0, 16(%%edi) \n\t"
2477
"movq %%mm0, 24(%%edi) \n\t"
2478
"movq %%mm0, 32(%%edi) \n\t"
2479
"movq %%mm0, 40(%%edi) \n\t"
2480
"movq %%mm0, 48(%%edi) \n\t"
2481
"subl $8, %%esi \n\t"
2482
"movq %%mm0, 56(%%edi) \n\t"
2483
"subl $64, %%edi \n\t"
2485
"jnz .loop8_pass0 \n\t"
2488
: "=c" (dummy_value_c), // output regs (dummy)
2489
"=S" (dummy_value_S),
2490
"=D" (dummy_value_D)
2492
: "1" (sptr), // esi // input regs
2496
#if 0 /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2497
: "%mm0" // clobber list
2501
else if (((pass == 2) || (pass == 3)) && width)
2503
// source is 8-byte RRGGBBAA
2504
// dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA
2505
// (recall that expansion is _in place_: sptr and dp
2506
// both point at locations within same row buffer)
2508
int dummy_value_c; // fix 'forbidden register spilled'
2512
__asm__ __volatile__ (
2513
"subl $24, %%edi \n\t" // start of last block
2515
".loop8_pass2: \n\t"
2516
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2517
"movq %%mm0, (%%edi) \n\t"
2518
"movq %%mm0, 8(%%edi) \n\t"
2519
"movq %%mm0, 16(%%edi) \n\t"
2520
"subl $8, %%esi \n\t"
2521
"movq %%mm0, 24(%%edi) \n\t"
2522
"subl $32, %%edi \n\t"
2524
"jnz .loop8_pass2 \n\t"
2527
: "=c" (dummy_value_c), // output regs (dummy)
2528
"=S" (dummy_value_S),
2529
"=D" (dummy_value_D)
2531
: "1" (sptr), // esi // input regs
2535
#if 0 /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2536
: "%mm0" // clobber list
2541
else if (width) // pass == 4 or 5
2543
// source is 8-byte RRGGBBAA
2544
// dest is 16-byte RRGGBBAA RRGGBBAA
2546
int dummy_value_c; // fix 'forbidden register spilled'
2550
__asm__ __volatile__ (
2551
"subl $8, %%edi \n\t" // start of last block
2553
".loop8_pass4: \n\t"
2554
"movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2555
"movq %%mm0, (%%edi) \n\t"
2556
"subl $8, %%esi \n\t"
2557
"movq %%mm0, 8(%%edi) \n\t"
2558
"subl $16, %%edi \n\t"
2560
"jnz .loop8_pass4 \n\t"
2563
: "=c" (dummy_value_c), // output regs (dummy)
2564
"=S" (dummy_value_S),
2565
"=D" (dummy_value_D)
2567
: "1" (sptr), // esi // input regs
2571
#if 0 /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2572
: "%mm0" // clobber list
2578
} /* end of pixel_bytes == 8 */
2580
//--------------------------------------------------------------
2581
else if (pixel_bytes == 6)
2583
for (i = width; i; i--)
2587
png_memcpy(v, sptr, 6);
2588
for (j = 0; j < png_pass_inc[pass]; j++)
2590
png_memcpy(dp, v, 6);
2595
} /* end of pixel_bytes == 6 */
2597
//--------------------------------------------------------------
2600
for (i = width; i; i--)
2604
png_memcpy(v, sptr, pixel_bytes);
2605
for (j = 0; j < png_pass_inc[pass]; j++)
2607
png_memcpy(dp, v, pixel_bytes);
2613
} // end of _mmx_supported ========================================
2615
else /* MMX not supported: use modified C code - takes advantage
2616
* of inlining of png_memcpy for a constant */
2617
/* GRR 19991007: does it? or should pixel_bytes in each
2618
* block be replaced with immediate value (e.g., 1)? */
2619
/* GRR 19991017: replaced with constants in each case */
2620
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
2622
if (pixel_bytes == 1)
2624
for (i = width; i; i--)
2627
for (j = 0; j < png_pass_inc[pass]; j++)
2634
else if (pixel_bytes == 3)
2636
for (i = width; i; i--)
2640
png_memcpy(v, sptr, 3);
2641
for (j = 0; j < png_pass_inc[pass]; j++)
2643
png_memcpy(dp, v, 3);
2649
else if (pixel_bytes == 2)
2651
for (i = width; i; i--)
2655
png_memcpy(v, sptr, 2);
2656
for (j = 0; j < png_pass_inc[pass]; j++)
2658
png_memcpy(dp, v, 2);
2664
else if (pixel_bytes == 4)
2666
for (i = width; i; i--)
2670
png_memcpy(v, sptr, 4);
2671
for (j = 0; j < png_pass_inc[pass]; j++)
2674
if (dp < row || dp+3 > row+png_ptr->row_buf_size)
2676
printf("dp out of bounds: row=%d, dp=%d, rp=%d\n",
2677
row, dp, row+png_ptr->row_buf_size);
2678
printf("row_buf=%d\n",png_ptr->row_buf_size);
2681
png_memcpy(dp, v, 4);
2687
else if (pixel_bytes == 6)
2689
for (i = width; i; i--)
2693
png_memcpy(v, sptr, 6);
2694
for (j = 0; j < png_pass_inc[pass]; j++)
2696
png_memcpy(dp, v, 6);
2702
else if (pixel_bytes == 8)
2704
for (i = width; i; i--)
2708
png_memcpy(v, sptr, 8);
2709
for (j = 0; j < png_pass_inc[pass]; j++)
2711
png_memcpy(dp, v, 8);
2717
else /* GRR: should never be reached */
2719
for (i = width; i; i--)
2723
png_memcpy(v, sptr, pixel_bytes);
2724
for (j = 0; j < png_pass_inc[pass]; j++)
2726
png_memcpy(dp, v, pixel_bytes);
2729
sptr -= pixel_bytes;
2733
} /* end if (MMX not supported) */
2736
} /* end switch (row_info->pixel_depth) */
2738
row_info->width = final_width;
2740
row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth,final_width);
2743
} /* end png_do_read_interlace() */
2745
#endif /* PNG_HAVE_ASSEMBLER_READ_INTERLACE */
2746
#endif /* PNG_READ_INTERLACING_SUPPORTED */
2750
#if defined(PNG_HAVE_ASSEMBLER_READ_FILTER_ROW)
2751
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
2753
// These variables are utilized in the functions below. They are declared
2754
// globally here to ensure alignment on 8-byte boundaries.
2759
} _LBCarryMask = {0x0101010101010101LL},
2760
_HBClearMask = {0x7f7f7f7f7f7f7f7fLL},
2761
_ActiveMask, _ActiveMask2, _ActiveMaskEnd, _ShiftBpp, _ShiftRem;
2763
#ifdef PNG_THREAD_UNSAFE_OK
2764
//===========================================================================//
2766
// P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G //
2768
//===========================================================================//
2770
// Optimized code for PNG Average filter decoder
2772
static void /* PRIVATE */
2773
png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row,
2777
int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error
2781
bpp = (row_info->pixel_depth + 7) >> 3; // get # bytes per pixel
2782
_FullLength = row_info->rowbytes; // # of bytes to filter
2784
__asm__ __volatile__ (
2785
// initialize address pointers and offset
2787
"pushl %%ebx \n\t" // save index to Global Offset Table
2789
//pre "movl row, %%edi \n\t" // edi: Avg(x)
2790
"xorl %%ebx, %%ebx \n\t" // ebx: x
2791
"movl %%edi, %%edx \n\t"
2792
//pre "movl prev_row, %%esi \n\t" // esi: Prior(x)
2793
//pre "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx)
2794
"subl %%ecx, %%edx \n\t" // edx: Raw(x-bpp)
2796
"xorl %%eax,%%eax \n\t"
2798
// Compute the Raw value for the first bpp bytes
2799
// Raw(x) = Avg(x) + (Prior(x)/2)
2801
"movb (%%esi,%%ebx,),%%al \n\t" // load al with Prior(x)
2803
"shrb %%al \n\t" // divide by 2
2804
"addb -1(%%edi,%%ebx,),%%al \n\t" // add Avg(x); -1 to offset inc ebx
2805
//pre "cmpl bpp, %%ebx \n\t" // (bpp is preloaded into ecx)
2806
"cmpl %%ecx, %%ebx \n\t"
2807
"movb %%al,-1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx
2808
"jb avg_rlp \n\t" // mov does not affect flags
2810
// get # of bytes to alignment
2811
"movl %%edi, _dif \n\t" // take start of row
2812
"addl %%ebx, _dif \n\t" // add bpp
2813
"addl $0xf, _dif \n\t" // add 7+8 to incr past alignment bdry
2814
"andl $0xfffffff8, _dif \n\t" // mask to alignment boundary
2815
"subl %%edi, _dif \n\t" // subtract from start => value ebx at
2816
"jz avg_go \n\t" // alignment
2819
// Compute the Raw value for the bytes up to the alignment boundary
2820
// Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
2821
"xorl %%ecx, %%ecx \n\t"
2824
"xorl %%eax, %%eax \n\t"
2825
"movb (%%esi,%%ebx,), %%cl \n\t" // load cl with Prior(x)
2826
"movb (%%edx,%%ebx,), %%al \n\t" // load al with Raw(x-bpp)
2827
"addw %%cx, %%ax \n\t"
2829
"shrw %%ax \n\t" // divide by 2
2830
"addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
2831
"cmpl _dif, %%ebx \n\t" // check if at alignment boundary
2832
"movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx
2833
"jb avg_lp1 \n\t" // repeat until at alignment boundary
2836
"movl _FullLength, %%eax \n\t"
2837
"movl %%eax, %%ecx \n\t"
2838
"subl %%ebx, %%eax \n\t" // subtract alignment fix
2839
"andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8
2840
"subl %%eax, %%ecx \n\t" // drop over bytes from original length
2841
"movl %%ecx, _MMXLength \n\t"
2843
"popl %%ebx \n\t" // restore index to Global Offset Table
2846
: "=c" (dummy_value_c), // output regs (dummy)
2847
"=S" (dummy_value_S),
2848
"=D" (dummy_value_D)
2850
: "0" (bpp), // ecx // input regs
2851
"1" (prev_row), // esi
2854
: "%eax", "%edx" // clobber list
2858
// GRR: INCLUDE "memory" as clobbered? (_dif, _MMXLength)
2859
// (seems to work fine without...)
2862
// now do the math for the rest of the row
2867
_ActiveMask.use = 0x0000000000ffffffLL;
2868
_ShiftBpp.use = 24; // == 3 * 8
2869
_ShiftRem.use = 40; // == 64 - 24
2871
__asm__ __volatile__ (
2872
// re-init address pointers and offset
2873
"movq _ActiveMask, %%mm7 \n\t"
2874
"movl _dif, %%ecx \n\t" // ecx: x = offset to
2875
"movq _LBCarryMask, %%mm5 \n\t" // alignment boundary
2876
// preload "movl row, %%edi \n\t" // edi: Avg(x)
2877
"movq _HBClearMask, %%mm4 \n\t"
2878
// preload "movl prev_row, %%esi \n\t" // esi: Prior(x)
2880
// prime the pump: load the first Raw(x-bpp) data set
2881
"movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
2882
// (correct pos. in loop below)
2884
"movq (%%edi,%%ecx,), %%mm0 \n\t" // load mm0 with Avg(x)
2885
"movq %%mm5, %%mm3 \n\t"
2886
"psrlq _ShiftRem, %%mm2 \n\t" // correct position Raw(x-bpp)
2888
"movq (%%esi,%%ecx,), %%mm1 \n\t" // load mm1 with Prior(x)
2889
"movq %%mm7, %%mm6 \n\t"
2890
"pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
2891
"psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
2892
"pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
2894
"paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
2896
// add 1st active group (Raw(x-bpp)/2) to average with LBCarry
2897
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
2899
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
2901
// lsb's were == 1 (only valid for active group)
2902
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
2903
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
2905
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2907
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 1
2908
// bytes to add to Avg
2909
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
2910
// Avg for each Active
2912
// add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
2913
"psllq _ShiftBpp, %%mm6 \n\t" // shift the mm6 mask to cover
2915
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
2916
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
2917
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
2919
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
2921
// lsb's were == 1 (only valid for active group)
2922
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
2923
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
2925
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2927
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
2928
// bytes to add to Avg
2929
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
2930
// Avg for each Active
2933
// add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
2934
"psllq _ShiftBpp, %%mm6 \n\t" // shift mm6 mask to cover last
2937
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
2938
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
2939
// Data only needs to be shifted once here to
2940
// get the correct x-bpp offset.
2941
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
2943
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
2945
// lsb's were == 1 (only valid for active group)
2946
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
2947
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
2949
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2951
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
2952
// bytes to add to Avg
2953
"addl $8, %%ecx \n\t"
2954
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
2955
// Avg for each Active
2957
// now ready to write back to memory
2958
"movq %%mm0, -8(%%edi,%%ecx,) \n\t"
2959
// move updated Raw(x) to use as Raw(x-bpp) for next loop
2960
"cmpl _MMXLength, %%ecx \n\t"
2961
"movq %%mm0, %%mm2 \n\t" // mov updated Raw(x) to mm2
2964
: "=S" (dummy_value_S), // output regs (dummy)
2965
"=D" (dummy_value_D)
2967
: "0" (prev_row), // esi // input regs
2970
: "%ecx" // clobber list
2971
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
2972
, "%mm0", "%mm1", "%mm2", "%mm3"
2973
, "%mm4", "%mm5", "%mm6", "%mm7"
2981
//case 7: // who wrote this? PNG doesn't support 5 or 7 bytes/pixel
2982
//case 5: // GRR BOGUS
2984
_ActiveMask.use = 0xffffffffffffffffLL; // use shift below to clear
2985
// appropriate inactive bytes
2986
_ShiftBpp.use = bpp << 3;
2987
_ShiftRem.use = 64 - _ShiftBpp.use;
2989
__asm__ __volatile__ (
2990
"movq _HBClearMask, %%mm4 \n\t"
2992
// re-init address pointers and offset
2993
"movl _dif, %%ecx \n\t" // ecx: x = offset to
2994
// alignment boundary
2996
// load _ActiveMask and clear all bytes except for 1st active group
2997
"movq _ActiveMask, %%mm7 \n\t"
2998
// preload "movl row, %%edi \n\t" // edi: Avg(x)
2999
"psrlq _ShiftRem, %%mm7 \n\t"
3000
// preload "movl prev_row, %%esi \n\t" // esi: Prior(x)
3001
"movq %%mm7, %%mm6 \n\t"
3002
"movq _LBCarryMask, %%mm5 \n\t"
3003
"psllq _ShiftBpp, %%mm6 \n\t" // create mask for 2nd active
3006
// prime the pump: load the first Raw(x-bpp) data set
3007
"movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3008
// (we correct pos. in loop below)
3010
"movq (%%edi,%%ecx,), %%mm0 \n\t"
3011
"psrlq _ShiftRem, %%mm2 \n\t" // shift data to pos. correctly
3012
"movq (%%esi,%%ecx,), %%mm1 \n\t"
3013
// add (Prev_row/2) to average
3014
"movq %%mm5, %%mm3 \n\t"
3015
"pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3016
"psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3017
"pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3019
"paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3021
// add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3022
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3024
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3026
// lsb's were == 1 (only valid for active group)
3027
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3028
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3030
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3032
"pand %%mm7, %%mm2 \n\t" // leave only Active Group 1
3033
// bytes to add to Avg
3034
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg
3037
// add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3038
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3039
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
3040
"addl $8, %%ecx \n\t"
3041
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3043
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3045
// lsb's were == 1 (only valid for active group)
3046
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3047
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3049
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3051
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3052
// bytes to add to Avg
3053
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3054
// Avg for each Active
3056
"cmpl _MMXLength, %%ecx \n\t"
3057
// now ready to write back to memory
3058
"movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3059
// prep Raw(x-bpp) for next loop
3060
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3063
: "=S" (dummy_value_S), // output regs (dummy)
3064
"=D" (dummy_value_D)
3066
: "0" (prev_row), // esi // input regs
3069
: "%ecx" // clobber list
3070
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3071
, "%mm0", "%mm1", "%mm2", "%mm3"
3072
, "%mm4", "%mm5", "%mm6", "%mm7"
3076
break; // end 4,6 bpp
3080
_ActiveMask.use = 0x000000000000ffffLL;
3081
_ShiftBpp.use = 16; // == 2 * 8
3082
_ShiftRem.use = 48; // == 64 - 16
3084
__asm__ __volatile__ (
3086
"movq _ActiveMask, %%mm7 \n\t"
3087
// re-init address pointers and offset
3088
"movl _dif, %%ecx \n\t" // ecx: x = offset to alignment
3090
"movq _LBCarryMask, %%mm5 \n\t"
3091
// preload "movl row, %%edi \n\t" // edi: Avg(x)
3092
"movq _HBClearMask, %%mm4 \n\t"
3093
// preload "movl prev_row, %%esi \n\t" // esi: Prior(x)
3095
// prime the pump: load the first Raw(x-bpp) data set
3096
"movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3097
// (we correct pos. in loop below)
3099
"movq (%%edi,%%ecx,), %%mm0 \n\t"
3100
"psrlq _ShiftRem, %%mm2 \n\t" // shift data to pos. correctly
3101
"movq (%%esi,%%ecx,), %%mm1 \n\t" // (GRR BUGFIX: was psllq)
3102
// add (Prev_row/2) to average
3103
"movq %%mm5, %%mm3 \n\t"
3104
"pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3105
"psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3106
"pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3108
"movq %%mm7, %%mm6 \n\t"
3109
"paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3112
// add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3113
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3115
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3117
// lsb's were == 1 (only valid
3118
// for active group)
3119
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3120
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3122
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3124
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 1
3125
// bytes to add to Avg
3126
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg
3127
// for each Active byte
3129
// add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3130
"psllq _ShiftBpp, %%mm6 \n\t" // shift the mm6 mask to cover
3132
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3133
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
3134
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3136
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3138
// lsb's were == 1 (only valid
3139
// for active group)
3140
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3141
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3143
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3145
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3146
// bytes to add to Avg
3147
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3148
// Avg for each Active byte
3150
// add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
3151
"psllq _ShiftBpp, %%mm6 \n\t" // shift the mm6 mask to cover
3153
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3154
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
3155
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3157
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3158
// where both lsb's were == 1
3159
// (only valid for active group)
3160
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3161
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3163
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3165
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3166
// bytes to add to Avg
3167
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3168
// Avg for each Active byte
3170
// add 4th active group (Raw(x-bpp)/2) to average with _LBCarry
3171
"psllq _ShiftBpp, %%mm6 \n\t" // shift the mm6 mask to cover
3173
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3174
"psllq _ShiftBpp, %%mm2 \n\t" // shift data to pos. correctly
3175
"addl $8, %%ecx \n\t"
3176
"movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3178
"pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3180
// lsb's were == 1 (only valid
3181
// for active group)
3182
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3183
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3185
"paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3187
"pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3188
// bytes to add to Avg
3189
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3190
// Avg for each Active byte
3192
"cmpl _MMXLength, %%ecx \n\t"
3193
// now ready to write back to memory
3194
"movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3195
// prep Raw(x-bpp) for next loop
3196
"movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3199
: "=S" (dummy_value_S), // output regs (dummy)
3200
"=D" (dummy_value_D)
3202
: "0" (prev_row), // esi // input regs
3205
: "%ecx" // clobber list
3206
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3207
, "%mm0", "%mm1", "%mm2", "%mm3"
3208
, "%mm4", "%mm5", "%mm6", "%mm7"
3216
__asm__ __volatile__ (
3217
// re-init address pointers and offset
3219
"pushl %%ebx \n\t" // save Global Offset Table index
3221
"movl _dif, %%ebx \n\t" // ebx: x = offset to alignment
3223
// preload "movl row, %%edi \n\t" // edi: Avg(x)
3224
"cmpl _FullLength, %%ebx \n\t" // test if offset at end of array
3226
// do Paeth decode for remaining bytes
3227
// preload "movl prev_row, %%esi \n\t" // esi: Prior(x)
3228
"movl %%edi, %%edx \n\t"
3229
// preload "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx)
3230
"subl %%ecx, %%edx \n\t" // edx: Raw(x-bpp)
3231
"xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx
3234
// Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3235
"xorl %%eax, %%eax \n\t"
3236
"movb (%%esi,%%ebx,), %%cl \n\t" // load cl with Prior(x)
3237
"movb (%%edx,%%ebx,), %%al \n\t" // load al with Raw(x-bpp)
3238
"addw %%cx, %%ax \n\t"
3240
"shrw %%ax \n\t" // divide by 2
3241
"addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset
3243
"cmpl _FullLength, %%ebx \n\t" // check if at end of array
3244
"movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x);
3245
// mov does not affect flags; -1 to offset inc ebx
3250
"popl %%ebx \n\t" // Global Offset Table index
3253
: "=c" (dummy_value_c), // output regs (dummy)
3254
"=S" (dummy_value_S),
3255
"=D" (dummy_value_D)
3257
: "0" (bpp), // ecx // input regs
3258
"1" (prev_row), // esi
3261
: "%eax", "%edx" // clobber list
3267
return; // end 1 bpp
3271
__asm__ __volatile__ (
3272
// re-init address pointers and offset
3273
"movl _dif, %%ecx \n\t" // ecx: x == offset to alignment
3274
"movq _LBCarryMask, %%mm5 \n\t" // boundary
3275
// preload "movl row, %%edi \n\t" // edi: Avg(x)
3276
"movq _HBClearMask, %%mm4 \n\t"
3277
// preload "movl prev_row, %%esi \n\t" // esi: Prior(x)
3279
// prime the pump: load the first Raw(x-bpp) data set
3280
"movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3281
// (NO NEED to correct pos. in loop below)
3284
"movq (%%edi,%%ecx,), %%mm0 \n\t"
3285
"movq %%mm5, %%mm3 \n\t"
3286
"movq (%%esi,%%ecx,), %%mm1 \n\t"
3287
"addl $8, %%ecx \n\t"
3288
"pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3289
"psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3290
"pand %%mm2, %%mm3 \n\t" // get LBCarrys for each byte
3291
// where both lsb's were == 1
3292
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3293
"pand %%mm4, %%mm1 \n\t" // clear invalid bit 7, each byte
3294
"paddb %%mm3, %%mm0 \n\t" // add LBCarrys to Avg, each byte
3295
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7, each byte
3296
"paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg, each
3297
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) to Avg for each
3298
"cmpl _MMXLength, %%ecx \n\t"
3299
"movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3300
"movq %%mm0, %%mm2 \n\t" // reuse as Raw(x-bpp)
3303
: "=S" (dummy_value_S), // output regs (dummy)
3304
"=D" (dummy_value_D)
3306
: "0" (prev_row), // esi // input regs
3309
: "%ecx" // clobber list
3310
#if 0 /* %mm0, ..., %mm5 not supported by gcc 2.7.2.3 or egcs 1.1 */
3311
, "%mm0", "%mm1", "%mm2"
3312
, "%mm3", "%mm4", "%mm5"
3318
default: // bpp greater than 8 (!= 1,2,3,4,[5],6,[7],8)
3322
// GRR: PRINT ERROR HERE: SHOULD NEVER BE REACHED
3324
"Internal logic error in pnggccrd (png_read_filter_row_mmx_avg())\n");
3328
__asm__ __volatile__ (
3329
"movq _LBCarryMask, %%mm5 \n\t"
3330
// re-init address pointers and offset
3331
"movl _dif, %%ebx \n\t" // ebx: x = offset to
3332
// alignment boundary
3333
"movl row, %%edi \n\t" // edi: Avg(x)
3334
"movq _HBClearMask, %%mm4 \n\t"
3335
"movl %%edi, %%edx \n\t"
3336
"movl prev_row, %%esi \n\t" // esi: Prior(x)
3337
"subl bpp, %%edx \n\t" // edx: Raw(x-bpp)
3339
"movq (%%edi,%%ebx,), %%mm0 \n\t"
3340
"movq %%mm5, %%mm3 \n\t"
3341
"movq (%%esi,%%ebx,), %%mm1 \n\t"
3342
"pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3343
"movq (%%edx,%%ebx,), %%mm2 \n\t"
3344
"psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3345
"pand %%mm2, %%mm3 \n\t" // get LBCarrys for each byte
3346
// where both lsb's were == 1
3347
"psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3348
"pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3350
"paddb %%mm3, %%mm0 \n\t" // add LBCarrys to Avg for each
3352
"pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3354
"paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3356
"addl $8, %%ebx \n\t"
3357
"paddb %%mm2, %%mm0 \n\t" // add (Raw/2) to Avg for each
3359
"cmpl _MMXLength, %%ebx \n\t"
3360
"movq %%mm0, -8(%%edi,%%ebx,) \n\t"
3363
: // FIXASM: output regs/vars go here, e.g.: "=m" (memory_var)
3365
: // FIXASM: input regs, e.g.: "c" (count), "S" (src), "D" (dest)
3367
: "%ebx", "%edx", "%edi", "%esi" // CHECKASM: clobber list
3369
#endif /* 0 - NEVER REACHED */
3373
} // end switch (bpp)
3375
__asm__ __volatile__ (
3376
// MMX acceleration complete; now do clean-up
3377
// check if any remaining bytes left to decode
3379
"pushl %%ebx \n\t" // save index to Global Offset Table
3381
"movl _MMXLength, %%ebx \n\t" // ebx: x == offset bytes after MMX
3382
//pre "movl row, %%edi \n\t" // edi: Avg(x)
3383
"cmpl _FullLength, %%ebx \n\t" // test if offset at end of array
3386
// do Avg decode for remaining bytes
3387
//pre "movl prev_row, %%esi \n\t" // esi: Prior(x)
3388
"movl %%edi, %%edx \n\t"
3389
//pre "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx)
3390
"subl %%ecx, %%edx \n\t" // edx: Raw(x-bpp)
3391
"xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below
3394
// Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3395
"xorl %%eax, %%eax \n\t"
3396
"movb (%%esi,%%ebx,), %%cl \n\t" // load cl with Prior(x)
3397
"movb (%%edx,%%ebx,), %%al \n\t" // load al with Raw(x-bpp)
3398
"addw %%cx, %%ax \n\t"
3400
"shrw %%ax \n\t" // divide by 2
3401
"addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
3402
"cmpl _FullLength, %%ebx \n\t" // check if at end of array
3403
"movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x) [mov does not
3404
"jb avg_lp2 \n\t" // affect flags; -1 to offset inc ebx]
3407
"EMMS \n\t" // end MMX; prep for poss. FP instrs.
3409
"popl %%ebx \n\t" // restore index to Global Offset Table
3412
: "=c" (dummy_value_c), // output regs (dummy)
3413
"=S" (dummy_value_S),
3414
"=D" (dummy_value_D)
3416
: "0" (bpp), // ecx // input regs
3417
"1" (prev_row), // esi
3420
: "%eax", "%edx" // clobber list
3426
} /* end png_read_filter_row_mmx_avg() */
3431
#ifdef PNG_THREAD_UNSAFE_OK
3432
//===========================================================================//
3434
// P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H //
3436
//===========================================================================//
3438
// Optimized code for PNG Paeth filter decoder
3440
static void /* PRIVATE */
3441
png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row,
3445
int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error
3449
bpp = (row_info->pixel_depth + 7) >> 3; // Get # bytes per pixel
3450
_FullLength = row_info->rowbytes; // # of bytes to filter
3452
__asm__ __volatile__ (
3454
"pushl %%ebx \n\t" // save index to Global Offset Table
3456
"xorl %%ebx, %%ebx \n\t" // ebx: x offset
3457
//pre "movl row, %%edi \n\t"
3458
"xorl %%edx, %%edx \n\t" // edx: x-bpp offset
3459
//pre "movl prev_row, %%esi \n\t"
3460
"xorl %%eax, %%eax \n\t"
3462
// Compute the Raw value for the first bpp bytes
3463
// Note: the formula works out to be always
3464
// Paeth(x) = Raw(x) + Prior(x) where x < bpp
3466
"movb (%%edi,%%ebx,), %%al \n\t"
3467
"addb (%%esi,%%ebx,), %%al \n\t"
3469
//pre "cmpl bpp, %%ebx \n\t" (bpp is preloaded into ecx)
3470
"cmpl %%ecx, %%ebx \n\t"
3471
"movb %%al, -1(%%edi,%%ebx,) \n\t"
3473
// get # of bytes to alignment
3474
"movl %%edi, _dif \n\t" // take start of row
3475
"addl %%ebx, _dif \n\t" // add bpp
3476
"xorl %%ecx, %%ecx \n\t"
3477
"addl $0xf, _dif \n\t" // add 7 + 8 to incr past alignment
3479
"andl $0xfffffff8, _dif \n\t" // mask to alignment boundary
3480
"subl %%edi, _dif \n\t" // subtract from start ==> value ebx
3486
"xorl %%eax, %%eax \n\t"
3487
// pav = p - a = (a + b - c) - a = b - c
3488
"movb (%%esi,%%ebx,), %%al \n\t" // load Prior(x) into al
3489
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
3490
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
3491
"movl %%eax, _patemp \n\t" // Save pav for later use
3492
"xorl %%eax, %%eax \n\t"
3493
// pbv = p - b = (a + b - c) - b = a - c
3494
"movb (%%edi,%%edx,), %%al \n\t" // load Raw(x-bpp) into al
3495
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
3496
"movl %%eax, %%ecx \n\t"
3497
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3498
"addl _patemp, %%eax \n\t" // pcv = pav + pbv
3500
"testl $0x80000000, %%eax \n\t"
3502
"negl %%eax \n\t" // reverse sign of neg values
3505
"movl %%eax, _pctemp \n\t" // save pc for later use
3507
"testl $0x80000000, %%ecx \n\t"
3509
"negl %%ecx \n\t" // reverse sign of neg values
3512
"movl %%ecx, _pbtemp \n\t" // save pb for later use
3514
"movl _patemp, %%eax \n\t"
3515
"testl $0x80000000, %%eax \n\t"
3517
"negl %%eax \n\t" // reverse sign of neg values
3520
"movl %%eax, _patemp \n\t" // save pa for later use
3522
"cmpl %%ecx, %%eax \n\t"
3523
"jna paeth_abb \n\t"
3524
// pa > pb; now test if pb <= pc
3525
"cmpl _pctemp, %%ecx \n\t"
3526
"jna paeth_bbc \n\t"
3527
// pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3528
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
3529
"jmp paeth_paeth \n\t"
3532
// pb <= pc; Raw(x) = Paeth(x) + Prior(x)
3533
"movb (%%esi,%%ebx,), %%cl \n\t" // load Prior(x) into cl
3534
"jmp paeth_paeth \n\t"
3537
// pa <= pb; now test if pa <= pc
3538
"cmpl _pctemp, %%eax \n\t"
3539
"jna paeth_abc \n\t"
3540
// pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3541
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
3542
"jmp paeth_paeth \n\t"
3545
// pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
3546
"movb (%%edi,%%edx,), %%cl \n\t" // load Raw(x-bpp) into cl
3551
// Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
3552
"addb %%cl, -1(%%edi,%%ebx,) \n\t"
3553
"cmpl _dif, %%ebx \n\t"
3557
"movl _FullLength, %%ecx \n\t"
3558
"movl %%ecx, %%eax \n\t"
3559
"subl %%ebx, %%eax \n\t" // subtract alignment fix
3560
"andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8
3561
"subl %%eax, %%ecx \n\t" // drop over bytes from original length
3562
"movl %%ecx, _MMXLength \n\t"
3564
"popl %%ebx \n\t" // restore index to Global Offset Table
3567
: "=c" (dummy_value_c), // output regs (dummy)
3568
"=S" (dummy_value_S),
3569
"=D" (dummy_value_D)
3571
: "0" (bpp), // ecx // input regs
3572
"1" (prev_row), // esi
3575
: "%eax", "%edx" // clobber list
3581
// now do the math for the rest of the row
3586
_ActiveMask.use = 0x0000000000ffffffLL;
3587
_ActiveMaskEnd.use = 0xffff000000000000LL;
3588
_ShiftBpp.use = 24; // == bpp(3) * 8
3589
_ShiftRem.use = 40; // == 64 - 24
3591
__asm__ __volatile__ (
3592
"movl _dif, %%ecx \n\t"
3593
// preload "movl row, %%edi \n\t"
3594
// preload "movl prev_row, %%esi \n\t"
3595
"pxor %%mm0, %%mm0 \n\t"
3596
// prime the pump: load the first Raw(x-bpp) data set
3597
"movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3599
"psrlq _ShiftRem, %%mm1 \n\t" // shift last 3 bytes to 1st
3601
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
3602
"punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
3603
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes
3604
"punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
3605
"psrlq _ShiftRem, %%mm3 \n\t" // shift last 3 bytes to 1st
3607
// pav = p - a = (a + b - c) - a = b - c
3608
"movq %%mm2, %%mm4 \n\t"
3609
"punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
3610
// pbv = p - b = (a + b - c) - b = a - c
3611
"movq %%mm1, %%mm5 \n\t"
3612
"psubw %%mm3, %%mm4 \n\t"
3613
"pxor %%mm7, %%mm7 \n\t"
3614
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3615
"movq %%mm4, %%mm6 \n\t"
3616
"psubw %%mm3, %%mm5 \n\t"
3618
// pa = abs(p-a) = abs(pav)
3619
// pb = abs(p-b) = abs(pbv)
3620
// pc = abs(p-c) = abs(pcv)
3621
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
3622
"paddw %%mm5, %%mm6 \n\t"
3623
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
3624
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
3625
"psubw %%mm0, %%mm4 \n\t"
3626
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
3627
"psubw %%mm0, %%mm4 \n\t"
3628
"psubw %%mm7, %%mm5 \n\t"
3629
"pxor %%mm0, %%mm0 \n\t"
3630
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
3631
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
3632
"psubw %%mm7, %%mm5 \n\t"
3633
"psubw %%mm0, %%mm6 \n\t"
3635
"movq %%mm4, %%mm7 \n\t"
3636
"psubw %%mm0, %%mm6 \n\t"
3637
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
3638
"movq %%mm7, %%mm0 \n\t"
3639
// use mm7 mask to merge pa & pb
3640
"pand %%mm7, %%mm5 \n\t"
3641
// use mm0 mask copy to merge a & b
3642
"pand %%mm0, %%mm2 \n\t"
3643
"pandn %%mm4, %%mm7 \n\t"
3644
"pandn %%mm1, %%mm0 \n\t"
3645
"paddw %%mm5, %%mm7 \n\t"
3646
"paddw %%mm2, %%mm0 \n\t"
3647
// test ((pa <= pb)? pa:pb) <= pc
3648
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
3649
"pxor %%mm1, %%mm1 \n\t"
3650
"pand %%mm7, %%mm3 \n\t"
3651
"pandn %%mm0, %%mm7 \n\t"
3652
"paddw %%mm3, %%mm7 \n\t"
3653
"pxor %%mm0, %%mm0 \n\t"
3654
"packuswb %%mm1, %%mm7 \n\t"
3655
"movq (%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
3656
"pand _ActiveMask, %%mm7 \n\t"
3657
"movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1
3658
"paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3659
"punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
3660
"movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value
3661
"movq %%mm7, %%mm1 \n\t" // now mm1 will be used as
3663
// now do Paeth for 2nd set of bytes (3-5)
3664
"psrlq _ShiftBpp, %%mm2 \n\t" // load b=Prior(x) step 2
3665
"punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
3666
"pxor %%mm7, %%mm7 \n\t"
3667
"punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
3668
// pbv = p - b = (a + b - c) - b = a - c
3669
"movq %%mm1, %%mm5 \n\t"
3670
// pav = p - a = (a + b - c) - a = b - c
3671
"movq %%mm2, %%mm4 \n\t"
3672
"psubw %%mm3, %%mm5 \n\t"
3673
"psubw %%mm3, %%mm4 \n\t"
3674
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) =
3675
// pav + pbv = pbv + pav
3676
"movq %%mm5, %%mm6 \n\t"
3677
"paddw %%mm4, %%mm6 \n\t"
3679
// pa = abs(p-a) = abs(pav)
3680
// pb = abs(p-b) = abs(pbv)
3681
// pc = abs(p-c) = abs(pcv)
3682
"pcmpgtw %%mm5, %%mm0 \n\t" // create mask pbv bytes < 0
3683
"pcmpgtw %%mm4, %%mm7 \n\t" // create mask pav bytes < 0
3684
"pand %%mm5, %%mm0 \n\t" // only pbv bytes < 0 in mm0
3685
"pand %%mm4, %%mm7 \n\t" // only pav bytes < 0 in mm7
3686
"psubw %%mm0, %%mm5 \n\t"
3687
"psubw %%mm7, %%mm4 \n\t"
3688
"psubw %%mm0, %%mm5 \n\t"
3689
"psubw %%mm7, %%mm4 \n\t"
3690
"pxor %%mm0, %%mm0 \n\t"
3691
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
3692
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
3693
"psubw %%mm0, %%mm6 \n\t"
3695
"movq %%mm4, %%mm7 \n\t"
3696
"psubw %%mm0, %%mm6 \n\t"
3697
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
3698
"movq %%mm7, %%mm0 \n\t"
3699
// use mm7 mask to merge pa & pb
3700
"pand %%mm7, %%mm5 \n\t"
3701
// use mm0 mask copy to merge a & b
3702
"pand %%mm0, %%mm2 \n\t"
3703
"pandn %%mm4, %%mm7 \n\t"
3704
"pandn %%mm1, %%mm0 \n\t"
3705
"paddw %%mm5, %%mm7 \n\t"
3706
"paddw %%mm2, %%mm0 \n\t"
3707
// test ((pa <= pb)? pa:pb) <= pc
3708
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
3709
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
3710
"pand %%mm7, %%mm3 \n\t"
3711
"pandn %%mm0, %%mm7 \n\t"
3712
"pxor %%mm1, %%mm1 \n\t"
3713
"paddw %%mm3, %%mm7 \n\t"
3714
"pxor %%mm0, %%mm0 \n\t"
3715
"packuswb %%mm1, %%mm7 \n\t"
3716
"movq %%mm2, %%mm3 \n\t" // load c=Prior(x-bpp) step 1
3717
"pand _ActiveMask, %%mm7 \n\t"
3718
"punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
3719
"psllq _ShiftBpp, %%mm7 \n\t" // shift bytes to 2nd group of
3721
// pav = p - a = (a + b - c) - a = b - c
3722
"movq %%mm2, %%mm4 \n\t"
3723
"paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3724
"psllq _ShiftBpp, %%mm3 \n\t" // load c=Prior(x-bpp) step 2
3725
"movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value
3726
"movq %%mm7, %%mm1 \n\t"
3727
"punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
3728
"psllq _ShiftBpp, %%mm1 \n\t" // shift bytes
3729
// now mm1 will be used as Raw(x-bpp)
3730
// now do Paeth for 3rd, and final, set of bytes (6-7)
3731
"pxor %%mm7, %%mm7 \n\t"
3732
"punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
3733
"psubw %%mm3, %%mm4 \n\t"
3734
// pbv = p - b = (a + b - c) - b = a - c
3735
"movq %%mm1, %%mm5 \n\t"
3736
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3737
"movq %%mm4, %%mm6 \n\t"
3738
"psubw %%mm3, %%mm5 \n\t"
3739
"pxor %%mm0, %%mm0 \n\t"
3740
"paddw %%mm5, %%mm6 \n\t"
3742
// pa = abs(p-a) = abs(pav)
3743
// pb = abs(p-b) = abs(pbv)
3744
// pc = abs(p-c) = abs(pcv)
3745
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
3746
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
3747
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
3748
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
3749
"psubw %%mm0, %%mm4 \n\t"
3750
"psubw %%mm7, %%mm5 \n\t"
3751
"psubw %%mm0, %%mm4 \n\t"
3752
"psubw %%mm7, %%mm5 \n\t"
3753
"pxor %%mm0, %%mm0 \n\t"
3754
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
3755
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
3756
"psubw %%mm0, %%mm6 \n\t"
3758
"movq %%mm4, %%mm7 \n\t"
3759
"psubw %%mm0, %%mm6 \n\t"
3760
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
3761
"movq %%mm7, %%mm0 \n\t"
3762
// use mm0 mask copy to merge a & b
3763
"pand %%mm0, %%mm2 \n\t"
3764
// use mm7 mask to merge pa & pb
3765
"pand %%mm7, %%mm5 \n\t"
3766
"pandn %%mm1, %%mm0 \n\t"
3767
"pandn %%mm4, %%mm7 \n\t"
3768
"paddw %%mm2, %%mm0 \n\t"
3769
"paddw %%mm5, %%mm7 \n\t"
3770
// test ((pa <= pb)? pa:pb) <= pc
3771
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
3772
"pand %%mm7, %%mm3 \n\t"
3773
"pandn %%mm0, %%mm7 \n\t"
3774
"paddw %%mm3, %%mm7 \n\t"
3775
"pxor %%mm1, %%mm1 \n\t"
3776
"packuswb %%mm7, %%mm1 \n\t"
3777
// step ecx to next set of 8 bytes and repeat loop til done
3778
"addl $8, %%ecx \n\t"
3779
"pand _ActiveMaskEnd, %%mm1 \n\t"
3780
"paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with
3783
"cmpl _MMXLength, %%ecx \n\t"
3784
"pxor %%mm0, %%mm0 \n\t" // pxor does not affect flags
3785
"movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3786
// mm1 will be used as Raw(x-bpp) next loop
3787
// mm3 ready to be used as Prior(x-bpp) next loop
3790
: "=S" (dummy_value_S), // output regs (dummy)
3791
"=D" (dummy_value_D)
3793
: "0" (prev_row), // esi // input regs
3796
: "%ecx" // clobber list
3797
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3798
, "%mm0", "%mm1", "%mm2", "%mm3"
3799
, "%mm4", "%mm5", "%mm6", "%mm7"
3806
//case 7: // GRR BOGUS
3807
//case 5: // GRR BOGUS
3809
_ActiveMask.use = 0x00000000ffffffffLL;
3810
_ActiveMask2.use = 0xffffffff00000000LL;
3811
_ShiftBpp.use = bpp << 3; // == bpp * 8
3812
_ShiftRem.use = 64 - _ShiftBpp.use;
3814
__asm__ __volatile__ (
3815
"movl _dif, %%ecx \n\t"
3816
// preload "movl row, %%edi \n\t"
3817
// preload "movl prev_row, %%esi \n\t"
3818
// prime the pump: load the first Raw(x-bpp) data set
3819
"movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3820
"pxor %%mm0, %%mm0 \n\t"
3823
// must shift to position Raw(x-bpp) data
3824
"psrlq _ShiftRem, %%mm1 \n\t"
3825
// do first set of 4 bytes
3826
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3827
"punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
3828
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
3829
"punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
3830
// must shift to position Prior(x-bpp) data
3831
"psrlq _ShiftRem, %%mm3 \n\t"
3832
// pav = p - a = (a + b - c) - a = b - c
3833
"movq %%mm2, %%mm4 \n\t"
3834
"punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c
3835
// pbv = p - b = (a + b - c) - b = a - c
3836
"movq %%mm1, %%mm5 \n\t"
3837
"psubw %%mm3, %%mm4 \n\t"
3838
"pxor %%mm7, %%mm7 \n\t"
3839
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3840
"movq %%mm4, %%mm6 \n\t"
3841
"psubw %%mm3, %%mm5 \n\t"
3842
// pa = abs(p-a) = abs(pav)
3843
// pb = abs(p-b) = abs(pbv)
3844
// pc = abs(p-c) = abs(pcv)
3845
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
3846
"paddw %%mm5, %%mm6 \n\t"
3847
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
3848
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
3849
"psubw %%mm0, %%mm4 \n\t"
3850
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
3851
"psubw %%mm0, %%mm4 \n\t"
3852
"psubw %%mm7, %%mm5 \n\t"
3853
"pxor %%mm0, %%mm0 \n\t"
3854
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
3855
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
3856
"psubw %%mm7, %%mm5 \n\t"
3857
"psubw %%mm0, %%mm6 \n\t"
3859
"movq %%mm4, %%mm7 \n\t"
3860
"psubw %%mm0, %%mm6 \n\t"
3861
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
3862
"movq %%mm7, %%mm0 \n\t"
3863
// use mm7 mask to merge pa & pb
3864
"pand %%mm7, %%mm5 \n\t"
3865
// use mm0 mask copy to merge a & b
3866
"pand %%mm0, %%mm2 \n\t"
3867
"pandn %%mm4, %%mm7 \n\t"
3868
"pandn %%mm1, %%mm0 \n\t"
3869
"paddw %%mm5, %%mm7 \n\t"
3870
"paddw %%mm2, %%mm0 \n\t"
3871
// test ((pa <= pb)? pa:pb) <= pc
3872
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
3873
"pxor %%mm1, %%mm1 \n\t"
3874
"pand %%mm7, %%mm3 \n\t"
3875
"pandn %%mm0, %%mm7 \n\t"
3876
"paddw %%mm3, %%mm7 \n\t"
3877
"pxor %%mm0, %%mm0 \n\t"
3878
"packuswb %%mm1, %%mm7 \n\t"
3879
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
3880
"pand _ActiveMask, %%mm7 \n\t"
3881
"psrlq _ShiftRem, %%mm3 \n\t"
3882
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) step 1
3883
"paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor and Raw(x)
3884
"movq %%mm2, %%mm6 \n\t"
3885
"movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value
3886
"movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3887
"psllq _ShiftBpp, %%mm6 \n\t"
3888
"movq %%mm7, %%mm5 \n\t"
3889
"psrlq _ShiftRem, %%mm1 \n\t"
3890
"por %%mm6, %%mm3 \n\t"
3891
"psllq _ShiftBpp, %%mm5 \n\t"
3892
"punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
3893
"por %%mm5, %%mm1 \n\t"
3894
// do second set of 4 bytes
3895
"punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
3896
"punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
3897
// pav = p - a = (a + b - c) - a = b - c
3898
"movq %%mm2, %%mm4 \n\t"
3899
// pbv = p - b = (a + b - c) - b = a - c
3900
"movq %%mm1, %%mm5 \n\t"
3901
"psubw %%mm3, %%mm4 \n\t"
3902
"pxor %%mm7, %%mm7 \n\t"
3903
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3904
"movq %%mm4, %%mm6 \n\t"
3905
"psubw %%mm3, %%mm5 \n\t"
3906
// pa = abs(p-a) = abs(pav)
3907
// pb = abs(p-b) = abs(pbv)
3908
// pc = abs(p-c) = abs(pcv)
3909
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
3910
"paddw %%mm5, %%mm6 \n\t"
3911
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
3912
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
3913
"psubw %%mm0, %%mm4 \n\t"
3914
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
3915
"psubw %%mm0, %%mm4 \n\t"
3916
"psubw %%mm7, %%mm5 \n\t"
3917
"pxor %%mm0, %%mm0 \n\t"
3918
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
3919
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
3920
"psubw %%mm7, %%mm5 \n\t"
3921
"psubw %%mm0, %%mm6 \n\t"
3923
"movq %%mm4, %%mm7 \n\t"
3924
"psubw %%mm0, %%mm6 \n\t"
3925
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
3926
"movq %%mm7, %%mm0 \n\t"
3927
// use mm7 mask to merge pa & pb
3928
"pand %%mm7, %%mm5 \n\t"
3929
// use mm0 mask copy to merge a & b
3930
"pand %%mm0, %%mm2 \n\t"
3931
"pandn %%mm4, %%mm7 \n\t"
3932
"pandn %%mm1, %%mm0 \n\t"
3933
"paddw %%mm5, %%mm7 \n\t"
3934
"paddw %%mm2, %%mm0 \n\t"
3935
// test ((pa <= pb)? pa:pb) <= pc
3936
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
3937
"pxor %%mm1, %%mm1 \n\t"
3938
"pand %%mm7, %%mm3 \n\t"
3939
"pandn %%mm0, %%mm7 \n\t"
3940
"pxor %%mm1, %%mm1 \n\t"
3941
"paddw %%mm3, %%mm7 \n\t"
3942
"pxor %%mm0, %%mm0 \n\t"
3943
// step ecx to next set of 8 bytes and repeat loop til done
3944
"addl $8, %%ecx \n\t"
3945
"packuswb %%mm7, %%mm1 \n\t"
3946
"paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
3947
"cmpl _MMXLength, %%ecx \n\t"
3948
"movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3949
// mm1 will be used as Raw(x-bpp) next loop
3952
: "=S" (dummy_value_S), // output regs (dummy)
3953
"=D" (dummy_value_D)
3955
: "0" (prev_row), // esi // input regs
3958
: "%ecx" // clobber list
3959
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3960
, "%mm0", "%mm1", "%mm2", "%mm3"
3961
, "%mm4", "%mm5", "%mm6", "%mm7"
3969
_ActiveMask.use = 0x00000000ffffffffLL;
3971
__asm__ __volatile__ (
3972
"movl _dif, %%ecx \n\t"
3973
// preload "movl row, %%edi \n\t"
3974
// preload "movl prev_row, %%esi \n\t"
3975
"pxor %%mm0, %%mm0 \n\t"
3976
// prime the pump: load the first Raw(x-bpp) data set
3977
"movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
3978
// a=Raw(x-bpp) bytes
3980
// do first set of 4 bytes
3981
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3982
"punpckhbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
3983
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
3984
"punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
3985
// pav = p - a = (a + b - c) - a = b - c
3986
"movq %%mm2, %%mm4 \n\t"
3987
"punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
3988
// pbv = p - b = (a + b - c) - b = a - c
3989
"movq %%mm1, %%mm5 \n\t"
3990
"psubw %%mm3, %%mm4 \n\t"
3991
"pxor %%mm7, %%mm7 \n\t"
3992
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3993
"movq %%mm4, %%mm6 \n\t"
3994
"psubw %%mm3, %%mm5 \n\t"
3995
// pa = abs(p-a) = abs(pav)
3996
// pb = abs(p-b) = abs(pbv)
3997
// pc = abs(p-c) = abs(pcv)
3998
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
3999
"paddw %%mm5, %%mm6 \n\t"
4000
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4001
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4002
"psubw %%mm0, %%mm4 \n\t"
4003
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4004
"psubw %%mm0, %%mm4 \n\t"
4005
"psubw %%mm7, %%mm5 \n\t"
4006
"pxor %%mm0, %%mm0 \n\t"
4007
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4008
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4009
"psubw %%mm7, %%mm5 \n\t"
4010
"psubw %%mm0, %%mm6 \n\t"
4012
"movq %%mm4, %%mm7 \n\t"
4013
"psubw %%mm0, %%mm6 \n\t"
4014
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4015
"movq %%mm7, %%mm0 \n\t"
4016
// use mm7 mask to merge pa & pb
4017
"pand %%mm7, %%mm5 \n\t"
4018
// use mm0 mask copy to merge a & b
4019
"pand %%mm0, %%mm2 \n\t"
4020
"pandn %%mm4, %%mm7 \n\t"
4021
"pandn %%mm1, %%mm0 \n\t"
4022
"paddw %%mm5, %%mm7 \n\t"
4023
"paddw %%mm2, %%mm0 \n\t"
4024
// test ((pa <= pb)? pa:pb) <= pc
4025
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4026
"pxor %%mm1, %%mm1 \n\t"
4027
"pand %%mm7, %%mm3 \n\t"
4028
"pandn %%mm0, %%mm7 \n\t"
4029
"paddw %%mm3, %%mm7 \n\t"
4030
"pxor %%mm0, %%mm0 \n\t"
4031
"packuswb %%mm1, %%mm7 \n\t"
4032
"movq (%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
4033
"pand _ActiveMask, %%mm7 \n\t"
4034
"movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1
4035
"paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4036
"punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4037
"movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value
4038
"movq %%mm7, %%mm1 \n\t" // now mm1 will be used as Raw(x-bpp)
4039
// do second set of 4 bytes
4040
"punpckhbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
4041
"punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4042
// pav = p - a = (a + b - c) - a = b - c
4043
"movq %%mm2, %%mm4 \n\t"
4044
// pbv = p - b = (a + b - c) - b = a - c
4045
"movq %%mm1, %%mm5 \n\t"
4046
"psubw %%mm3, %%mm4 \n\t"
4047
"pxor %%mm7, %%mm7 \n\t"
4048
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4049
"movq %%mm4, %%mm6 \n\t"
4050
"psubw %%mm3, %%mm5 \n\t"
4051
// pa = abs(p-a) = abs(pav)
4052
// pb = abs(p-b) = abs(pbv)
4053
// pc = abs(p-c) = abs(pcv)
4054
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4055
"paddw %%mm5, %%mm6 \n\t"
4056
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4057
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4058
"psubw %%mm0, %%mm4 \n\t"
4059
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4060
"psubw %%mm0, %%mm4 \n\t"
4061
"psubw %%mm7, %%mm5 \n\t"
4062
"pxor %%mm0, %%mm0 \n\t"
4063
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4064
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4065
"psubw %%mm7, %%mm5 \n\t"
4066
"psubw %%mm0, %%mm6 \n\t"
4068
"movq %%mm4, %%mm7 \n\t"
4069
"psubw %%mm0, %%mm6 \n\t"
4070
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4071
"movq %%mm7, %%mm0 \n\t"
4072
// use mm7 mask to merge pa & pb
4073
"pand %%mm7, %%mm5 \n\t"
4074
// use mm0 mask copy to merge a & b
4075
"pand %%mm0, %%mm2 \n\t"
4076
"pandn %%mm4, %%mm7 \n\t"
4077
"pandn %%mm1, %%mm0 \n\t"
4078
"paddw %%mm5, %%mm7 \n\t"
4079
"paddw %%mm2, %%mm0 \n\t"
4080
// test ((pa <= pb)? pa:pb) <= pc
4081
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4082
"pxor %%mm1, %%mm1 \n\t"
4083
"pand %%mm7, %%mm3 \n\t"
4084
"pandn %%mm0, %%mm7 \n\t"
4085
"pxor %%mm1, %%mm1 \n\t"
4086
"paddw %%mm3, %%mm7 \n\t"
4087
"pxor %%mm0, %%mm0 \n\t"
4088
// step ecx to next set of 8 bytes and repeat loop til done
4089
"addl $8, %%ecx \n\t"
4090
"packuswb %%mm7, %%mm1 \n\t"
4091
"paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add predictor with Raw(x)
4092
"cmpl _MMXLength, %%ecx \n\t"
4093
"movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4094
// mm1 will be used as Raw(x-bpp) next loop
4097
: "=S" (dummy_value_S), // output regs (dummy)
4098
"=D" (dummy_value_D)
4100
: "0" (prev_row), // esi // input regs
4103
: "%ecx" // clobber list
4104
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4105
, "%mm0", "%mm1", "%mm2", "%mm3"
4106
, "%mm4", "%mm5", "%mm6", "%mm7"
4114
_ActiveMask.use = 0x00000000ffffffffLL;
4116
__asm__ __volatile__ (
4117
"movl _dif, %%ecx \n\t"
4118
// preload "movl row, %%edi \n\t"
4119
// preload "movl prev_row, %%esi \n\t"
4120
"pxor %%mm0, %%mm0 \n\t"
4121
// prime the pump: load the first Raw(x-bpp) data set
4122
"movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
4123
// a=Raw(x-bpp) bytes
4125
// do first set of 4 bytes
4126
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4127
"punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4128
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
4129
"punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
4130
// pav = p - a = (a + b - c) - a = b - c
4131
"movq %%mm2, %%mm4 \n\t"
4132
"punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c
4133
// pbv = p - b = (a + b - c) - b = a - c
4134
"movq %%mm1, %%mm5 \n\t"
4135
"psubw %%mm3, %%mm4 \n\t"
4136
"pxor %%mm7, %%mm7 \n\t"
4137
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4138
"movq %%mm4, %%mm6 \n\t"
4139
"psubw %%mm3, %%mm5 \n\t"
4140
// pa = abs(p-a) = abs(pav)
4141
// pb = abs(p-b) = abs(pbv)
4142
// pc = abs(p-c) = abs(pcv)
4143
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4144
"paddw %%mm5, %%mm6 \n\t"
4145
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4146
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4147
"psubw %%mm0, %%mm4 \n\t"
4148
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4149
"psubw %%mm0, %%mm4 \n\t"
4150
"psubw %%mm7, %%mm5 \n\t"
4151
"pxor %%mm0, %%mm0 \n\t"
4152
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4153
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4154
"psubw %%mm7, %%mm5 \n\t"
4155
"psubw %%mm0, %%mm6 \n\t"
4157
"movq %%mm4, %%mm7 \n\t"
4158
"psubw %%mm0, %%mm6 \n\t"
4159
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4160
"movq %%mm7, %%mm0 \n\t"
4161
// use mm7 mask to merge pa & pb
4162
"pand %%mm7, %%mm5 \n\t"
4163
// use mm0 mask copy to merge a & b
4164
"pand %%mm0, %%mm2 \n\t"
4165
"pandn %%mm4, %%mm7 \n\t"
4166
"pandn %%mm1, %%mm0 \n\t"
4167
"paddw %%mm5, %%mm7 \n\t"
4168
"paddw %%mm2, %%mm0 \n\t"
4169
// test ((pa <= pb)? pa:pb) <= pc
4170
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4171
"pxor %%mm1, %%mm1 \n\t"
4172
"pand %%mm7, %%mm3 \n\t"
4173
"pandn %%mm0, %%mm7 \n\t"
4174
"paddw %%mm3, %%mm7 \n\t"
4175
"pxor %%mm0, %%mm0 \n\t"
4176
"packuswb %%mm1, %%mm7 \n\t"
4177
"movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4178
"pand _ActiveMask, %%mm7 \n\t"
4179
"movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x)
4180
"paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4181
"punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4182
"movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value
4183
"movq -8(%%edi,%%ecx,), %%mm1 \n\t" // read a=Raw(x-bpp) bytes
4185
// do second set of 4 bytes
4186
"punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4187
"punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4188
// pav = p - a = (a + b - c) - a = b - c
4189
"movq %%mm2, %%mm4 \n\t"
4190
// pbv = p - b = (a + b - c) - b = a - c
4191
"movq %%mm1, %%mm5 \n\t"
4192
"psubw %%mm3, %%mm4 \n\t"
4193
"pxor %%mm7, %%mm7 \n\t"
4194
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4195
"movq %%mm4, %%mm6 \n\t"
4196
"psubw %%mm3, %%mm5 \n\t"
4197
// pa = abs(p-a) = abs(pav)
4198
// pb = abs(p-b) = abs(pbv)
4199
// pc = abs(p-c) = abs(pcv)
4200
"pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4201
"paddw %%mm5, %%mm6 \n\t"
4202
"pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4203
"pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4204
"psubw %%mm0, %%mm4 \n\t"
4205
"pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4206
"psubw %%mm0, %%mm4 \n\t"
4207
"psubw %%mm7, %%mm5 \n\t"
4208
"pxor %%mm0, %%mm0 \n\t"
4209
"pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4210
"pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4211
"psubw %%mm7, %%mm5 \n\t"
4212
"psubw %%mm0, %%mm6 \n\t"
4214
"movq %%mm4, %%mm7 \n\t"
4215
"psubw %%mm0, %%mm6 \n\t"
4216
"pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4217
"movq %%mm7, %%mm0 \n\t"
4218
// use mm7 mask to merge pa & pb
4219
"pand %%mm7, %%mm5 \n\t"
4220
// use mm0 mask copy to merge a & b
4221
"pand %%mm0, %%mm2 \n\t"
4222
"pandn %%mm4, %%mm7 \n\t"
4223
"pandn %%mm1, %%mm0 \n\t"
4224
"paddw %%mm5, %%mm7 \n\t"
4225
"paddw %%mm2, %%mm0 \n\t"
4226
// test ((pa <= pb)? pa:pb) <= pc
4227
"pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4228
"pxor %%mm1, %%mm1 \n\t"
4229
"pand %%mm7, %%mm3 \n\t"
4230
"pandn %%mm0, %%mm7 \n\t"
4231
"pxor %%mm1, %%mm1 \n\t"
4232
"paddw %%mm3, %%mm7 \n\t"
4233
"pxor %%mm0, %%mm0 \n\t"
4234
// step ecx to next set of 8 bytes and repeat loop til done
4235
"addl $8, %%ecx \n\t"
4236
"packuswb %%mm7, %%mm1 \n\t"
4237
"paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
4238
"cmpl _MMXLength, %%ecx \n\t"
4239
"movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4240
// mm1 will be used as Raw(x-bpp) next loop
4243
: "=S" (dummy_value_S), // output regs (dummy)
4244
"=D" (dummy_value_D)
4246
: "0" (prev_row), // esi // input regs
4249
: "%ecx" // clobber list
4250
#if 0 /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4251
, "%mm0", "%mm1", "%mm2", "%mm3"
4252
, "%mm4", "%mm5", "%mm6", "%mm7"
4262
__asm__ __volatile__ (
4264
"pushl %%ebx \n\t" // save Global Offset Table index
4266
"movl _dif, %%ebx \n\t"
4267
"cmpl _FullLength, %%ebx \n\t"
4268
"jnb paeth_dend \n\t"
4270
// preload "movl row, %%edi \n\t"
4271
// preload "movl prev_row, %%esi \n\t"
4272
// do Paeth decode for remaining bytes
4273
"movl %%ebx, %%edx \n\t"
4274
// preload "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx)
4275
"subl %%ecx, %%edx \n\t" // edx = ebx - bpp
4276
"xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx
4279
"xorl %%eax, %%eax \n\t"
4280
// pav = p - a = (a + b - c) - a = b - c
4281
"movb (%%esi,%%ebx,), %%al \n\t" // load Prior(x) into al
4282
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4283
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4284
"movl %%eax, _patemp \n\t" // Save pav for later use
4285
"xorl %%eax, %%eax \n\t"
4286
// pbv = p - b = (a + b - c) - b = a - c
4287
"movb (%%edi,%%edx,), %%al \n\t" // load Raw(x-bpp) into al
4288
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4289
"movl %%eax, %%ecx \n\t"
4290
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4291
"addl _patemp, %%eax \n\t" // pcv = pav + pbv
4293
"testl $0x80000000, %%eax \n\t"
4294
"jz paeth_dpca \n\t"
4295
"negl %%eax \n\t" // reverse sign of neg values
4298
"movl %%eax, _pctemp \n\t" // save pc for later use
4300
"testl $0x80000000, %%ecx \n\t"
4301
"jz paeth_dpba \n\t"
4302
"negl %%ecx \n\t" // reverse sign of neg values
4305
"movl %%ecx, _pbtemp \n\t" // save pb for later use
4307
"movl _patemp, %%eax \n\t"
4308
"testl $0x80000000, %%eax \n\t"
4309
"jz paeth_dpaa \n\t"
4310
"negl %%eax \n\t" // reverse sign of neg values
4313
"movl %%eax, _patemp \n\t" // save pa for later use
4315
"cmpl %%ecx, %%eax \n\t"
4316
"jna paeth_dabb \n\t"
4317
// pa > pb; now test if pb <= pc
4318
"cmpl _pctemp, %%ecx \n\t"
4319
"jna paeth_dbbc \n\t"
4320
// pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4321
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4322
"jmp paeth_dpaeth \n\t"
4325
// pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4326
"movb (%%esi,%%ebx,), %%cl \n\t" // load Prior(x) into cl
4327
"jmp paeth_dpaeth \n\t"
4330
// pa <= pb; now test if pa <= pc
4331
"cmpl _pctemp, %%eax \n\t"
4332
"jna paeth_dabc \n\t"
4333
// pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4334
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4335
"jmp paeth_dpaeth \n\t"
4338
// pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4339
"movb (%%edi,%%edx,), %%cl \n\t" // load Raw(x-bpp) into cl
4341
"paeth_dpaeth: \n\t"
4344
// Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4345
"addb %%cl, -1(%%edi,%%ebx,) \n\t"
4346
"cmpl _FullLength, %%ebx \n\t"
4351
"popl %%ebx \n\t" // index to Global Offset Table
4354
: "=c" (dummy_value_c), // output regs (dummy)
4355
"=S" (dummy_value_S),
4356
"=D" (dummy_value_D)
4358
: "0" (bpp), // ecx // input regs
4359
"1" (prev_row), // esi
4362
: "%eax", "%edx" // clobber list
4368
return; // No need to go further with this one
4370
} // end switch (bpp)
4372
__asm__ __volatile__ (
4373
// MMX acceleration complete; now do clean-up
4374
// check if any remaining bytes left to decode
4376
"pushl %%ebx \n\t" // save index to Global Offset Table
4378
"movl _MMXLength, %%ebx \n\t"
4379
"cmpl _FullLength, %%ebx \n\t"
4380
"jnb paeth_end \n\t"
4381
//pre "movl row, %%edi \n\t"
4382
//pre "movl prev_row, %%esi \n\t"
4383
// do Paeth decode for remaining bytes
4384
"movl %%ebx, %%edx \n\t"
4385
//pre "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx)
4386
"subl %%ecx, %%edx \n\t" // edx = ebx - bpp
4387
"xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below
4390
"xorl %%eax, %%eax \n\t"
4391
// pav = p - a = (a + b - c) - a = b - c
4392
"movb (%%esi,%%ebx,), %%al \n\t" // load Prior(x) into al
4393
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4394
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4395
"movl %%eax, _patemp \n\t" // Save pav for later use
4396
"xorl %%eax, %%eax \n\t"
4397
// pbv = p - b = (a + b - c) - b = a - c
4398
"movb (%%edi,%%edx,), %%al \n\t" // load Raw(x-bpp) into al
4399
"subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4400
"movl %%eax, %%ecx \n\t"
4401
// pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4402
"addl _patemp, %%eax \n\t" // pcv = pav + pbv
4404
"testl $0x80000000, %%eax \n\t"
4405
"jz paeth_pca2 \n\t"
4406
"negl %%eax \n\t" // reverse sign of neg values
4409
"movl %%eax, _pctemp \n\t" // save pc for later use
4411
"testl $0x80000000, %%ecx \n\t"
4412
"jz paeth_pba2 \n\t"
4413
"negl %%ecx \n\t" // reverse sign of neg values
4416
"movl %%ecx, _pbtemp \n\t" // save pb for later use
4418
"movl _patemp, %%eax \n\t"
4419
"testl $0x80000000, %%eax \n\t"
4420
"jz paeth_paa2 \n\t"
4421
"negl %%eax \n\t" // reverse sign of neg values
4424
"movl %%eax, _patemp \n\t" // save pa for later use
4426
"cmpl %%ecx, %%eax \n\t"
4427
"jna paeth_abb2 \n\t"
4428
// pa > pb; now test if pb <= pc
4429
"cmpl _pctemp, %%ecx \n\t"
4430
"jna paeth_bbc2 \n\t"
4431
// pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4432
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4433
"jmp paeth_paeth2 \n\t"
4436
// pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4437
"movb (%%esi,%%ebx,), %%cl \n\t" // load Prior(x) into cl
4438
"jmp paeth_paeth2 \n\t"
4441
// pa <= pb; now test if pa <= pc
4442
"cmpl _pctemp, %%eax \n\t"
4443
"jna paeth_abc2 \n\t"
4444
// pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4445
"movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl
4446
"jmp paeth_paeth2 \n\t"
4449
// pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4450
"movb (%%edi,%%edx,), %%cl \n\t" // load Raw(x-bpp) into cl
4452
"paeth_paeth2: \n\t"
4455
// Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4456
"addb %%cl, -1(%%edi,%%ebx,) \n\t"
4457
"cmpl _FullLength, %%ebx \n\t"
4461
"EMMS \n\t" // end MMX; prep for poss. FP instrs.
4463
"popl %%ebx \n\t" // restore index to Global Offset Table
4466
: "=c" (dummy_value_c), // output regs (dummy)
4467
"=S" (dummy_value_S),
4468
"=D" (dummy_value_D)
4470
: "0" (bpp), // ecx // input regs
4471
"1" (prev_row), // esi
4474
: "%eax", "%edx" // clobber list (no input regs!)
4480
} /* end png_read_filter_row_mmx_paeth() */
4486
#ifdef PNG_THREAD_UNSAFE_OK
4487
//===========================================================================//
4489
// P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B //
4491
//===========================================================================//
4493
// Optimized code for PNG Sub filter decoder
4495
static void /* PRIVATE */
4496
png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row)
4502
bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel
4503
_FullLength = row_info->rowbytes - bpp; // number of bytes to filter
4505
__asm__ __volatile__ (
4506
//pre "movl row, %%edi \n\t"
4507
"movl %%edi, %%esi \n\t" // lp = row
4508
//pre "movl bpp, %%eax \n\t"
4509
"addl %%eax, %%edi \n\t" // rp = row + bpp
4510
//irr "xorl %%eax, %%eax \n\t"
4511
// get # of bytes to alignment
4512
"movl %%edi, _dif \n\t" // take start of row
4513
"addl $0xf, _dif \n\t" // add 7 + 8 to incr past
4514
// alignment boundary
4515
"xorl %%ecx, %%ecx \n\t"
4516
"andl $0xfffffff8, _dif \n\t" // mask to alignment boundary
4517
"subl %%edi, _dif \n\t" // subtract from start ==> value
4518
"jz sub_go \n\t" // ecx at alignment
4520
"sub_lp1: \n\t" // fix alignment
4521
"movb (%%esi,%%ecx,), %%al \n\t"
4522
"addb %%al, (%%edi,%%ecx,) \n\t"
4524
"cmpl _dif, %%ecx \n\t"
4528
"movl _FullLength, %%eax \n\t"
4529
"movl %%eax, %%edx \n\t"
4530
"subl %%ecx, %%edx \n\t" // subtract alignment fix
4531
"andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8
4532
"subl %%edx, %%eax \n\t" // drop over bytes from length
4533
"movl %%eax, _MMXLength \n\t"
4535
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4536
"=D" (dummy_value_D) // 1
4538
: "0" (bpp), // eax // input regs
4541
: "%esi", "%ecx", "%edx" // clobber list
4543
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4544
, "%mm0", "%mm1", "%mm2", "%mm3"
4545
, "%mm4", "%mm5", "%mm6", "%mm7"
4549
// now do the math for the rest of the row
4554
_ActiveMask.use = 0x0000ffffff000000LL;
4555
_ShiftBpp.use = 24; // == 3 * 8
4556
_ShiftRem.use = 40; // == 64 - 24
4558
__asm__ __volatile__ (
4559
// preload "movl row, %%edi \n\t"
4560
"movq _ActiveMask, %%mm7 \n\t" // load _ActiveMask for 2nd
4561
// active byte group
4562
"movl %%edi, %%esi \n\t" // lp = row
4563
// preload "movl bpp, %%eax \n\t"
4564
"addl %%eax, %%edi \n\t" // rp = row + bpp
4565
"movq %%mm7, %%mm6 \n\t"
4566
"movl _dif, %%edx \n\t"
4567
"psllq _ShiftBpp, %%mm6 \n\t" // move mask in mm6 to cover
4568
// 3rd active byte group
4569
// prime the pump: load the first Raw(x-bpp) data set
4570
"movq -8(%%edi,%%edx,), %%mm1 \n\t"
4572
"sub_3lp: \n\t" // shift data for adding first
4573
"psrlq _ShiftRem, %%mm1 \n\t" // bpp bytes (no need for mask;
4574
// shift clears inactive bytes)
4575
// add 1st active group
4576
"movq (%%edi,%%edx,), %%mm0 \n\t"
4577
"paddb %%mm1, %%mm0 \n\t"
4579
// add 2nd active group
4580
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4581
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4582
"pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group
4583
"paddb %%mm1, %%mm0 \n\t"
4585
// add 3rd active group
4586
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4587
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4588
"pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group
4589
"addl $8, %%edx \n\t"
4590
"paddb %%mm1, %%mm0 \n\t"
4592
"cmpl _MMXLength, %%edx \n\t"
4593
"movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4594
"movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
4597
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4598
"=D" (dummy_value_D) // 1
4600
: "0" (bpp), // eax // input regs
4603
: "%edx", "%esi" // clobber list
4604
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4605
, "%mm0", "%mm1", "%mm6", "%mm7"
4613
__asm__ __volatile__ (
4614
"movl _dif, %%edx \n\t"
4615
// preload "movl row, %%edi \n\t"
4616
"cmpl _FullLength, %%edx \n\t"
4618
"movl %%edi, %%esi \n\t" // lp = row
4619
"xorl %%eax, %%eax \n\t"
4620
// preload "movl bpp, %%eax \n\t"
4621
"addl %%eax, %%edi \n\t" // rp = row + bpp
4624
"movb (%%esi,%%edx,), %%al \n\t"
4625
"addb %%al, (%%edi,%%edx,) \n\t"
4627
"cmpl _FullLength, %%edx \n\t"
4632
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4633
"=D" (dummy_value_D) // 1
4635
: "0" (bpp), // eax // input regs
4638
: "%edx", "%esi" // clobber list
4645
//case 7: // GRR BOGUS
4646
//case 5: // GRR BOGUS
4648
_ShiftBpp.use = bpp << 3;
4649
_ShiftRem.use = 64 - _ShiftBpp.use;
4651
__asm__ __volatile__ (
4652
// preload "movl row, %%edi \n\t"
4653
"movl _dif, %%edx \n\t"
4654
"movl %%edi, %%esi \n\t" // lp = row
4655
// preload "movl bpp, %%eax \n\t"
4656
"addl %%eax, %%edi \n\t" // rp = row + bpp
4658
// prime the pump: load the first Raw(x-bpp) data set
4659
"movq -8(%%edi,%%edx,), %%mm1 \n\t"
4661
"sub_4lp: \n\t" // shift data for adding first
4662
"psrlq _ShiftRem, %%mm1 \n\t" // bpp bytes (no need for mask;
4663
// shift clears inactive bytes)
4664
"movq (%%edi,%%edx,), %%mm0 \n\t"
4665
"paddb %%mm1, %%mm0 \n\t"
4667
// add 2nd active group
4668
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4669
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4670
"addl $8, %%edx \n\t"
4671
"paddb %%mm1, %%mm0 \n\t"
4673
"cmpl _MMXLength, %%edx \n\t"
4674
"movq %%mm0, -8(%%edi,%%edx,) \n\t"
4675
"movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
4678
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4679
"=D" (dummy_value_D) // 1
4681
: "0" (bpp), // eax // input regs
4684
: "%edx", "%esi" // clobber list
4685
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4694
_ActiveMask.use = 0x00000000ffff0000LL;
4695
_ShiftBpp.use = 16; // == 2 * 8
4696
_ShiftRem.use = 48; // == 64 - 16
4698
__asm__ __volatile__ (
4699
"movq _ActiveMask, %%mm7 \n\t" // load _ActiveMask for 2nd
4700
// active byte group
4701
"movl _dif, %%edx \n\t"
4702
"movq %%mm7, %%mm6 \n\t"
4703
// preload "movl row, %%edi \n\t"
4704
"psllq _ShiftBpp, %%mm6 \n\t" // move mask in mm6 to cover
4705
// 3rd active byte group
4706
"movl %%edi, %%esi \n\t" // lp = row
4707
"movq %%mm6, %%mm5 \n\t"
4708
// preload "movl bpp, %%eax \n\t"
4709
"addl %%eax, %%edi \n\t" // rp = row + bpp
4710
"psllq _ShiftBpp, %%mm5 \n\t" // move mask in mm5 to cover
4711
// 4th active byte group
4712
// prime the pump: load the first Raw(x-bpp) data set
4713
"movq -8(%%edi,%%edx,), %%mm1 \n\t"
4715
"sub_2lp: \n\t" // shift data for adding first
4716
"psrlq _ShiftRem, %%mm1 \n\t" // bpp bytes (no need for mask;
4717
// shift clears inactive bytes)
4718
// add 1st active group
4719
"movq (%%edi,%%edx,), %%mm0 \n\t"
4720
"paddb %%mm1, %%mm0 \n\t"
4722
// add 2nd active group
4723
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4724
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4725
"pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group
4726
"paddb %%mm1, %%mm0 \n\t"
4728
// add 3rd active group
4729
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4730
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4731
"pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group
4732
"paddb %%mm1, %%mm0 \n\t"
4734
// add 4th active group
4735
"movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
4736
"psllq _ShiftBpp, %%mm1 \n\t" // shift data to pos. correctly
4737
"pand %%mm5, %%mm1 \n\t" // mask to use 4th active group
4738
"addl $8, %%edx \n\t"
4739
"paddb %%mm1, %%mm0 \n\t"
4740
"cmpl _MMXLength, %%edx \n\t"
4741
"movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4742
"movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
4745
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4746
"=D" (dummy_value_D) // 1
4748
: "0" (bpp), // eax // input regs
4751
: "%edx", "%esi" // clobber list
4752
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4753
, "%mm0", "%mm1", "%mm5", "%mm6", "%mm7"
4761
__asm__ __volatile__ (
4762
// preload "movl row, %%edi \n\t"
4763
"movl _dif, %%edx \n\t"
4764
"movl %%edi, %%esi \n\t" // lp = row
4765
// preload "movl bpp, %%eax \n\t"
4766
"addl %%eax, %%edi \n\t" // rp = row + bpp
4767
"movl _MMXLength, %%ecx \n\t"
4769
// prime the pump: load the first Raw(x-bpp) data set
4770
"movq -8(%%edi,%%edx,), %%mm7 \n\t"
4771
"andl $0x0000003f, %%ecx \n\t" // calc bytes over mult of 64
4774
"movq (%%edi,%%edx,), %%mm0 \n\t" // load Sub(x) for 1st 8 bytes
4775
"paddb %%mm7, %%mm0 \n\t"
4776
"movq 8(%%edi,%%edx,), %%mm1 \n\t" // load Sub(x) for 2nd 8 bytes
4777
"movq %%mm0, (%%edi,%%edx,) \n\t" // write Raw(x) for 1st 8 bytes
4779
// Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes.
4780
// This will be repeated for each group of 8 bytes with the 8th
4781
// group being used as the Raw(x-bpp) for the 1st group of the
4784
"paddb %%mm0, %%mm1 \n\t"
4785
"movq 16(%%edi,%%edx,), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes
4786
"movq %%mm1, 8(%%edi,%%edx,) \n\t" // write Raw(x) for 2nd 8 bytes
4787
"paddb %%mm1, %%mm2 \n\t"
4788
"movq 24(%%edi,%%edx,), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes
4789
"movq %%mm2, 16(%%edi,%%edx,) \n\t" // write Raw(x) for 3rd 8 bytes
4790
"paddb %%mm2, %%mm3 \n\t"
4791
"movq 32(%%edi,%%edx,), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes
4792
"movq %%mm3, 24(%%edi,%%edx,) \n\t" // write Raw(x) for 4th 8 bytes
4793
"paddb %%mm3, %%mm4 \n\t"
4794
"movq 40(%%edi,%%edx,), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes
4795
"movq %%mm4, 32(%%edi,%%edx,) \n\t" // write Raw(x) for 5th 8 bytes
4796
"paddb %%mm4, %%mm5 \n\t"
4797
"movq 48(%%edi,%%edx,), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes
4798
"movq %%mm5, 40(%%edi,%%edx,) \n\t" // write Raw(x) for 6th 8 bytes
4799
"paddb %%mm5, %%mm6 \n\t"
4800
"movq 56(%%edi,%%edx,), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes
4801
"movq %%mm6, 48(%%edi,%%edx,) \n\t" // write Raw(x) for 7th 8 bytes
4802
"addl $64, %%edx \n\t"
4803
"paddb %%mm6, %%mm7 \n\t"
4804
"cmpl %%ecx, %%edx \n\t"
4805
"movq %%mm7, -8(%%edi,%%edx,) \n\t" // write Raw(x) for 8th 8 bytes
4808
"cmpl _MMXLength, %%edx \n\t"
4812
"movq (%%edi,%%edx,), %%mm0 \n\t"
4813
"addl $8, %%edx \n\t"
4814
"paddb %%mm7, %%mm0 \n\t"
4815
"cmpl _MMXLength, %%edx \n\t"
4816
"movq %%mm0, -8(%%edi,%%edx,) \n\t" // -8 to offset early addl edx
4817
"movq %%mm0, %%mm7 \n\t" // move calculated Raw(x) data
4818
// to mm1 to be new Raw(x-bpp)
4824
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4825
"=D" (dummy_value_D) // 1
4827
: "0" (bpp), // eax // input regs
4830
: "%ecx", "%edx", "%esi" // clobber list
4831
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4832
, "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7"
4838
default: // bpp greater than 8 bytes GRR BOGUS
4840
__asm__ __volatile__ (
4841
"movl _dif, %%edx \n\t"
4842
// preload "movl row, %%edi \n\t"
4843
"movl %%edi, %%esi \n\t" // lp = row
4844
// preload "movl bpp, %%eax \n\t"
4845
"addl %%eax, %%edi \n\t" // rp = row + bpp
4848
"movq (%%edi,%%edx,), %%mm0 \n\t"
4849
"movq (%%esi,%%edx,), %%mm1 \n\t"
4850
"addl $8, %%edx \n\t"
4851
"paddb %%mm1, %%mm0 \n\t"
4852
"cmpl _MMXLength, %%edx \n\t"
4853
"movq %%mm0, -8(%%edi,%%edx,) \n\t" // mov does not affect flags;
4854
// -8 to offset addl edx
4857
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4858
"=D" (dummy_value_D) // 1
4860
: "0" (bpp), // eax // input regs
4863
: "%edx", "%esi" // clobber list
4864
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4871
} // end switch (bpp)
4873
__asm__ __volatile__ (
4874
"movl _MMXLength, %%edx \n\t"
4875
//pre "movl row, %%edi \n\t"
4876
"cmpl _FullLength, %%edx \n\t"
4879
"movl %%edi, %%esi \n\t" // lp = row
4880
//pre "movl bpp, %%eax \n\t"
4881
"addl %%eax, %%edi \n\t" // rp = row + bpp
4882
"xorl %%eax, %%eax \n\t"
4885
"movb (%%esi,%%edx,), %%al \n\t"
4886
"addb %%al, (%%edi,%%edx,) \n\t"
4888
"cmpl _FullLength, %%edx \n\t"
4892
"EMMS \n\t" // end MMX instructions
4894
: "=a" (dummy_value_a), // 0 // output regs (dummy)
4895
"=D" (dummy_value_D) // 1
4897
: "0" (bpp), // eax // input regs
4900
: "%edx", "%esi" // clobber list
4903
} // end of png_read_filter_row_mmx_sub()
4909
//===========================================================================//
4911
// P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P //
4913
//===========================================================================//
4915
// Optimized code for PNG Up filter decoder
4917
static void /* PRIVATE */
4918
png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row,
4922
int dummy_value_d; // fix 'forbidden register 3 (dx) was spilled' error
4926
len = row_info->rowbytes; // number of bytes to filter
4928
__asm__ __volatile__ (
4929
//pre "movl row, %%edi \n\t"
4930
// get # of bytes to alignment
4934
"movl %%edi, %%ecx \n\t"
4935
"xorl %%ebx, %%ebx \n\t"
4936
"addl $0x7, %%ecx \n\t"
4937
"xorl %%eax, %%eax \n\t"
4938
"andl $0xfffffff8, %%ecx \n\t"
4939
//pre "movl prev_row, %%esi \n\t"
4940
"subl %%edi, %%ecx \n\t"
4943
"up_lp1: \n\t" // fix alignment
4944
"movb (%%edi,%%ebx,), %%al \n\t"
4945
"addb (%%esi,%%ebx,), %%al \n\t"
4947
"cmpl %%ecx, %%ebx \n\t"
4948
"movb %%al, -1(%%edi,%%ebx,) \n\t" // mov does not affect flags; -1 to
4949
"jb up_lp1 \n\t" // offset incl ebx
4952
//pre "movl len, %%edx \n\t"
4953
"movl %%edx, %%ecx \n\t"
4954
"subl %%ebx, %%edx \n\t" // subtract alignment fix
4955
"andl $0x0000003f, %%edx \n\t" // calc bytes over mult of 64
4956
"subl %%edx, %%ecx \n\t" // drop over bytes from length
4958
// unrolled loop - use all MMX registers and interleave to reduce
4959
// number of branch instructions (loops) and reduce partial stalls
4961
"movq (%%esi,%%ebx,), %%mm1 \n\t"
4962
"movq (%%edi,%%ebx,), %%mm0 \n\t"
4963
"movq 8(%%esi,%%ebx,), %%mm3 \n\t"
4964
"paddb %%mm1, %%mm0 \n\t"
4965
"movq 8(%%edi,%%ebx,), %%mm2 \n\t"
4966
"movq %%mm0, (%%edi,%%ebx,) \n\t"
4967
"paddb %%mm3, %%mm2 \n\t"
4968
"movq 16(%%esi,%%ebx,), %%mm5 \n\t"
4969
"movq %%mm2, 8(%%edi,%%ebx,) \n\t"
4970
"movq 16(%%edi,%%ebx,), %%mm4 \n\t"
4971
"movq 24(%%esi,%%ebx,), %%mm7 \n\t"
4972
"paddb %%mm5, %%mm4 \n\t"
4973
"movq 24(%%edi,%%ebx,), %%mm6 \n\t"
4974
"movq %%mm4, 16(%%edi,%%ebx,) \n\t"
4975
"paddb %%mm7, %%mm6 \n\t"
4976
"movq 32(%%esi,%%ebx,), %%mm1 \n\t"
4977
"movq %%mm6, 24(%%edi,%%ebx,) \n\t"
4978
"movq 32(%%edi,%%ebx,), %%mm0 \n\t"
4979
"movq 40(%%esi,%%ebx,), %%mm3 \n\t"
4980
"paddb %%mm1, %%mm0 \n\t"
4981
"movq 40(%%edi,%%ebx,), %%mm2 \n\t"
4982
"movq %%mm0, 32(%%edi,%%ebx,) \n\t"
4983
"paddb %%mm3, %%mm2 \n\t"
4984
"movq 48(%%esi,%%ebx,), %%mm5 \n\t"
4985
"movq %%mm2, 40(%%edi,%%ebx,) \n\t"
4986
"movq 48(%%edi,%%ebx,), %%mm4 \n\t"
4987
"movq 56(%%esi,%%ebx,), %%mm7 \n\t"
4988
"paddb %%mm5, %%mm4 \n\t"
4989
"movq 56(%%edi,%%ebx,), %%mm6 \n\t"
4990
"movq %%mm4, 48(%%edi,%%ebx,) \n\t"
4991
"addl $64, %%ebx \n\t"
4992
"paddb %%mm7, %%mm6 \n\t"
4993
"cmpl %%ecx, %%ebx \n\t"
4994
"movq %%mm6, -8(%%edi,%%ebx,) \n\t" // (+56)movq does not affect flags;
4995
"jb up_loop \n\t" // -8 to offset addl ebx
4997
"cmpl $0, %%edx \n\t" // test for bytes over mult of 64
5000
"cmpl $8, %%edx \n\t" // test for less than 8 bytes
5001
"jb up_lt8 \n\t" // [added by lcreeve at netins.net]
5003
"addl %%edx, %%ecx \n\t"
5004
"andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8
5005
"subl %%edx, %%ecx \n\t" // drop over bytes from length
5008
"up_lpA: \n\t" // use MMX regs to update 8 bytes sim.
5009
"movq (%%esi,%%ebx,), %%mm1 \n\t"
5010
"movq (%%edi,%%ebx,), %%mm0 \n\t"
5011
"addl $8, %%ebx \n\t"
5012
"paddb %%mm1, %%mm0 \n\t"
5013
"cmpl %%ecx, %%ebx \n\t"
5014
"movq %%mm0, -8(%%edi,%%ebx,) \n\t" // movq does not affect flags; -8 to
5015
"jb up_lpA \n\t" // offset add ebx
5016
"cmpl $0, %%edx \n\t" // test for bytes over mult of 8
5020
"xorl %%eax, %%eax \n\t"
5021
"addl %%edx, %%ecx \n\t" // move over byte count into counter
5023
"up_lp2: \n\t" // use x86 regs for remaining bytes
5024
"movb (%%edi,%%ebx,), %%al \n\t"
5025
"addb (%%esi,%%ebx,), %%al \n\t"
5027
"cmpl %%ecx, %%ebx \n\t"
5028
"movb %%al, -1(%%edi,%%ebx,) \n\t" // mov does not affect flags; -1 to
5029
"jb up_lp2 \n\t" // offset inc ebx
5032
"EMMS \n\t" // conversion of filtered row complete
5037
: "=d" (dummy_value_d), // 0 // output regs (dummy)
5038
"=S" (dummy_value_S), // 1
5039
"=D" (dummy_value_D) // 2
5041
: "0" (len), // edx // input regs
5042
"1" (prev_row), // esi
5045
: "%eax", "%ecx" // clobber list (no input regs!)
5050
#if 0 /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
5051
, "%mm0", "%mm1", "%mm2", "%mm3"
5052
, "%mm4", "%mm5", "%mm6", "%mm7"
5056
} // end of png_read_filter_row_mmx_up()
5058
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5063
/*===========================================================================*/
5065
/* P N G _ R E A D _ F I L T E R _ R O W */
5067
/*===========================================================================*/
5070
/* Optimized png_read_filter_row routines */
5073
png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep
5074
row, png_bytep prev_row, int filter)
5080
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5081
/* GRR: these are superseded by png_ptr->asm_flags: */
5082
#define UseMMX_sub 1 // GRR: converted 20000730
5083
#define UseMMX_up 1 // GRR: converted 20000729
5084
#define UseMMX_avg 1 // GRR: converted 20000828 (+ 16-bit bugfix 20000916)
5085
#define UseMMX_paeth 1 // GRR: converted 20000828
5087
if (_mmx_supported == 2) {
5088
/* this should have happened in png_init_mmx_flags() already */
5089
#if !defined(PNG_1_0_X)
5090
png_warning(png_ptr, "asm_flags may not have been initialized");
5094
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5097
png_debug(1, "in png_read_filter_row (pnggccrd.c)\n");
5100
case 0: sprintf(filnm, "none");
5102
case 1: sprintf(filnm, "sub-%s",
5103
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5104
#if !defined(PNG_1_0_X)
5105
(png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB)? "MMX" :
5110
case 2: sprintf(filnm, "up-%s",
5111
#ifdef PNG_ASSEMBLER_CODE_SUPPORTED
5112
#if !defined(PNG_1_0_X)
5113
(png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP)? "MMX" :
5118
case 3: sprintf(filnm, "avg-%s",
5119
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5120
#if !defined(PNG_1_0_X)
5121
(png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG)? "MMX" :
5126
case 4: sprintf(filnm, "Paeth-%s",
5127
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5128
#if !defined(PNG_1_0_X)
5129
(png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH)? "MMX":
5134
default: sprintf(filnm, "unknw");
5137
png_debug2(0, "row_number=%5ld, %5s, ", png_ptr->row_number, filnm);
5138
png_debug1(0, "row=0x%08lx, ", (unsigned long)row);
5139
png_debug2(0, "pixdepth=%2d, bytes=%d, ", (int)row_info->pixel_depth,
5140
(int)((row_info->pixel_depth + 7) >> 3));
5141
png_debug1(0,"rowbytes=%8ld\n", row_info->rowbytes);
5142
#endif /* PNG_DEBUG */
5146
case PNG_FILTER_VALUE_NONE:
5149
case PNG_FILTER_VALUE_SUB:
5150
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5151
#if !defined(PNG_1_0_X)
5152
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
5153
(row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5154
(row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5159
png_read_filter_row_mmx_sub(row_info, row);
5162
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5165
png_uint_32 istop = row_info->rowbytes;
5166
png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5167
png_bytep rp = row + bpp;
5170
for (i = bpp; i < istop; i++)
5172
*rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff);
5175
} /* end !UseMMX_sub */
5178
case PNG_FILTER_VALUE_UP:
5179
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5180
#if !defined(PNG_1_0_X)
5181
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
5182
(row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5183
(row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5188
png_read_filter_row_mmx_up(row_info, row, prev_row);
5191
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5194
png_uint_32 istop = row_info->rowbytes;
5196
png_bytep pp = prev_row;
5198
for (i = 0; i < istop; ++i)
5200
*rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5203
} /* end !UseMMX_up */
5206
case PNG_FILTER_VALUE_AVG:
5207
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5208
#if !defined(PNG_1_0_X)
5209
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
5210
(row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5211
(row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5216
png_read_filter_row_mmx_avg(row_info, row, prev_row);
5219
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5223
png_bytep pp = prev_row;
5225
png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5226
png_uint_32 istop = row_info->rowbytes - bpp;
5228
for (i = 0; i < bpp; i++)
5230
*rp = (png_byte)(((int)(*rp) +
5231
((int)(*pp++) >> 1)) & 0xff);
5235
for (i = 0; i < istop; i++)
5237
*rp = (png_byte)(((int)(*rp) +
5238
((int)(*pp++ + *lp++) >> 1)) & 0xff);
5241
} /* end !UseMMX_avg */
5244
case PNG_FILTER_VALUE_PAETH:
5245
#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5246
#if !defined(PNG_1_0_X)
5247
if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
5248
(row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5249
(row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5254
png_read_filter_row_mmx_paeth(row_info, row, prev_row);
5257
#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5261
png_bytep pp = prev_row;
5263
png_bytep cp = prev_row;
5264
png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5265
png_uint_32 istop = row_info->rowbytes - bpp;
5267
for (i = 0; i < bpp; i++)
5269
*rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5273
for (i = 0; i < istop; i++) /* use leftover rp,pp */
5275
int a, b, c, pa, pb, pc, p;
5289
pa = p < 0 ? -p : p;
5290
pb = pc < 0 ? -pc : pc;
5291
pc = (p + pc) < 0 ? -(p + pc) : p + pc;
5295
if (pa <= pb && pa <= pc)
5303
p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c;
5305
*rp = (png_byte)(((int)(*rp) + p) & 0xff);
5308
} /* end !UseMMX_paeth */
5312
png_warning(png_ptr, "Ignoring bad row-filter type");
5318
#endif /* PNG_HAVE_ASSEMBLER_READ_FILTER_ROW */
5321
/*===========================================================================*/
5323
/* P N G _ M M X _ S U P P O R T */
5325
/*===========================================================================*/
5327
/* GRR NOTES: (1) the following code assumes 386 or better (pushfl/popfl)
5328
* (2) all instructions compile with gcc 2.7.2.3 and later
5329
* (3) the function is moved down here to prevent gcc from
5330
* inlining it in multiple places and then barfing be-
5331
* cause the ".NOT_SUPPORTED" label is multiply defined
5332
* [is there a way to signal that a *single* function should
5333
* not be inlined? is there a way to modify the label for
5334
* each inlined instance, e.g., by appending _1, _2, etc.?
5335
* maybe if don't use leading "." in label name? (nope...sigh)]
5339
png_mmx_support(void)
5341
#if defined(PNG_MMX_CODE_SUPPORTED)
5342
__asm__ __volatile__ (
5343
"pushl %%ebx \n\t" // ebx gets clobbered by CPUID instruction
5344
"pushl %%ecx \n\t" // so does ecx...
5345
"pushl %%edx \n\t" // ...and edx (but ecx & edx safe on Linux)
5346
// ".byte 0x66 \n\t" // convert 16-bit pushf to 32-bit pushfd
5347
// "pushf \n\t" // 16-bit pushf
5348
"pushfl \n\t" // save Eflag to stack
5349
"popl %%eax \n\t" // get Eflag from stack into eax
5350
"movl %%eax, %%ecx \n\t" // make another copy of Eflag in ecx
5351
"xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
5352
"pushl %%eax \n\t" // save modified Eflag back to stack
5353
// ".byte 0x66 \n\t" // convert 16-bit popf to 32-bit popfd
5354
// "popf \n\t" // 16-bit popf
5355
"popfl \n\t" // restore modified value to Eflag reg
5356
"pushfl \n\t" // save Eflag to stack
5357
"popl %%eax \n\t" // get Eflag from stack
5358
"pushl %%ecx \n\t" // save original Eflag to stack
5359
"popfl \n\t" // restore original Eflag
5360
"xorl %%ecx, %%eax \n\t" // compare new Eflag with original Eflag
5361
"jz 0f \n\t" // if same, CPUID instr. is not supported
5363
"xorl %%eax, %%eax \n\t" // set eax to zero
5364
// ".byte 0x0f, 0xa2 \n\t" // CPUID instruction (two-byte opcode)
5365
"cpuid \n\t" // get the CPU identification info
5366
"cmpl $1, %%eax \n\t" // make sure eax return non-zero value
5367
"jl 0f \n\t" // if eax is zero, MMX is not supported
5369
"xorl %%eax, %%eax \n\t" // set eax to zero and...
5370
"incl %%eax \n\t" // ...increment eax to 1. This pair is
5371
// faster than the instruction "mov eax, 1"
5372
"cpuid \n\t" // get the CPU identification info again
5373
"andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23)
5374
"cmpl $0, %%edx \n\t" // 0 = MMX not supported
5375
"jz 0f \n\t" // non-zero = yes, MMX IS supported
5377
"movl $1, %%eax \n\t" // set return value to 1
5378
"jmp 1f \n\t" // DONE: have MMX support
5380
"0: \n\t" // .NOT_SUPPORTED: target label for jump instructions
5381
"movl $0, %%eax \n\t" // set return value to 0
5382
"1: \n\t" // .RETURN: target label for jump instructions
5383
"movl %%eax, _mmx_supported \n\t" // save in global static variable, too
5384
"popl %%edx \n\t" // restore edx
5385
"popl %%ecx \n\t" // restore ecx
5386
"popl %%ebx \n\t" // restore ebx
5388
// "ret \n\t" // DONE: no MMX support
5389
// (fall through to standard C "ret")
5391
: // output list (none)
5393
: // any variables used on input (none)
5395
: "%eax" // clobber list
5396
// , "%ebx", "%ecx", "%edx" // GRR: we handle these manually
5397
// , "memory" // if write to a variable gcc thought was in a reg
5398
// , "cc" // "condition codes" (flag bits)
5402
#endif /* PNG_MMX_CODE_SUPPORTED */
5404
return _mmx_supported;
5408
#endif /* PNG_USE_PNGGCCRD */
added
removed
src/3rdparty/libpng/pnggccrd.c
