File indexing completed on 2025-03-09 04:09:53

0001 /* Flattning functions for xcftools
0002  *
0003  * This file was written by Henning Makholm <henning@makholm.net>
0004  * It is hereby in the public domain.
0005  *
0006  * In jurisdictions that do not recognise grants of copyright to the
0007  * public domain: I, the author and (presumably, in those jurisdictions)
0008  * copyright holder, hereby permit anyone to distribute and use this code,
0009  * in source code or binary form, with or without modifications. This
0010  * permission is world-wide and irrevocable.
0011  *
0012  * Of course, I will not be liable for any errors or shortcomings in the
0013  * code, since I give it away without asking any compenstations.
0014  *
0015  * If you use or distribute this code, I would appreciate receiving
0016  * credit for writing it, in whichever way you find proper and customary.
0017  */
0018 
0019 #include "xcftools.h"
0020 #include "flatten.h"
0021 #include "pixels.h"
0022 #include <string.h>
0023 #include <stdlib.h>
0024 #include <assert.h>
0025 
0026 static rgba __ATTRIBUTE__((noinline,const))
0027 composite_one(rgba bot,rgba top)
0028 {
0029     unsigned tfrac, alpha ;
0030 
0031     tfrac = ALPHA(top) ;
0032     alpha = 255 ;
0033     if( !FULLALPHA(bot) ) {
0034         alpha = 255 ^ scaletable[255-ALPHA(bot)][255-ALPHA(top)] ;
0035         /* This peculiar combination of ^ and - makes the GCC code
0036          * generator for i386 particularly happy.
0037          */
0038         tfrac = (256*ALPHA(top) - 1) / alpha ;
0039         /* Tfrac is the fraction of the coposited pixel's covered area
0040          * that comes from the top pixel.
0041          * For mathematical accuracy we ought to scale by 255 and
0042          * subtract alpha/2, but this is faster, and never misses the
0043          * true value by more than one 1/255. This effect is completely
0044          * overshadowed by the linear interpolation in the first place.
0045          * (I.e. gamma is ignored when combining intensities).
0046          *   [In any case, complete fairness is not possible: if the
0047          *    bottom pixel had alpha=170 and the top has alpha=102,
0048          *    each should contribute equally to the color of the
0049          *    resulting alpha=204 pixel, which is not possible in general]
0050          * Subtracting one helps the topfrac never be 256, which would
0051          * be bad.
0052          * On the other hand it means that we would get tfrac=-1 if the
0053          * top pixel is completely transparent, and we get a division
0054          * by zero if _both_ pixels are fully transparent. These cases
0055          * must be handled by all callers.
0056          *    More snooping in the Gimp sources reveal that it uses
0057          *    floating-point for its equivalent of tfrac when the
0058          *    bottom layer has an alpha channel. (alphify() macro
0059          *    in paint-funcs.c). What gives?
0060          */
0061     }
0062     return (alpha << ALPHA_SHIFT)
0063             + ((uint32_t)scaletable[  tfrac  ][255&(top>>RED_SHIFT  )] << RED_SHIFT   )
0064             + ((uint32_t)scaletable[  tfrac  ][255&(top>>GREEN_SHIFT)] << GREEN_SHIFT )
0065             + ((uint32_t)scaletable[  tfrac  ][255&(top>>BLUE_SHIFT )] << BLUE_SHIFT  )
0066             + ((uint32_t)scaletable[255^tfrac][255&(bot>>RED_SHIFT  )] << RED_SHIFT   )
0067             + ((uint32_t)scaletable[255^tfrac][255&(bot>>GREEN_SHIFT)] << GREEN_SHIFT )
0068             + ((uint32_t)scaletable[255^tfrac][255&(bot>>BLUE_SHIFT )] << BLUE_SHIFT  )
0069             ;
0070 }
0071 
0072 /* merge_normal() takes ownership of bot.
0073  * merge_normal() will share ownership of top.
0074  * Return: may be shared.
0075  */
0076 static struct Tile * __ATTRIBUTE__((noinline))
0077 merge_normal(struct Tile *bot, struct Tile *top)
0078 {
0079     unsigned i ;
0080     assertTileCompatibility(bot,top);
0081 
0082     /* See if there is an easy winner */
0083     if( (bot->summary & TILESUMMARY_ALLNULL) ||
0084             (top->summary & TILESUMMARY_ALLFULL) ) {
0085         freeTile(bot);
0086         return top ;
0087     }
0088     if( top->summary & TILESUMMARY_ALLNULL ) {
0089         freeTile(top);
0090         return bot ;
0091     }
0092 
0093     /* Try hard to make top win */
0094     for( i=0; ; i++ ) {
0095         if( i == top->count ) {
0096             freeTile(bot);
0097             return top ;
0098         }
0099         if( !(NULLALPHA(bot->pixels[i]) || FULLALPHA(top->pixels[i])) )
0100             break ;
0101     }
0102 
0103     INIT_SCALETABLE_IF( !(top->summary & TILESUMMARY_CRISP) );
0104 
0105     /* Otherwise bot wins, but is forever changed ... */
0106     if( (top->summary & TILESUMMARY_ALLNULL) == 0 ) {
0107         unsigned i ;
0108         invalidateSummary(bot,0);
0109         for( i=0 ; i < top->count ; i++ ) {
0110             if( !NULLALPHA(top->pixels[i]) ) {
0111                 if( FULLALPHA(top->pixels[i]) || NULLALPHA(bot->pixels[i]) )
0112                     bot->pixels[i] = top->pixels[i] ;
0113                 else
0114                     bot->pixels[i] = composite_one(bot->pixels[i],top->pixels[i]);
0115             }
0116         }
0117     }
0118     freeTile(top);
0119     return bot ;
0120 }
0121 
0122 #define exotic_combinator static inline unsigned __ATTRIBUTE__((const))
0123 
0124 
0125 
0126 exotic_combinator
0127 ucombine_ADDITION(uint8_t bot,uint8_t top)
0128 {
0129     return bot+top > 255 ? 255 : bot+top ;
0130 }
0131 
0132 exotic_combinator
0133 ucombine_SUBTRACT(uint8_t bot,uint8_t top)
0134 {
0135     return top>bot ? 0 : bot-top ;
0136 }
0137 
0138 exotic_combinator
0139 ucombine_LIGHTEN_ONLY(uint8_t bot,uint8_t top)
0140 {
0141     return top > bot ? top : bot ;
0142 }
0143 
0144 exotic_combinator
0145 ucombine_DARKEN_ONLY(uint8_t bot,uint8_t top)
0146 {
0147     return top < bot ? top : bot ;
0148 }
0149 
0150 exotic_combinator
0151 ucombine_DIFFERENCE(uint8_t bot,uint8_t top)
0152 {
0153     return top > bot ? top-bot : bot-top ;
0154 }
0155 
0156 exotic_combinator
0157 ucombine_MULTIPLY(uint8_t bot,uint8_t top)
0158 {
0159     return scaletable[bot][top] ;
0160 }
0161 
0162 exotic_combinator
0163 ucombine_DIVIDE(uint8_t bot,uint8_t top)
0164 {
0165     int result = (int)bot*256 / (1+top) ;
0166     return result >= 256 ? 255 : result ;
0167 }
0168 
0169 exotic_combinator
0170 ucombine_SCREEN(uint8_t bot,uint8_t top)
0171 {
0172     /* An inverted version of "multiply" */
0173     return 255 ^ scaletable[255-bot][255-top] ;
0174 }
0175 
0176 exotic_combinator
0177 ucombine_OVERLAY(uint8_t bot,uint8_t top)
0178 {
0179     return scaletable[bot][bot] +
0180             2*scaletable[top][scaletable[bot][255-bot]] ;
0181     /* This strange formula is equivalent to
0182      *   (1-top)*(bot^2) + top*(1-(1-top)^2)
0183      * that is, the top value is used to interpolate between
0184      * the self-multiply and the self-screen of the bottom.
0185      */
0186     /* Note: This is exactly what the "Soft light" effect also
0187      * does, though with different code in the Gimp.
0188      */
0189 }
0190 
0191 exotic_combinator
0192 ucombine_DODGE(uint8_t bot,uint8_t top)
0193 {
0194     return ucombine_DIVIDE(bot,255-top);
0195 }
0196 
0197 exotic_combinator
0198 ucombine_BURN(uint8_t bot,uint8_t top)
0199 {
0200     return 255 - ucombine_DIVIDE(255-bot,top);
0201 }
0202 
0203 exotic_combinator
0204 ucombine_HARDLIGHT(uint8_t bot,uint8_t top)
0205 {
0206     if( top >= 128 )
0207         return 255 ^ scaletable[255-bot][2*(255-top)] ;
0208     else
0209         return scaletable[bot][2*top];
0210     /* The code that implements "hardlight" in Gimp 2.2.10 has some
0211      * rounding errors, but this is undoubtedly what is meant.
0212      */
0213 }
0214 
0215 exotic_combinator
0216 ucombine_GRAIN_EXTRACT(uint8_t bot,uint8_t top)
0217 {
0218     int temp = (int)bot - (int)top + 128 ;
0219     return temp < 0 ? 0 : temp >= 256 ? 255 : temp ;
0220 }
0221 
0222 exotic_combinator
0223 ucombine_GRAIN_MERGE(uint8_t bot,uint8_t top)
0224 {
0225     int temp = (int)bot + (int)top - 128 ;
0226     return temp < 0 ? 0 : temp >= 256 ? 255 : temp ;
0227 }
0228 
0229 struct HSV {
0230     enum { HUE_RED_GREEN_BLUE,HUE_RED_BLUE_GREEN,HUE_BLUE_RED_GREEN,
0231            HUE_BLUE_GREEN_RED,HUE_GREEN_BLUE_RED,HUE_GREEN_RED_BLUE } hue;
0232     unsigned ch1, ch2, ch3 ;
0233 };
0234 
0235 static void
0236 RGBtoHSV(rgba rgb,struct HSV *hsv)
0237 {
0238     unsigned RED = (uint8_t)(rgb >> RED_SHIFT);
0239     unsigned GREEN = (uint8_t)(rgb >> GREEN_SHIFT);
0240     unsigned BLUE = (uint8_t)(rgb >> BLUE_SHIFT) ;
0241 #define HEXTANT(b,m,t) hsv->ch1 = b, hsv->ch2 = m, hsv->ch3 = t, \
0242     hsv->hue = HUE_ ## b ## _ ## m ## _ ## t
0243     if( GREEN <= RED )
0244         if( BLUE <= RED )
0245             if( GREEN <= BLUE )
0246                 HEXTANT(GREEN,BLUE,RED);
0247             else
0248                 HEXTANT(BLUE,GREEN,RED);
0249         else
0250             HEXTANT(GREEN,RED,BLUE);
0251     else if( BLUE <= RED )
0252         HEXTANT(BLUE,RED,GREEN);
0253     else if( BLUE <= GREEN )
0254         HEXTANT(RED,BLUE,GREEN);
0255     else
0256         HEXTANT(RED,GREEN,BLUE);
0257 #undef HEXTANT
0258 }
0259 
0260 /* merge_exotic() destructively updates bot.
0261  * merge_exotic() reads but does not free top.
0262  */
0263 static int __ATTRIBUTE__((noinline))
0264 merge_exotic(struct Tile *bot, const struct Tile *top,
0265              GimpLayerModeEffects mode)
0266 {
0267     unsigned i ;
0268     assertTileCompatibility(bot,top);
0269     if( (bot->summary & TILESUMMARY_ALLNULL) != 0 ) return XCF_OK;
0270     if( (top->summary & TILESUMMARY_ALLNULL) != 0 ) return XCF_OK;
0271     assert( bot->refcount == 1 );
0272     /* The transparency status of bot never changes */
0273 
0274     INIT_SCALETABLE_IF(1);
0275 
0276     for( i=0; i < top->count ; i++ ) {
0277         uint32_t RED, GREEN, BLUE ;
0278         if( NULLALPHA(bot->pixels[i]) || NULLALPHA(top->pixels[i]) )
0279             continue ;
0280 #define UNIFORM(mode) case GIMP_ ## mode ## _MODE: \
0281     RED   = ucombine_ ## mode (bot->pixels[i]>>RED_SHIFT  ,  \
0282     top->pixels[i]>>RED_SHIFT  ); \
0283     GREEN = ucombine_ ## mode (bot->pixels[i]>>GREEN_SHIFT,  \
0284     top->pixels[i]>>GREEN_SHIFT); \
0285     BLUE  = ucombine_ ## mode (bot->pixels[i]>>BLUE_SHIFT ,  \
0286     top->pixels[i]>>BLUE_SHIFT ); \
0287     break ;
0288         switch( mode ) {
0289         case GIMP_NORMAL_MODE:
0290         case GIMP_DISSOLVE_MODE:
0291             {
0292                 FatalUnexpected("Normal and Dissolve mode can't happen here!");
0293                 return XCF_ERROR;
0294             }
0295             UNIFORM(ADDITION);
0296             UNIFORM(SUBTRACT);
0297             UNIFORM(LIGHTEN_ONLY);
0298             UNIFORM(DARKEN_ONLY);
0299             UNIFORM(DIFFERENCE);
0300             UNIFORM(MULTIPLY);
0301             UNIFORM(DIVIDE);
0302             UNIFORM(SCREEN);
0303         case GIMP_SOFTLIGHT_MODE: /* A synonym for "overlay"! */
0304             UNIFORM(OVERLAY);
0305             UNIFORM(DODGE);
0306             UNIFORM(BURN);
0307             UNIFORM(HARDLIGHT);
0308             UNIFORM(GRAIN_EXTRACT);
0309             UNIFORM(GRAIN_MERGE);
0310         case GIMP_HUE_MODE:
0311         case GIMP_SATURATION_MODE:
0312         case GIMP_VALUE_MODE:
0313         case GIMP_COLOR_MODE:
0314         {
0315             static struct HSV hsvTop, hsvBot ;
0316             RGBtoHSV(top->pixels[i],&hsvTop);
0317             if( mode == GIMP_HUE_MODE && hsvTop.ch1 == hsvTop.ch3 )
0318                 continue ;
0319             RGBtoHSV(bot->pixels[i],&hsvBot);
0320             if( mode == GIMP_VALUE_MODE ) {
0321                 if( hsvBot.ch3 ) {
0322                     hsvBot.ch1 = (hsvBot.ch1*hsvTop.ch3 + hsvBot.ch3/2) / hsvBot.ch3;
0323                     hsvBot.ch2 = (hsvBot.ch2*hsvTop.ch3 + hsvBot.ch3/2) / hsvBot.ch3;
0324                     hsvBot.ch3 = hsvTop.ch3 ;
0325                 } else {
0326                     hsvBot.ch1 = hsvBot.ch2 = hsvBot.ch3 = hsvTop.ch3 ;
0327                 }
0328             } else {
0329                 unsigned mfNum, mfDenom ;
0330                 if( mode == GIMP_HUE_MODE || mode == GIMP_COLOR_MODE ) {
0331                     mfNum   = hsvTop.ch2-hsvTop.ch1 ;
0332                     mfDenom = hsvTop.ch3-hsvTop.ch1 ;
0333                     hsvBot.hue = hsvTop.hue ;
0334                 } else {
0335                     mfNum   = hsvBot.ch2-hsvBot.ch1 ;
0336                     mfDenom = hsvBot.ch3-hsvBot.ch1 ;
0337                 }
0338                 if( mode == GIMP_SATURATION_MODE ) {
0339                     if( hsvTop.ch3 == 0 )
0340                         hsvBot.ch1 = hsvBot.ch3 ; /* Black has no saturation */
0341                     else
0342                         hsvBot.ch1 = (hsvTop.ch1*hsvBot.ch3 + hsvTop.ch3/2) / hsvTop.ch3;
0343                 } else if( mode == GIMP_COLOR_MODE ) {
0344                     /* GIMP_COLOR_MODE works in HSL space instead of HSV. We must
0345                      * transfer H and S, keeping the L = ch1+ch3 of the bottom pixel,
0346                      * but the S we transfer works differently from the S in HSV.
0347                      */
0348                     unsigned L = hsvTop.ch1 + hsvTop.ch3 ;
0349                     unsigned sNum = hsvTop.ch3 - hsvTop.ch1 ;
0350                     unsigned sDenom = L < 256 ? L : 510-L ;
0351                     if( sDenom == 0 ) sDenom = 1 ; /* sNum will be 0 */
0352                     L = hsvBot.ch1 + hsvBot.ch3 ;
0353                     if( L < 256 ) {
0354                         /* Ideally we want to compute L/2 * (1-sNum/sDenom)
0355                          * But shuffle this a bit so we can use integer arithmetic.
0356                          * The "-1" in the rounding prevents us from ending up with
0357                          * ch1 > ch3.
0358                          */
0359                         hsvBot.ch1 = (L*(sDenom-sNum)+sDenom-1)/(2*sDenom);
0360                         hsvBot.ch3 = L - hsvBot.ch1 ;
0361                     } else {
0362                         /* Here our goal is 255 - (510-L)/2 * (1-sNum/sDenom) */
0363                         hsvBot.ch3 = 255 - ((510-L)*(sDenom-sNum)+sDenom-1)/(2*sDenom);
0364                         hsvBot.ch1 = L - hsvBot.ch3 ;
0365                     }
0366                     assert(hsvBot.ch3 <= 255);
0367                     assert(hsvBot.ch3 >= hsvBot.ch1);
0368                 }
0369                 if( mfDenom == 0 )
0370                     hsvBot.ch2 = hsvBot.ch1 ;
0371                 else
0372                     hsvBot.ch2 = hsvBot.ch1 +
0373                             (mfNum*(hsvBot.ch3-hsvBot.ch1) + mfDenom/2) / mfDenom ;
0374             }
0375             switch( hsvBot.hue ) {
0376 #define HEXTANT(b,m,t) case HUE_ ## b ## _ ## m ## _ ## t : \
0377     b = hsvBot.ch1; m = hsvBot.ch2; t = hsvBot.ch3; break;
0378             HEXTANT(RED,GREEN,BLUE);
0379             HEXTANT(RED,BLUE,GREEN);
0380             HEXTANT(BLUE,RED,GREEN);
0381             HEXTANT(BLUE,GREEN,RED);
0382             HEXTANT(GREEN,BLUE,RED);
0383             HEXTANT(GREEN,RED,BLUE);
0384 #undef HEXTANT
0385             default: {
0386 
0387                     FatalUnexpected("Hue hextant is %d", hsvBot.hue);
0388                     return XCF_ERROR;
0389                 }
0390             }
0391             break ;
0392         }
0393         default:
0394             {
0395                 FatalUnsupportedXCF(_("'%s' layer mode"),
0396                                 _(showGimpLayerModeEffects(mode)));
0397                 return XCF_ERROR;
0398             }
0399         }
0400         if( FULLALPHA(bot->pixels[i] & top->pixels[i]) )
0401             bot->pixels[i] = (bot->pixels[i] & (255 << ALPHA_SHIFT)) +
0402                     (RED << RED_SHIFT) +
0403                     (GREEN << GREEN_SHIFT) +
0404                     (BLUE << BLUE_SHIFT) ;
0405         else {
0406             rgba bp = bot->pixels[i] ;
0407             /* In a sane world, the alpha of the top pixel would simply be
0408              * used to interpolate linearly between the bottom pixel's base
0409              * color and the effect-computed color.
0410              * But no! What the Gimp actually does is empirically
0411              * described by the following (which borrows code from
0412              * composite_one() that makes no theoretical sense here):
0413              */
0414             unsigned tfrac = ALPHA(top->pixels[i]) ;
0415             if( !FULLALPHA(bp) ) {
0416                 unsigned pseudotop = (tfrac < ALPHA(bp) ? tfrac : ALPHA(bp));
0417                 unsigned alpha = 255 ^ scaletable[255-ALPHA(bp)][255-pseudotop] ;
0418                 tfrac = (256*pseudotop - 1) / alpha ;
0419             }
0420             bot->pixels[i] = (bp & (255 << ALPHA_SHIFT)) +
0421                     ((rgba)scaletable[  tfrac  ][  RED                ] << RED_SHIFT  ) +
0422                     ((rgba)scaletable[  tfrac  ][  GREEN              ] << GREEN_SHIFT) +
0423                     ((rgba)scaletable[  tfrac  ][  BLUE               ] << BLUE_SHIFT ) +
0424                     ((rgba)scaletable[255^tfrac][255&(bp>>RED_SHIFT  )] << RED_SHIFT  ) +
0425                     ((rgba)scaletable[255^tfrac][255&(bp>>GREEN_SHIFT)] << GREEN_SHIFT) +
0426                     ((rgba)scaletable[255^tfrac][255&(bp>>BLUE_SHIFT )] << BLUE_SHIFT ) ;
0427         }
0428     }
0429     return XCF_OK;
0430 }
0431 
0432 static void
0433 dissolveTile(struct Tile *tile)
0434 {
0435     unsigned i ;
0436     summary_t summary ;
0437     assert( tile->refcount == 1 );
0438     if( (tile->summary & TILESUMMARY_CRISP) )
0439         return ;
0440     summary = TILESUMMARY_UPTODATE + TILESUMMARY_ALLNULL
0441             + TILESUMMARY_ALLFULL + TILESUMMARY_CRISP ;
0442     for( i = 0 ; i < tile->count ; i++ ) {
0443         if( FULLALPHA(tile->pixels[i]) )
0444             summary &= ~TILESUMMARY_ALLNULL ;
0445         else if ( NULLALPHA(tile->pixels[i]) )
0446             summary &= ~TILESUMMARY_ALLFULL ;
0447         else if( ALPHA(tile->pixels[i]) > rand() % 0xFF ) {
0448             tile->pixels[i] |= 255 << ALPHA_SHIFT ;
0449             summary &= ~TILESUMMARY_ALLNULL ;
0450         } else {
0451             tile->pixels[i] = 0 ;
0452             summary &= ~TILESUMMARY_ALLFULL ;
0453         }
0454     }
0455     tile->summary = summary ;
0456 }
0457 
0458 static void
0459 roundAlpha(struct Tile *tile)
0460 {
0461     unsigned i ;
0462     summary_t summary ;
0463     assert( tile->refcount == 1 );
0464     if( (tile->summary & TILESUMMARY_CRISP) )
0465         return ;
0466     summary = TILESUMMARY_UPTODATE + TILESUMMARY_ALLNULL
0467             + TILESUMMARY_ALLFULL + TILESUMMARY_CRISP ;
0468     for( i = 0 ; i < tile->count ; i++ ) {
0469         if( ALPHA(tile->pixels[i]) >= 128 ) {
0470             tile->pixels[i] |= 255 << ALPHA_SHIFT ;
0471             summary &= ~TILESUMMARY_ALLNULL ;
0472         } else {
0473             tile->pixels[i] = 0 ;
0474             summary &= ~TILESUMMARY_ALLFULL ;
0475         }
0476     }
0477     tile->summary = summary ;
0478 }
0479 
0480 /* flattenTopdown() shares ownership of top.
0481  * The return value may be a shared tile.
0482  */
0483 static struct Tile *
0484 flattenTopdown(struct FlattenSpec *spec, struct Tile *top,
0485                unsigned nlayers, const struct rect *where)
0486 {
0487     struct Tile *tile = 0;
0488 
0489     while( nlayers-- ) {
0490         if( tileSummary(top) & TILESUMMARY_ALLFULL ) {
0491             if (tile) {
0492                 /* it should always be not null here though */
0493                 freeTile(tile);
0494             }
0495             return top ;
0496         }
0497         if( !spec->layers[nlayers].isVisible )
0498             continue ;
0499 
0500         freeTile(tile);
0501         tile = getLayerTile(&spec->layers[nlayers],where);
0502         if (tile == XCF_PTR_EMPTY) {
0503             return XCF_PTR_EMPTY;
0504         }
0505 
0506         if( tile->summary & TILESUMMARY_ALLNULL )
0507             continue ; /* Simulate a tail call */
0508 
0509         switch( spec->layers[nlayers].mode ) {
0510         case GIMP_NORMAL_NOPARTIAL_MODE:
0511             roundAlpha(tile) ;
0512             /* Falls through */
0513         case GIMP_DISSOLVE_MODE:
0514             dissolveTile(tile);
0515             /* Falls through */
0516         case GIMP_NORMAL_MODE:
0517             top = merge_normal(tile,top);
0518             break ;
0519         default:
0520         {
0521             struct Tile *below, *above ;
0522             unsigned i ;
0523             if( !(top->summary & TILESUMMARY_ALLNULL) ) {
0524                 rgba tile_or = 0 ;
0525                 invalidateSummary(tile,0);
0526                 for( i=0; i<top->count; i++ )
0527                     if( FULLALPHA(top->pixels[i]) )
0528                         tile->pixels[i] = 0 ;
0529                     else
0530                         tile_or |= tile->pixels[i] ;
0531                 /* If the tile only has pixels that will be covered by 'top' anyway,
0532                  * forget it anyway.
0533                  */
0534                 if( ALPHA(tile_or) == 0 ) {
0535                     freeTile(tile);
0536                     break ; /* from the switch, which will continue the while */
0537                 }
0538             }
0539             /* Create a dummy top for the layers below this */
0540             if( top->summary & TILESUMMARY_CRISP ) {
0541                 above = forkTile(top);
0542                 if(above == XCF_PTR_EMPTY) {
0543                     freeTile(tile);
0544                     return XCF_PTR_EMPTY;
0545                 }
0546             } else {
0547                 summary_t summary = TILESUMMARY_ALLNULL ;
0548                 above = newTile(*where);
0549                 for( i=0; i<top->count; i++ )
0550                     if( FULLALPHA(top->pixels[i]) ) {
0551                         above->pixels[i] = -1 ;
0552                         summary = 0 ;
0553                     } else
0554                         above->pixels[i] = 0 ;
0555                 above->summary = TILESUMMARY_UPTODATE + TILESUMMARY_CRISP + summary;
0556             }
0557             below = flattenTopdown(spec, above, nlayers, where);
0558             if (below == XCF_PTR_EMPTY) {
0559                 freeTile(tile);
0560                 return XCF_PTR_EMPTY;
0561             }
0562             if( below->refcount > 1 ) {
0563                 if (below != top) {
0564                     freeTile(tile);
0565                     return XCF_PTR_EMPTY;
0566                 }
0567                 /* This can only happen if 'below' is a copy of 'top'
0568                  * THROUGH 'above', which in turn means that none of all
0569                  * this is visible after all. So just free it and return 'top'.
0570                  */
0571                 freeTile(below);
0572                 freeTile(tile);
0573                 return top ;
0574             }
0575             if (merge_exotic(below,tile,spec->layers[nlayers].mode) != XCF_OK) {
0576                 return XCF_PTR_EMPTY;
0577             }
0578             freeTile(tile);
0579             top = merge_normal(below,top);
0580             return top ;
0581         }
0582         }
0583     }
0584     freeTile(tile);
0585     return top ;
0586 }
0587 
0588 static int
0589 addBackground(struct FlattenSpec *spec, struct Tile *tile, unsigned ncols)
0590 {
0591     unsigned i ;
0592 
0593     if( tileSummary(tile) & TILESUMMARY_ALLFULL )
0594         return XCF_OK;
0595 
0596     switch( spec->partial_transparency_mode ) {
0597     case FORBID_PARTIAL_TRANSPARENCY:
0598         if( !(tileSummary(tile) & TILESUMMARY_CRISP) ) {
0599             FatalGeneric(102,_("Flattened image has partially transparent pixels"));
0600             return XCF_ERROR;
0601         }
0602         break ;
0603     case DISSOLVE_PARTIAL_TRANSPARENCY:
0604         dissolveTile(tile);
0605         break ;
0606     case ALLOW_PARTIAL_TRANSPARENCY:
0607     case PARTIAL_TRANSPARENCY_IMPOSSIBLE:
0608         break ;
0609     }
0610 
0611     if( spec->default_pixel == CHECKERED_BACKGROUND ) {
0612         INIT_SCALETABLE_IF( !(tile->summary & TILESUMMARY_CRISP ) );
0613         for( i=0; i<tile->count; i++ )
0614             if( !FULLALPHA(tile->pixels[i]) ) {
0615                 rgba fillwith = ((i/ncols)^(i%ncols))&8 ? 0x66 : 0x99 ;
0616                 fillwith = graytable[fillwith] + (255 << ALPHA_SHIFT) ;
0617                 if( NULLALPHA(tile->pixels[i]) )
0618                     tile->pixels[i] = fillwith ;
0619                 else
0620                     tile->pixels[i] = composite_one(fillwith,tile->pixels[i]);
0621             }
0622         tile->summary = TILESUMMARY_UPTODATE +
0623                 TILESUMMARY_ALLFULL + TILESUMMARY_CRISP ;
0624         return XCF_OK;
0625     }
0626     if( !FULLALPHA(spec->default_pixel) )  return XCF_OK;
0627     if( tileSummary(tile) & TILESUMMARY_ALLNULL ) {
0628         fillTile(tile,spec->default_pixel);
0629     } else {
0630         INIT_SCALETABLE_IF( !(tile->summary & TILESUMMARY_CRISP) );
0631         for( i=0; i<tile->count; i++ )
0632             if( NULLALPHA(tile->pixels[i]) )
0633                 tile->pixels[i] = spec->default_pixel ;
0634             else if( FULLALPHA(tile->pixels[i]) )
0635                 ;
0636             else
0637                 tile->pixels[i] = composite_one(spec->default_pixel,tile->pixels[i]);
0638 
0639         tile->summary = TILESUMMARY_UPTODATE +
0640                 TILESUMMARY_ALLFULL + TILESUMMARY_CRISP ;
0641     }
0642     return XCF_OK;
0643 }
0644 
0645 int
0646 flattenIncrementally(struct FlattenSpec *spec,lineCallback callback)
0647 {
0648     rgba *rows[TILE_HEIGHT] ;
0649     unsigned i, y, nrows, ncols ;
0650     struct rect where ;
0651     struct Tile *tile ;
0652     static struct Tile toptile ;
0653 
0654     toptile.count = TILE_HEIGHT * TILE_WIDTH ;
0655     fillTile(&toptile,0);
0656 
0657     for( where.t = spec->dim.c.t; where.t < spec->dim.c.b; where.t=where.b ) {
0658         where.b = TILE_TOP(where.t)+TILE_HEIGHT ;
0659         if( where.b > spec->dim.c.b ) where.b = spec->dim.c.b ;
0660         nrows = where.b - where.t ;
0661         for( y = 0; y < nrows ; y++ )
0662             rows[y] = xcfmalloc(4*(spec->dim.c.r-spec->dim.c.l));
0663 
0664         for( where.l = spec->dim.c.l; where.l < spec->dim.c.r; where.l=where.r ) {
0665             where.r = TILE_LEFT(where.l)+TILE_WIDTH ;
0666             if( where.r > spec->dim.c.r ) where.r = spec->dim.c.r ;
0667             ncols = where.r - where.l ;
0668 
0669             toptile.count = ncols * nrows ;
0670             toptile.refcount = 2 ; /* For bug checking */
0671             assert( toptile.summary == TILESUMMARY_UPTODATE +
0672                     TILESUMMARY_ALLNULL + TILESUMMARY_CRISP );
0673             tile = flattenTopdown(spec,&toptile,spec->numLayers,&where) ;
0674             if (tile == XCF_PTR_EMPTY) {
0675                 return XCF_ERROR;
0676             }
0677             toptile.refcount-- ; /* addBackground may change destructively */
0678             if (addBackground(spec,tile,ncols) != XCF_OK) {
0679                 return XCF_ERROR;
0680             }
0681 
0682             for( i = 0 ; i < tile->count ; i++ )
0683                 if( NULLALPHA(tile->pixels[i]) )
0684                     tile->pixels[i] = 0 ;
0685             for( y = 0 ; y < nrows ; y++ )
0686                 memcpy(rows[y] + (where.l - spec->dim.c.l),
0687                        tile->pixels + y * ncols, ncols*4);
0688 
0689             if( tile == &toptile ) {
0690                 fillTile(&toptile,0);
0691             } else {
0692                 freeTile(tile);
0693             }
0694         }
0695         for( y = 0 ; y < nrows ; y++ )
0696             callback(spec->dim.width,rows[y]);
0697     }
0698     return XCF_OK;
0699 }
0700 
0701 static rgba **collectPointer ;
0702 
0703 static void
0704 collector(unsigned num,rgba *row)
0705 {
0706     (void)num;
0707     *collectPointer++ = row ;
0708 }
0709 
0710 rgba **
0711 flattenAll(struct FlattenSpec *spec)
0712 {
0713     rgba **rows = xcfmalloc(spec->dim.height * sizeof(rgba*));
0714     if( verboseFlag )
0715         fprintf(stderr,_("Flattening image ..."));
0716     collectPointer = rows ;
0717     if (flattenIncrementally(spec,collector) != XCF_OK) {
0718         xcffree(rows);
0719         collectPointer = XCF_PTR_EMPTY;
0720         return XCF_PTR_EMPTY;
0721     }
0722     if( verboseFlag )
0723         fprintf(stderr,"\n");
0724     return rows ;
0725 }
0726 
0727 void
0728 shipoutWithCallback(struct FlattenSpec *spec, rgba **pixels,
0729                     lineCallback callback)
0730 {
0731     unsigned i ;
0732     for( i = 0; i < spec->dim.height; i++ ) {
0733         callback(spec->dim.width,pixels[i]);
0734     }
0735     xcffree(pixels);
0736 }