Skip to main content

atmos/os_lib/webp/
lossless_transform.rs

1use alloc::vec;
2use alloc::vec::Vec;
3use core::ops::Range;
4
5use crate::os_lib::webp::decoder::DecodingError;
6
7use super::lossless::subsample_size;
8
9#[derive(Debug, Clone)]
10pub(crate) enum TransformType {
11    PredictorTransform {
12        size_bits: u8,
13        predictor_data: Vec<u8>,
14    },
15    ColorTransform {
16        size_bits: u8,
17        transform_data: Vec<u8>,
18    },
19    SubtractGreen,
20    ColorIndexingTransform {
21        table_size: u16,
22        table_data: Vec<u8>,
23    },
24}
25
26pub(crate) fn apply_predictor_transform(
27    image_data: &mut [u8],
28    width: u16,
29    height: u16,
30    size_bits: u8,
31    predictor_data: &[u8],
32) -> Result<(), DecodingError> {
33    let block_xsize = usize::from(subsample_size(width, size_bits));
34    let width = usize::from(width);
35    let height = usize::from(height);
36
37    // Handle top and left borders specially. This involves ignoring mode and using specific
38    // predictors for each.
39    image_data[3] = image_data[3].wrapping_add(255);
40    apply_predictor_transform_1(image_data, 4..width * 4, width);
41    for y in 1..height {
42        for i in 0..4 {
43            image_data[y * width * 4 + i] =
44                image_data[y * width * 4 + i].wrapping_add(image_data[(y - 1) * width * 4 + i]);
45        }
46    }
47
48    for y in 1..height {
49        for block_x in 0..block_xsize {
50            let block_index = (y >> size_bits) * block_xsize + block_x;
51            let predictor = predictor_data[block_index * 4 + 1];
52            let start_index = (y * width + (block_x << size_bits).max(1)) * 4;
53            let end_index = (y * width + ((block_x + 1) << size_bits).min(width)) * 4;
54
55            match predictor {
56                0 => apply_predictor_transform_0(image_data, start_index..end_index, width),
57                1 => apply_predictor_transform_1(image_data, start_index..end_index, width),
58                2 => apply_predictor_transform_2(image_data, start_index..end_index, width),
59                3 => apply_predictor_transform_3(image_data, start_index..end_index, width),
60                4 => apply_predictor_transform_4(image_data, start_index..end_index, width),
61                5 => apply_predictor_transform_5(image_data, start_index..end_index, width),
62                6 => apply_predictor_transform_6(image_data, start_index..end_index, width),
63                7 => apply_predictor_transform_7(image_data, start_index..end_index, width),
64                8 => apply_predictor_transform_8(image_data, start_index..end_index, width),
65                9 => apply_predictor_transform_9(image_data, start_index..end_index, width),
66                10 => apply_predictor_transform_10(image_data, start_index..end_index, width),
67                11 => apply_predictor_transform_11(image_data, start_index..end_index, width),
68                12 => apply_predictor_transform_12(image_data, start_index..end_index, width),
69                13 => apply_predictor_transform_13(image_data, start_index..end_index, width),
70                _ => {}
71            }
72        }
73    }
74
75    Ok(())
76}
77pub fn apply_predictor_transform_0(image_data: &mut [u8], range: Range<usize>, _width: usize) {
78    assert!(range.end <= image_data.len());
79    let mut i = range.start + 3;
80    while i < range.end {
81        image_data[i] = image_data[i].wrapping_add(0xff);
82        i += 4;
83    }
84}
85pub fn apply_predictor_transform_1(image_data: &mut [u8], range: Range<usize>, _width: usize) {
86    assert!(range.end <= image_data.len());
87    let mut i = range.start;
88    while i < range.end {
89        image_data[i] = image_data[i].wrapping_add(image_data[i - 4]);
90        i += 1;
91    }
92}
93pub fn apply_predictor_transform_2(image_data: &mut [u8], range: Range<usize>, width: usize) {
94    assert!(range.end <= image_data.len());
95    let mut i = range.start;
96    while i < range.end {
97        image_data[i] = image_data[i].wrapping_add(image_data[i - width * 4]);
98        i += 1;
99    }
100}
101pub fn apply_predictor_transform_3(image_data: &mut [u8], range: Range<usize>, width: usize) {
102    assert!(range.end <= image_data.len());
103    let mut i = range.start;
104    while i < range.end {
105        image_data[i] = image_data[i].wrapping_add(image_data[i - width * 4 + 4]);
106        i += 1;
107    }
108}
109pub fn apply_predictor_transform_4(image_data: &mut [u8], range: Range<usize>, width: usize) {
110    assert!(range.end <= image_data.len());
111    let mut i = range.start;
112    while i < range.end {
113        image_data[i] = image_data[i].wrapping_add(image_data[i - width * 4 - 4]);
114        i += 1;
115    }
116}
117pub fn apply_predictor_transform_5(image_data: &mut [u8], range: Range<usize>, width: usize) {
118    let (old, current) = image_data[..range.end].split_at_mut(range.start);
119
120    let mut prev: [u8; 4] = old[range.start - 4..][..4].try_into().unwrap();
121    let top_right = &old[range.start - width * 4 + 4..];
122    let top = &old[range.start - width * 4..];
123
124    for ((chunk, tr), t) in current
125        .chunks_exact_mut(4)
126        .zip(top_right.chunks_exact(4))
127        .zip(top.chunks_exact(4))
128    {
129        prev = [
130            chunk[0].wrapping_add(average2_autovec(average2_autovec(prev[0], tr[0]), t[0])),
131            chunk[1].wrapping_add(average2_autovec(average2_autovec(prev[1], tr[1]), t[1])),
132            chunk[2].wrapping_add(average2_autovec(average2_autovec(prev[2], tr[2]), t[2])),
133            chunk[3].wrapping_add(average2_autovec(average2_autovec(prev[3], tr[3]), t[3])),
134        ];
135        chunk.copy_from_slice(&prev);
136    }
137}
138pub fn apply_predictor_transform_6(image_data: &mut [u8], range: Range<usize>, width: usize) {
139    assert!(range.end <= image_data.len());
140    let mut i = range.start;
141    while i < range.end {
142        image_data[i] =
143            image_data[i].wrapping_add(average2(image_data[i - 4], image_data[i - width * 4 - 4]));
144        i += 1;
145    }
146}
147pub fn apply_predictor_transform_7(image_data: &mut [u8], range: Range<usize>, width: usize) {
148    let (old, current) = image_data[..range.end].split_at_mut(range.start);
149
150    let mut prev: [u8; 4] = old[range.start - 4..][..4].try_into().unwrap();
151    let top = &old[range.start - width * 4..][..(range.end - range.start)];
152
153    let mut current_chunks = current.chunks_exact_mut(64);
154    let mut top_chunks = top.chunks_exact(64);
155
156    for (current, top) in (&mut current_chunks).zip(&mut top_chunks) {
157        for (chunk, t) in current.chunks_exact_mut(4).zip(top.chunks_exact(4)) {
158            prev = [
159                chunk[0].wrapping_add(average2_autovec(prev[0], t[0])),
160                chunk[1].wrapping_add(average2_autovec(prev[1], t[1])),
161                chunk[2].wrapping_add(average2_autovec(prev[2], t[2])),
162                chunk[3].wrapping_add(average2_autovec(prev[3], t[3])),
163            ];
164            chunk.copy_from_slice(&prev);
165        }
166    }
167    for (chunk, t) in current_chunks
168        .into_remainder()
169        .chunks_exact_mut(4)
170        .zip(top_chunks.remainder().chunks_exact(4))
171    {
172        prev = [
173            chunk[0].wrapping_add(average2_autovec(prev[0], t[0])),
174            chunk[1].wrapping_add(average2_autovec(prev[1], t[1])),
175            chunk[2].wrapping_add(average2_autovec(prev[2], t[2])),
176            chunk[3].wrapping_add(average2_autovec(prev[3], t[3])),
177        ];
178        chunk.copy_from_slice(&prev);
179    }
180}
181pub fn apply_predictor_transform_8(image_data: &mut [u8], range: Range<usize>, width: usize) {
182    assert!(range.end <= image_data.len());
183    let mut i = range.start;
184    while i < range.end {
185        image_data[i] = image_data[i].wrapping_add(average2(
186            image_data[i - width * 4 - 4],
187            image_data[i - width * 4],
188        ));
189        i += 1;
190    }
191}
192pub fn apply_predictor_transform_9(image_data: &mut [u8], range: Range<usize>, width: usize) {
193    assert!(range.end <= image_data.len());
194    let mut i = range.start;
195    while i < range.end {
196        image_data[i] = image_data[i].wrapping_add(average2(
197            image_data[i - width * 4],
198            image_data[i - width * 4 + 4],
199        ));
200        i += 1;
201    }
202}
203pub fn apply_predictor_transform_10(image_data: &mut [u8], range: Range<usize>, width: usize) {
204    let (old, current) = image_data[..range.end].split_at_mut(range.start);
205    let mut prev: [u8; 4] = old[range.start - 4..][..4].try_into().unwrap();
206
207    let top_left = &old[range.start - width * 4 - 4..];
208    let top = &old[range.start - width * 4..];
209    let top_right = &old[range.start - width * 4 + 4..];
210
211    for (((chunk, tl), t), tr) in current
212        .chunks_exact_mut(4)
213        .zip(top_left.chunks_exact(4))
214        .zip(top.chunks_exact(4))
215        .zip(top_right.chunks_exact(4))
216    {
217        prev = [
218            chunk[0].wrapping_add(average2(average2(prev[0], tl[0]), average2(t[0], tr[0]))),
219            chunk[1].wrapping_add(average2(average2(prev[1], tl[1]), average2(t[1], tr[1]))),
220            chunk[2].wrapping_add(average2(average2(prev[2], tl[2]), average2(t[2], tr[2]))),
221            chunk[3].wrapping_add(average2(average2(prev[3], tl[3]), average2(t[3], tr[3]))),
222        ];
223        chunk.copy_from_slice(&prev);
224    }
225}
226pub fn apply_predictor_transform_11(image_data: &mut [u8], range: Range<usize>, width: usize) {
227    let (old, current) = image_data[..range.end].split_at_mut(range.start);
228    let top = &old[range.start - width * 4..];
229
230    let mut l = [
231        i16::from(old[range.start - 4]),
232        i16::from(old[range.start - 3]),
233        i16::from(old[range.start - 2]),
234        i16::from(old[range.start - 1]),
235    ];
236    let mut tl = [
237        i16::from(old[range.start - width * 4 - 4]),
238        i16::from(old[range.start - width * 4 - 3]),
239        i16::from(old[range.start - width * 4 - 2]),
240        i16::from(old[range.start - width * 4 - 1]),
241    ];
242
243    for (chunk, top) in current.chunks_exact_mut(4).zip(top.chunks_exact(4)) {
244        let t = [
245            i16::from(top[0]),
246            i16::from(top[1]),
247            i16::from(top[2]),
248            i16::from(top[3]),
249        ];
250
251        let mut predict_left = 0;
252        let mut predict_top = 0;
253        for i in 0..4 {
254            let predict = l[i] + t[i] - tl[i];
255            predict_left += i16::abs(predict - l[i]);
256            predict_top += i16::abs(predict - t[i]);
257        }
258
259        if predict_left < predict_top {
260            chunk.copy_from_slice(&[
261                chunk[0].wrapping_add(l[0] as u8),
262                chunk[1].wrapping_add(l[1] as u8),
263                chunk[2].wrapping_add(l[2] as u8),
264                chunk[3].wrapping_add(l[3] as u8),
265            ]);
266        } else {
267            chunk.copy_from_slice(&[
268                chunk[0].wrapping_add(t[0] as u8),
269                chunk[1].wrapping_add(t[1] as u8),
270                chunk[2].wrapping_add(t[2] as u8),
271                chunk[3].wrapping_add(t[3] as u8),
272            ]);
273        }
274
275        tl = t;
276        l = [
277            i16::from(chunk[0]),
278            i16::from(chunk[1]),
279            i16::from(chunk[2]),
280            i16::from(chunk[3]),
281        ];
282    }
283}
284pub fn apply_predictor_transform_12(image_data: &mut [u8], range: Range<usize>, width: usize) {
285    let (old, current) = image_data[..range.end].split_at_mut(range.start);
286    let mut prev: [u8; 4] = old[range.start - 4..][..4].try_into().unwrap();
287
288    let top_left = &old[range.start - width * 4 - 4..];
289    let top = &old[range.start - width * 4..];
290
291    for ((chunk, tl), t) in current
292        .chunks_exact_mut(4)
293        .zip(top_left.chunks_exact(4))
294        .zip(top.chunks_exact(4))
295    {
296        prev = [
297            chunk[0].wrapping_add(clamp_add_subtract_full(
298                i16::from(prev[0]),
299                i16::from(t[0]),
300                i16::from(tl[0]),
301            )),
302            chunk[1].wrapping_add(clamp_add_subtract_full(
303                i16::from(prev[1]),
304                i16::from(t[1]),
305                i16::from(tl[1]),
306            )),
307            chunk[2].wrapping_add(clamp_add_subtract_full(
308                i16::from(prev[2]),
309                i16::from(t[2]),
310                i16::from(tl[2]),
311            )),
312            chunk[3].wrapping_add(clamp_add_subtract_full(
313                i16::from(prev[3]),
314                i16::from(t[3]),
315                i16::from(tl[3]),
316            )),
317        ];
318        chunk.copy_from_slice(&prev);
319    }
320}
321pub fn apply_predictor_transform_13(image_data: &mut [u8], range: Range<usize>, width: usize) {
322    let (old, current) = image_data[..range.end].split_at_mut(range.start);
323    let mut prev: [u8; 4] = old[range.start - 4..][..4].try_into().unwrap();
324
325    let top_left = &old[range.start - width * 4 - 4..][..(range.end - range.start)];
326    let top = &old[range.start - width * 4..][..(range.end - range.start)];
327
328    for ((chunk, tl), t) in current
329        .chunks_exact_mut(4)
330        .zip(top_left.chunks_exact(4))
331        .zip(top.chunks_exact(4))
332    {
333        prev = [
334            chunk[0].wrapping_add(clamp_add_subtract_half(
335                (i16::from(prev[0]) + i16::from(t[0])) / 2,
336                i16::from(tl[0]),
337            )),
338            chunk[1].wrapping_add(clamp_add_subtract_half(
339                (i16::from(prev[1]) + i16::from(t[1])) / 2,
340                i16::from(tl[1]),
341            )),
342            chunk[2].wrapping_add(clamp_add_subtract_half(
343                (i16::from(prev[2]) + i16::from(t[2])) / 2,
344                i16::from(tl[2]),
345            )),
346            chunk[3].wrapping_add(clamp_add_subtract_half(
347                (i16::from(prev[3]) + i16::from(t[3])) / 2,
348                i16::from(tl[3]),
349            )),
350        ];
351        chunk.copy_from_slice(&prev);
352    }
353}
354
355pub(crate) fn apply_color_transform(
356    image_data: &mut [u8],
357    width: u16,
358    size_bits: u8,
359    transform_data: &[u8],
360) {
361    let block_xsize = usize::from(subsample_size(width, size_bits));
362    let width = usize::from(width);
363
364    for (y, row) in image_data.chunks_exact_mut(width * 4).enumerate() {
365        let row_transform_data_start = (y >> size_bits) * block_xsize * 4;
366        // the length of block_tf_data should be `block_xsize * 4`, so we could slice it with [..block_xsize * 4]
367        // but there is no point - `.zip()` runs until either of the iterators is consumed,
368        // so the extra slicing operation would be doing more work for no reason
369        let row_tf_data = &transform_data[row_transform_data_start..];
370
371        for (block, transform) in row
372            .chunks_mut(4 << size_bits)
373            .zip(row_tf_data.chunks_exact(4))
374        {
375            let red_to_blue = transform[0];
376            let green_to_blue = transform[1];
377            let green_to_red = transform[2];
378
379            for pixel in block.chunks_exact_mut(4) {
380                let green = u32::from(pixel[1]);
381                let mut temp_red = u32::from(pixel[0]);
382                let mut temp_blue = u32::from(pixel[2]);
383
384                temp_red += color_transform_delta(green_to_red as i8, green as i8);
385                temp_blue += color_transform_delta(green_to_blue as i8, green as i8);
386                temp_blue += color_transform_delta(red_to_blue as i8, temp_red as i8);
387
388                pixel[0] = (temp_red & 0xff) as u8;
389                pixel[2] = (temp_blue & 0xff) as u8;
390            }
391        }
392    }
393}
394
395pub(crate) fn apply_subtract_green_transform(image_data: &mut [u8]) {
396    for pixel in image_data.chunks_exact_mut(4) {
397        pixel[0] = pixel[0].wrapping_add(pixel[1]);
398        pixel[2] = pixel[2].wrapping_add(pixel[1]);
399    }
400}
401
402pub(crate) fn apply_color_indexing_transform(
403    image_data: &mut [u8],
404    width: u16,
405    height: u16,
406    table_size: u16,
407    table_data: &[u8],
408) {
409    assert!(table_size > 0);
410    if table_size > 16 {
411        // convert the table of colors into a Vec of color values that can be directly indexed
412        let mut table: Vec<[u8; 4]> = table_data
413            .chunks_exact(4)
414            // convince the compiler that each chunk is 4 bytes long, important for optimizations in the loop below
415            .map(|c| TryInto::<[u8; 4]>::try_into(c).unwrap())
416            .collect();
417        // pad the table to 256 values if it's smaller than that so we could index into it by u8 without bounds checks
418        // also required for correctness: WebP spec requires out-of-bounds indices to be treated as [0,0,0,0]
419        table.resize(256, [0; 4]);
420        // convince the compiler that the length of the table is 256 to avoid bounds checks in the loop below
421        let table: &[[u8; 4]; 256] = table.as_slice().try_into().unwrap();
422
423        for pixel in image_data.chunks_exact_mut(4) {
424            // Index is in G channel.
425            // WebP format encodes ARGB pixels, but we permute to RGBA immediately after reading from the bitstream.
426            pixel.copy_from_slice(&table[pixel[1] as usize]);
427        }
428    } else {
429        // table_size_u16 is 1 to 16
430        let table_size = table_size as u8;
431
432        // Dispatch to specialized implementation for each table size band for performance.
433        // Otherwise the compiler doesn't know the size of our copies
434        // and ends up calling out to memmove for every pixel even though a single load is sufficient.
435        if table_size <= 2 {
436            // Max 2 colors, 1 bit per pixel index -> W_BITS = 3
437            const W_BITS_VAL: u8 = 3;
438            // EXP_ENTRY_SIZE is 4 bytes/pixel * (1 << W_BITS_VAL) pixels/entry
439            const EXP_ENTRY_SIZE_VAL: usize = 4 * (1 << W_BITS_VAL); // 4 * 8 = 32
440            apply_color_indexing_transform_small_table::<W_BITS_VAL, EXP_ENTRY_SIZE_VAL>(
441                image_data, width, height, table_size, table_data,
442            );
443        } else if table_size <= 4 {
444            // Max 4 colors, 2 bits per pixel index -> W_BITS = 2
445            const W_BITS_VAL: u8 = 2;
446            const EXP_ENTRY_SIZE_VAL: usize = 4 * (1 << W_BITS_VAL); // 4 * 4 = 16
447            apply_color_indexing_transform_small_table::<W_BITS_VAL, EXP_ENTRY_SIZE_VAL>(
448                image_data, width, height, table_size, table_data,
449            );
450        } else {
451            // Max 16 colors (5 to 16), 4 bits per pixel index -> W_BITS = 1
452            // table_size_u16 must be <= 16 here
453            const W_BITS_VAL: u8 = 1;
454            const EXP_ENTRY_SIZE_VAL: usize = 4 * (1 << W_BITS_VAL); // 4 * 2 = 8
455            apply_color_indexing_transform_small_table::<W_BITS_VAL, EXP_ENTRY_SIZE_VAL>(
456                image_data, width, height, table_size, table_data,
457            );
458        }
459    }
460}
461
462// Helper function with const generics for W_BITS and EXP_ENTRY_SIZE
463fn apply_color_indexing_transform_small_table<const W_BITS: u8, const EXP_ENTRY_SIZE: usize>(
464    image_data: &mut [u8],
465    width: u16,
466    height: u16,
467    table_size: u8, // Max 16
468    table_data: &[u8],
469) {
470    // As of Rust 1.87 we cannot use `const` here. The compiler can still optimize them heavily
471    // because W_BITS is a const generic for each instantiation of this function.
472    let pixels_per_packed_byte_u8: u8 = 1 << W_BITS;
473    let bits_per_entry_u8: u8 = 8 / pixels_per_packed_byte_u8;
474    let mask_u8: u8 = (1 << bits_per_entry_u8) - 1;
475
476    // This is also effectively a compile-time constant for each instantiation.
477    let pixels_per_packed_byte_usize: usize = pixels_per_packed_byte_u8 as usize;
478
479    // Verify that the passed EXP_ENTRY_SIZE matches our calculation based on W_BITS, just as a sanity check.
480    debug_assert_eq!(
481        EXP_ENTRY_SIZE,
482        4 * pixels_per_packed_byte_usize,
483        "Mismatch in EXP_ENTRY_SIZE"
484    );
485
486    // Precompute the full lookup table.
487    // Each of the 256 possible packed byte values maps to an array of RGBA pixels.
488    // The array type uses the const generic EXP_ENTRY_SIZE.
489    let expanded_lookup_table_storage: Vec<[u8; EXP_ENTRY_SIZE]> = (0..256u16)
490        .map(|packed_byte_value_u16| {
491            let mut entry_pixels_array = [0u8; EXP_ENTRY_SIZE]; // Uses const generic
492            let packed_byte_value = packed_byte_value_u16 as u8;
493
494            // Loop bound is effectively constant for each instantiation.
495            for pixel_sub_index in 0..pixels_per_packed_byte_usize {
496                let shift_amount = (pixel_sub_index as u8) * bits_per_entry_u8;
497                let k = (packed_byte_value >> shift_amount) & mask_u8;
498
499                let color_source_array: [u8; 4] = if k < table_size {
500                    let color_data_offset = usize::from(k) * 4;
501                    table_data[color_data_offset..color_data_offset + 4]
502                        .try_into()
503                        .unwrap()
504                } else {
505                    [0u8; 4] // WebP spec: out-of-bounds indices are [0,0,0,0]
506                };
507
508                let array_fill_offset = pixel_sub_index * 4;
509                entry_pixels_array[array_fill_offset..array_fill_offset + 4]
510                    .copy_from_slice(&color_source_array);
511            }
512            entry_pixels_array
513        })
514        .collect();
515
516    let expanded_lookup_table_array: &[[u8; EXP_ENTRY_SIZE]; 256] =
517        expanded_lookup_table_storage.as_slice().try_into().unwrap();
518
519    let packed_image_width_in_blocks = width.div_ceil(pixels_per_packed_byte_u8.into()) as usize;
520
521    if width == 0 || height == 0 {
522        return;
523    }
524
525    let final_block_expanded_size_bytes =
526        (width as usize * 4) - EXP_ENTRY_SIZE * (packed_image_width_in_blocks.saturating_sub(1));
527
528    let input_stride_bytes_packed = packed_image_width_in_blocks * 4;
529    let output_stride_bytes_expanded = width as usize * 4;
530
531    let mut packed_indices_for_row: Vec<u8> = vec![0; packed_image_width_in_blocks];
532
533    for y_rev_idx in 0..height as usize {
534        let y = height as usize - 1 - y_rev_idx;
535
536        let packed_row_input_global_offset = y * input_stride_bytes_packed;
537        let packed_argb_row_slice =
538            &image_data[packed_row_input_global_offset..][..input_stride_bytes_packed];
539
540        for (packed_argb_chunk, packed_idx) in packed_argb_row_slice
541            .chunks_exact(4)
542            .zip(packed_indices_for_row.iter_mut())
543        {
544            *packed_idx = packed_argb_chunk[1];
545        }
546
547        let output_row_global_offset = y * output_stride_bytes_expanded;
548        let output_row_slice_mut =
549            &mut image_data[output_row_global_offset..][..output_stride_bytes_expanded];
550
551        let num_full_blocks = packed_image_width_in_blocks.saturating_sub(1);
552
553        let (full_blocks_part, final_block_part) =
554            output_row_slice_mut.split_at_mut(num_full_blocks * EXP_ENTRY_SIZE);
555
556        for (output_chunk_slice, &packed_index_byte) in full_blocks_part
557            .chunks_exact_mut(EXP_ENTRY_SIZE) // Uses const generic to avoid expensive memmove call
558            .zip(packed_indices_for_row.iter())
559        {
560            let output_chunk_array: &mut [u8; EXP_ENTRY_SIZE] =
561                output_chunk_slice.try_into().unwrap();
562
563            let colors_data_array = &expanded_lookup_table_array[packed_index_byte as usize];
564
565            *output_chunk_array = *colors_data_array;
566        }
567
568        if packed_image_width_in_blocks > 0 {
569            let final_packed_index_byte = packed_indices_for_row[packed_image_width_in_blocks - 1];
570            let colors_data_full_array =
571                &expanded_lookup_table_array[final_packed_index_byte as usize];
572
573            final_block_part
574                .copy_from_slice(&colors_data_full_array[..final_block_expanded_size_bytes]);
575        }
576    }
577}
578
579//predictor functions
580
581/// Get average of 2 bytes
582fn average2(a: u8, b: u8) -> u8 {
583    ((u16::from(a) + u16::from(b)) / 2) as u8
584}
585
586/// Get average of 2 bytes, allows some predictors to be autovectorized by
587/// keeping computation within lanes of `u8`.
588///
589/// LLVM is capable of optimizing `average2` into this but not in all cases.
590fn average2_autovec(a: u8, b: u8) -> u8 {
591    (a & b) + ((a ^ b) >> 1)
592}
593
594/// Clamp add subtract full on one part
595fn clamp_add_subtract_full(a: i16, b: i16, c: i16) -> u8 {
596    // Clippy suggests the clamp method, but it seems to optimize worse as of rustc 1.82.0 nightly.
597    #![allow(clippy::manual_clamp)]
598    (a + b - c).max(0).min(255) as u8
599}
600
601/// Clamp add subtract half on one part
602fn clamp_add_subtract_half(a: i16, b: i16) -> u8 {
603    // Clippy suggests the clamp method, but it seems to optimize worse as of rustc 1.82.0 nightly.
604    #![allow(clippy::manual_clamp)]
605    (a + (a - b) / 2).max(0).min(255) as u8
606}
607
608/// Does color transform on 2 numbers
609fn color_transform_delta(t: i8, c: i8) -> u32 {
610    (i32::from(t) * i32::from(c)) as u32 >> 5
611}
612
613#[cfg(all(test, feature = "_benchmarks"))]
614mod benches {
615    use rand::Rng;
616    use test::{black_box, Bencher};
617
618    fn measure_predictor(b: &mut Bencher, predictor: fn(&mut [u8], std::ops::Range<usize>, usize)) {
619        let width = 256;
620        let mut data = vec![0u8; width * 8];
621        rand::thread_rng().fill(&mut data[..]);
622        b.bytes = 4 * width as u64 - 4;
623        b.iter(|| {
624            predictor(
625                black_box(&mut data),
626                black_box(width * 4 + 4..width * 8),
627                black_box(width),
628            )
629        });
630    }
631
632    #[bench]
633    fn predictor00(b: &mut Bencher) {
634        measure_predictor(b, super::apply_predictor_transform_0);
635    }
636    #[bench]
637    fn predictor01(b: &mut Bencher) {
638        measure_predictor(b, super::apply_predictor_transform_1);
639    }
640    #[bench]
641    fn predictor02(b: &mut Bencher) {
642        measure_predictor(b, super::apply_predictor_transform_2);
643    }
644    #[bench]
645    fn predictor03(b: &mut Bencher) {
646        measure_predictor(b, super::apply_predictor_transform_3);
647    }
648    #[bench]
649    fn predictor04(b: &mut Bencher) {
650        measure_predictor(b, super::apply_predictor_transform_4);
651    }
652    #[bench]
653    fn predictor05(b: &mut Bencher) {
654        measure_predictor(b, super::apply_predictor_transform_5);
655    }
656    #[bench]
657    fn predictor06(b: &mut Bencher) {
658        measure_predictor(b, super::apply_predictor_transform_6);
659    }
660    #[bench]
661    fn predictor07(b: &mut Bencher) {
662        measure_predictor(b, super::apply_predictor_transform_7);
663    }
664    #[bench]
665    fn predictor08(b: &mut Bencher) {
666        measure_predictor(b, super::apply_predictor_transform_8);
667    }
668    #[bench]
669    fn predictor09(b: &mut Bencher) {
670        measure_predictor(b, super::apply_predictor_transform_9);
671    }
672    #[bench]
673    fn predictor10(b: &mut Bencher) {
674        measure_predictor(b, super::apply_predictor_transform_10);
675    }
676    #[bench]
677    fn predictor11(b: &mut Bencher) {
678        measure_predictor(b, super::apply_predictor_transform_11);
679    }
680    #[bench]
681    fn predictor12(b: &mut Bencher) {
682        measure_predictor(b, super::apply_predictor_transform_12);
683    }
684    #[bench]
685    fn predictor13(b: &mut Bencher) {
686        measure_predictor(b, super::apply_predictor_transform_13);
687    }
688
689    #[bench]
690    fn color_transform(b: &mut Bencher) {
691        let width = 256;
692        let height = 256;
693        let size_bits = 3;
694        let mut data = vec![0u8; width * height * 4];
695        let mut transform_data = vec![0u8; (width * height * 4) >> (size_bits * 2)];
696        rand::thread_rng().fill(&mut data[..]);
697        rand::thread_rng().fill(&mut transform_data[..]);
698        b.bytes = 4 * width as u64 * height as u64;
699        b.iter(|| {
700            super::apply_color_transform(
701                black_box(&mut data),
702                black_box(width as u16),
703                black_box(size_bits),
704                black_box(&transform_data),
705            );
706        });
707    }
708
709    #[bench]
710    fn subtract_green(b: &mut Bencher) {
711        let mut data = vec![0u8; 1024 * 4];
712        rand::thread_rng().fill(&mut data[..]);
713        b.bytes = data.len() as u64;
714        b.iter(|| {
715            super::apply_subtract_green_transform(black_box(&mut data));
716        });
717    }
718}