Handle multiple image format (#1988)

* Polymorphic copy_region

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Polymorphic qr_code

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Polymorphic detect_border

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Implement luminance conversion

* Implement RGB_565 and RGBA_F16 support

* Revert "Workaround for high bit depth images"

This reverts commit 12f46e2e4e.

* Add ptr_as_image

Signed-off-by: Dude so hot <djohn@fbi.gov>

* ptr_as_image_mut

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Refine poly

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Refine

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Use fake channel

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Cleanup

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Naming

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Refine

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Cleanup

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Refine

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Use typealias

Signed-off-by: Dude so hot <djohn@fbi.gov>

* just unwarp

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Cleanup

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Just unwarp

Signed-off-by: Dude so hot <djohn@fbi.gov>

* Address code review

Signed-off-by: Dude so hot <djohn@fbi.gov>

---------

Signed-off-by: Dude so hot <djohn@fbi.gov>
Co-authored-by: Dude so hot <djohn@fbi.gov>
Co-authored-by: FooIbar <118464521+FooIbar@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2024-12-04 19:30:46 +08:00
1 parent 3bbf195870
commit da29fc2f25
13 files changed
+219 -94

No files matched your search

+3 -3
View File
@@ -21,7 +21,7 @@ jobs:
java-version: 21
- name: Install Rust Toolchain
run: rustup default 1.81.0 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
run: rustup default nightly-2024-08-30 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
- name: Rust Cache
uses: Swatinem/rust-cache@v2
@@ -61,7 +61,7 @@ jobs:
java-version: 21
- name: Install Rust Toolchain
run: rustup default 1.81.0 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
run: rustup default nightly-2024-08-30 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
- name: Rust Cache
uses: Swatinem/rust-cache@v2
@@ -129,7 +129,7 @@ jobs:
java-version: 21
- name: Install Rust Toolchain
run: rustup default 1.81.0 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
run: rustup default nightly-2024-08-30 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android && rustup component add rustfmt clippy
- name: Rust Cache
uses: Swatinem/rust-cache@v2
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
java-version: 21
- name: Install Rust Toolchain
run: rustup default 1.81.0 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android
run: rustup default nightly-2024-08-30 && rustup target add armv7-linux-androideabi aarch64-linux-android x86_64-linux-android
- name: Rust Cache
uses: Swatinem/rust-cache@v2
@@ -17,9 +17,6 @@ import com.hippo.ehviewer.util.isAtLeastQ
import eu.kanade.tachiyomi.util.system.logcat
import moe.tarsin.coroutines.runSuspendCatching
@RequiresApi(Build.VERSION_CODES.O)
private const val FORMAT = HardwareBuffer.RGBA_8888
@RequiresApi(Build.VERSION_CODES.O)
private const val USAGE = HardwareBuffer.USAGE_CPU_WRITE_RARELY or HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
@@ -42,8 +39,14 @@ object CropBorderInterceptor : Interceptor {
val meetHardwareThreshold = maxOf(w, h) <= chain.request.hardwareThreshold
val bitmap = when {
isAtLeastQ && meetHardwareThreshold -> runSuspendCatching {
val format = when (val config = src.config) {
Bitmap.Config.ARGB_8888 -> HardwareBuffer.RGBA_8888
Bitmap.Config.RGB_565 -> HardwareBuffer.RGB_565
Bitmap.Config.RGBA_F16 -> HardwareBuffer.RGBA_FP16
else -> error("Unsupported bitmap config: $config")
}
resourceScope {
val buffer = autoCloseable { HardwareBuffer.create(w, h, FORMAT, 1, USAGE) }
val buffer = autoCloseable { HardwareBuffer.create(w, h, format, 1, USAGE) }
copyBitmapToAHB(src, buffer, x, y)
Bitmap.wrapHardwareBuffer(buffer, src.colorSpace)
}
@@ -1,6 +1,5 @@
package com.hippo.ehviewer.coil
import android.graphics.Bitmap
import androidx.compose.ui.unit.IntRect
import coil3.Extras
import coil3.getExtra
@@ -35,7 +34,7 @@ object DetectBorderInterceptor : Interceptor {
} else {
image.width / image.height
}
if (ratio < RATIO_THRESHOLD && bitmap.config == Bitmap.Config.ARGB_8888) {
if (ratio < RATIO_THRESHOLD) {
val array = detectBorder(bitmap)
val minWidth = image.width * CROP_THRESHOLD
val minHeight = image.height * CROP_THRESHOLD
@@ -1,6 +1,5 @@
package com.hippo.ehviewer.coil
import android.graphics.Bitmap
import coil3.Extras
import coil3.getExtra
import coil3.intercept.Interceptor
@@ -24,12 +23,9 @@ object QrCodeInterceptor : Interceptor {
if (chain.request.detectQrCode && result is SuccessResult) {
val image = result.image
if (image is BitmapImageWithExtraInfo) {
val bitmap = image.image.bitmap
if (bitmap.config == Bitmap.Config.ARGB_8888) {
val hasQrCode = hasQrCode(bitmap)
val new = image.copy(hasQrCode = hasQrCode)
return result.copy(image = new)
}
val hasQrCode = hasQrCode(image.image.bitmap)
val new = image.copy(hasQrCode = hasQrCode)
return result.copy(image = new)
}
}
return result
+32 -20
View File
@@ -1,16 +1,17 @@
#![cfg(feature = "android")]
use super::jvm::jni_throwing;
use crate::img::border::detect_border;
use crate::img::qr_code::detect_image_ad;
use anyhow::{Context, Result};
use image::{ImageBuffer, Rgba};
use crate::img::border::DetectBorder;
use crate::img::core::{CustomPixel, ImageConsumer, Rgb565, Rgba8888, RgbaF16};
use crate::img::qr_code::QrCode;
use anyhow::{anyhow, Ok, Result};
use image::ImageBuffer;
use jni::objects::JClass;
use jni::sys::jboolean;
use jni::sys::{jintArray, jobject};
use jni::JNIEnv;
use jni_fn::jni_fn;
use ndk::bitmap::Bitmap;
use ndk::bitmap::{Bitmap, BitmapFormat};
use std::ptr::slice_from_raw_parts;
#[no_mangle]
@@ -18,9 +19,9 @@ use std::ptr::slice_from_raw_parts;
#[jni_fn("com.hippo.ehviewer.image.ImageKt")]
pub fn detectBorder(mut env: JNIEnv, _class: JClass, object: jobject) -> jintArray {
jni_throwing(&mut env, |env| {
let slice = with_bitmap_content(env, object, |img| Ok(detect_border(&img)))?;
let slice = use_bitmap_content(env, object, DetectBorder)?;
let array = env.new_int_array(4)?;
env.set_int_array_region(&array, 0, &slice.context("Image too small!")?)?;
env.set_int_array_region(&array, 0, &slice)?;
Ok(array.into_raw())
})
}
@@ -30,26 +31,37 @@ pub fn detectBorder(mut env: JNIEnv, _class: JClass, object: jobject) -> jintArr
#[jni_fn("com.hippo.ehviewer.image.ImageKt")]
pub fn hasQrCode(mut env: JNIEnv, _class: JClass, object: jobject) -> jboolean {
jni_throwing(&mut env, |env| {
with_bitmap_content(env, object, |image| Ok(detect_image_ad(image) as jboolean))
Ok(use_bitmap_content(env, object, QrCode)? as jboolean)
})
}
pub fn with_bitmap_content<F, R>(env: &mut JNIEnv, bitmap: jobject, f: F) -> Result<R>
where
F: FnOnce(ImageBuffer<Rgba<u8>, &[u8]>) -> Result<R>,
{
fn ptr_as_image<'local, P: CustomPixel>(
ptr: *const !,
w: u32,
h: u32,
) -> ImageBuffer<P, &'local [P::Subpixel]> {
let size = (w * h * P::CHANNEL_COUNT as u32) as usize;
let buffer = unsafe { &*slice_from_raw_parts(ptr as *const P::Subpixel, size) };
ImageBuffer::from_raw(w, h, buffer).unwrap()
}
pub fn use_bitmap_content<R, F: ImageConsumer<R>>(
env: &mut JNIEnv,
bitmap: jobject,
f: F,
) -> Result<R> {
// SAFETY: kotlin caller must ensure bitmap is valid.
let handle = unsafe { Bitmap::from_jni(env.get_raw(), bitmap) };
let info = handle.info()?;
let (width, height) = (info.width(), info.height());
let ptr = handle.lock_pixels()? as *const u8;
// SAFETY: maybe unsafe if bitmap buffer not RGBA8888 format.
let buffer = unsafe { &*slice_from_raw_parts(ptr, (width * height * 4) as usize) };
let image = ImageBuffer::from_raw(width, height, buffer);
let result = image.context("Image buffer not RGBA8888!!!").and_then(f);
let (w, h, format) = (info.width(), info.height(), info.format());
let p = handle.lock_pixels()? as *const !;
let result = match format {
BitmapFormat::RGBA_8888 => f.apply(&ptr_as_image::<Rgba8888>(p, w, h)),
BitmapFormat::RGB_565 => f.apply(&ptr_as_image::<Rgb565>(p, w, h)),
BitmapFormat::RGBA_F16 => f.apply(&ptr_as_image::<RgbaF16>(p, w, h)),
_ => Err(anyhow!("Unsupported bitmap format")),
};
handle.unlock_pixels()?;
result
}
+13 -16
View File
@@ -1,15 +1,13 @@
#![cfg(feature = "android-26")]
use super::android::with_bitmap_content;
use super::android::use_bitmap_content;
use super::jvm::jni_throwing;
use anyhow::Context;
use image::{GenericImage, GenericImageView, ImageBuffer};
use crate::img::copy_region::CopyRegion;
use jni::objects::JClass;
use jni::sys::{jint, jobject};
use jni::JNIEnv;
use jni_fn::jni_fn;
use ndk::hardware_buffer::{HardwareBuffer, HardwareBufferUsage};
use std::{ops::Deref, ptr::slice_from_raw_parts_mut};
#[no_mangle]
#[allow(non_snake_case)]
@@ -17,17 +15,16 @@ use std::{ops::Deref, ptr::slice_from_raw_parts_mut};
pub fn copyBitmapToAHB(mut env: JNIEnv, _: JClass, bm: jobject, ahb: jobject, x: jint, y: jint) {
let ahb = unsafe { HardwareBuffer::from_jni(env.get_raw(), ahb) };
jni_throwing(&mut env, |env| {
with_bitmap_content(env, bm, |src| {
let desc = ahb.describe();
let (w, h, stride) = (desc.width, desc.height, desc.stride);
let view = src.view(x as u32, y as u32, w, h);
let ptr = ahb.lock(HardwareBufferUsage::CPU_WRITE_RARELY, None, None)? as *mut u8;
let s = unsafe { &mut *slice_from_raw_parts_mut(ptr, (stride * h * 4) as usize) };
let dst = ImageBuffer::from_raw(stride, h, s).context("Illegal AHB format!");
let result = dst.map(|mut dst| dst.copy_from(view.deref(), 0, 0));
ahb.unlock()?;
result??;
Ok(())
})
let desc = ahb.describe();
let (w, h, stride) = (desc.width, desc.height, desc.stride);
let ptr = ahb.lock(HardwareBufferUsage::CPU_WRITE_RARELY, None, None)?;
let s = CopyRegion {
ptr: ptr as *mut !,
target_dim: (stride, h),
src_rect: (x as u32, y as u32, w, h),
};
let result = use_bitmap_content(env, bm, s);
ahb.unlock()?;
result
})
}
+2
View File
@@ -1,2 +1,4 @@
pub mod border;
pub mod copy_region;
pub mod core;
pub mod qr_code;
+26 -20
View File
@@ -1,4 +1,7 @@
use image::{ImageBuffer, Luma, Pixel, Rgba};
use super::core::ImageConsumer;
use crate::img::core::CustomPixel;
use anyhow::Result;
use image::{ImageBuffer, Luma};
/** A line will be considered as having content if 0.25% of it is filled. */
static FILLED_RATIO_LIMIT: f32 = 0.0025;
@@ -9,25 +12,25 @@ static THRESHOLD: f32 = 0.75;
static THRESHOLD_FOR_BLACK: u8 = (255.0 * THRESHOLD) as u8;
static THRESHOLD_FOR_WHITE: u8 = (255.0 - 255.0 * THRESHOLD) as u8;
fn is_white(pixel: &Luma<u8>) -> bool {
fn is_white(pixel: Luma<u8>) -> bool {
pixel.0[0] > THRESHOLD_FOR_WHITE
}
fn is_black(pixel: &Luma<u8>) -> bool {
fn is_black(pixel: Luma<u8>) -> bool {
pixel.0[0] < THRESHOLD_FOR_BLACK
}
fn line_not_filled_by<'pixel>(
line: impl Iterator<Item = &'pixel Rgba<u8>>,
fn line_not_filled_by<'pixel, P: CustomPixel + 'pixel>(
line: impl Iterator<Item = &'pixel P>,
white: bool,
limit: i32,
) -> bool {
let f = if white { is_black } else { is_white };
line.step_by(2).filter(|p| f(&p.to_luma())).count() as i32 > limit
line.step_by(2).filter(|p| f(p.to_luma8())).count() as i32 > limit
}
fn try_count_lines<'pixel>(
iter: impl Iterator<Item = impl Iterator<Item = &'pixel Rgba<u8>>>,
fn try_count_lines<'pixel, P: CustomPixel + 'pixel>(
iter: impl Iterator<Item = impl Iterator<Item = &'pixel P>>,
white: bool,
limit: i32,
) -> i32 {
@@ -42,15 +45,15 @@ fn try_count_lines<'pixel>(
count
}
fn detect_border_lines<'pixel>(
mut iter: impl Iterator<Item = impl Iterator<Item = &'pixel Rgba<u8>>>,
fn detect_border_lines<'pixel, P: CustomPixel + 'pixel>(
mut iter: impl Iterator<Item = impl Iterator<Item = &'pixel P>>,
line_len: u32,
) -> i32 {
let first_row = iter.next().expect("Image is empty!");
let filled_limit = (line_len as f32 * FILLED_RATIO_LIMIT / 2.0).round() as i32;
let (mut black, mut white) = (0, 0);
for pixel in first_row.step_by(2) {
let luma = &pixel.to_luma();
let luma = pixel.to_luma8();
if is_black(luma) {
black += 1;
} else if is_white(luma) {
@@ -66,13 +69,16 @@ fn detect_border_lines<'pixel>(
}
}
#[allow(dead_code)]
pub fn detect_border(image: &ImageBuffer<Rgba<u8>, &[u8]>) -> Option<[i32; 4]> {
let (w, h) = image.dimensions();
let top = detect_border_lines(image.rows(), w);
let bottom = detect_border_lines(image.rows().rev(), w);
let iter = (0..w).map(|row| (0..h).map(move |col| image.get_pixel(row, col)));
let left = detect_border_lines(iter.clone(), h);
let right = detect_border_lines(iter.rev(), h);
Some([left, top, w as i32 - right, h as i32 - bottom])
pub struct DetectBorder;
impl ImageConsumer<[i32; 4]> for DetectBorder {
fn apply<P: CustomPixel>(self, image: &ImageBuffer<P, &[P::Subpixel]>) -> Result<[i32; 4]> {
let (w, h) = image.dimensions();
let top = detect_border_lines(image.rows(), w);
let bottom = detect_border_lines(image.rows().rev(), w);
let iter = (0..w).map(|row| (0..h).map(move |col| image.get_pixel(row, col)));
let left = detect_border_lines(iter.clone(), h);
let right = detect_border_lines(iter.rev(), h);
Ok([left, top, w as i32 - right, h as i32 - bottom])
}
}
+30
View File
@@ -0,0 +1,30 @@
use super::core::{CustomPixel, ImageConsumer};
use anyhow::Result;
use image::{GenericImage, GenericImageView, ImageBuffer};
use std::ptr::slice_from_raw_parts_mut;
fn ptr_as_image_mut<'a, P: CustomPixel>(
ptr: *mut !,
w: u32,
h: u32,
) -> ImageBuffer<P, &'a mut [P::Subpixel]> {
let size = (w * h * P::CHANNEL_COUNT as u32) as usize;
let buffer = unsafe { &mut *slice_from_raw_parts_mut(ptr as *mut P::Subpixel, size) };
ImageBuffer::from_raw(w, h, buffer).unwrap()
}
pub struct CopyRegion {
pub ptr: *mut !,
pub target_dim: (u32, u32),
pub src_rect: (u32, u32, u32, u32),
}
impl ImageConsumer<()> for CopyRegion {
fn apply<P: CustomPixel>(self, src: &ImageBuffer<P, &[P::Subpixel]>) -> Result<()> {
let (x, y, w, h) = self.src_rect;
let (dst_w, dst_h) = self.target_dim;
let mut dst = ptr_as_image_mut(self.ptr, dst_w, dst_h);
let view = src.view(x, y, w, h);
Ok(dst.copy_from(&*view, 0, 0)?)
}
}
+62
View File
@@ -0,0 +1,62 @@
use anyhow::Result;
use image::{ImageBuffer, Luma, Pixel, Rgb, Rgba};
pub type Rgba8888 = Rgba<u8>;
pub type Rgb565 = Luma<u16>;
pub type RgbaF16 = Luma<u64>;
#[allow(dead_code)]
pub trait ImageConsumer<R> {
fn apply<T: CustomPixel>(self, buffer: &ImageBuffer<T, &[T::Subpixel]>) -> Result<R>;
}
pub trait CustomPixel: Pixel {
fn to_luma8(&self) -> Luma<u8>;
}
impl CustomPixel for Rgba8888 {
fn to_luma8(&self) -> Luma<u8> {
self.to_luma()
}
}
impl CustomPixel for Rgb565 {
fn to_luma8(&self) -> Luma<u8> {
let packed = self.channels()[0];
let rgb = Rgb([
l5_to_l8(packed >> 11),
l6_to_l8(packed >> 5),
l5_to_l8(packed),
]);
rgb.to_luma()
}
}
#[inline]
fn l5_to_l8(v: u16) -> u8 {
(((v & 0x1f) * 527 + 23) >> 6) as u8
}
#[inline]
fn l6_to_l8(v: u16) -> u8 {
(((v & 0x3f) * 259 + 33) >> 6) as u8
}
impl CustomPixel for RgbaF16 {
fn to_luma8(&self) -> Luma<u8> {
let packed = self.channels()[0];
let rgba = Rgba([
f16_to_f32(packed),
f16_to_f32(packed >> 16),
f16_to_f32(packed >> 32),
f16_to_f32(packed >> 48),
]);
let l = rgba.to_luma().channels()[0];
Luma([(l.clamp(0.0, 1.0) * u8::MAX as f32).round() as u8])
}
}
#[inline]
fn f16_to_f32(v: u64) -> f32 {
f16::from_bits((v & 0xffff) as u16) as f32
}
+36 -21
View File
@@ -1,28 +1,43 @@
use image::{ImageBuffer, Rgba};
use super::core::ImageConsumer;
use crate::img::core::CustomPixel;
use anyhow::Result;
use image::{GrayImage, ImageBuffer};
use rxing::common::HybridBinarizer;
use rxing::qrcode::detector::FinderPatternFinder;
use rxing::{BinaryBitmap, DecodingHintDictionary, Point, RGBLuminanceSource};
use rxing::{BinaryBitmap, DecodingHintDictionary, Luma8LuminanceSource, Point};
use std::f32::consts::FRAC_1_SQRT_2;
use std::ptr::slice_from_raw_parts;
#[allow(dead_code)]
pub fn detect_image_ad(image: ImageBuffer<Rgba<u8>, &[u8]>) -> bool {
let (width, height) = (image.width() as usize, image.height() as usize);
let buffer = image.into_raw();
let pixels = unsafe { &*slice_from_raw_parts(buffer.as_ptr() as *const u32, buffer.len()) };
let source = RGBLuminanceSource::new_with_width_height_pixels(width, height, pixels);
let image = BinaryBitmap::new(HybridBinarizer::new(source));
match FinderPatternFinder::new(image.get_black_matrix()).find(&DecodingHintDictionary::new()) {
Ok(info) => {
// Check if the region is squarish
let top_left = Point::from(info.getTopLeft());
let top_right = Point::from(info.getTopRight());
let bottom_left = Point::from(info.getBottomLeft());
let length = top_right.distance(top_left);
let epsilon = length * 0.05;
(bottom_left.distance(top_left) - length).abs() < epsilon
&& (bottom_left.distance(top_right) * FRAC_1_SQRT_2 - length).abs() < epsilon
fn image_buffer_to_luma8<P: CustomPixel>(
src: &ImageBuffer<P, &[P::Subpixel]>,
) -> Luma8LuminanceSource {
let (w, h) = src.dimensions();
let mut dst = GrayImage::new(w, h);
for (to, from) in dst.pixels_mut().zip(src.pixels()) {
*to = from.to_luma8();
}
Luma8LuminanceSource::new(dst.into_raw(), w, h)
}
pub struct QrCode;
impl ImageConsumer<bool> for QrCode {
fn apply<P: CustomPixel>(self, image: &ImageBuffer<P, &[P::Subpixel]>) -> Result<bool> {
let source = image_buffer_to_luma8(image);
let image = BinaryBitmap::new(HybridBinarizer::new(source));
match FinderPatternFinder::new(image.get_black_matrix())
.find(&DecodingHintDictionary::new())
{
Ok(info) => {
// Check if the region is squarish
let top_left = Point::from(info.getTopLeft());
let top_right = Point::from(info.getTopRight());
let bottom_left = Point::from(info.getBottomLeft());
let length = top_right.distance(top_left);
let epsilon = length * 0.05;
Ok((bottom_left.distance(top_left) - length).abs() < epsilon
&& (bottom_left.distance(top_right) * FRAC_1_SQRT_2 - length).abs() < epsilon)
}
Err(_) => Ok(false),
}
Err(_) => false,
}
}
+3
View File
@@ -1,3 +1,6 @@
#![feature(f16)]
#![feature(never_type)]
mod ffi;
mod img;
mod parser;