Logo
Computer Vision in Action

Similar & Duplicate Image Detection

Implement a highly efficient Average Hash (aHash) algorithm combined with Disjoint Set Union (DSU) in C++ to group thousands of images without Out-Of-Memory (OOM) crashes.

Step 1: Architecture & Algorithm

Why aHash and file paths?

aHash + Hamming Distance

Average Hashing (aHash) shrinks an image to 8x8 pixels, converts it to grayscale, and compares each pixel to the mean to generate a 64-bit integer. We then use __builtin_popcountll to rapidly compute the Hamming distance.

Anti-OOM Strategy

Passing an Array<Bitmap> of 100+ images from Kotlin to JNI will quickly exhaust memory. Instead, we pass an Array<String> of file paths. C++ reads, hashes, and releases them sequentially.

Step 2: Native C++ Implementation

Hash extraction and Disjoint Set Union grouping

app/src/main/cpp/image_matcher.cpp
#include <jni.h>
#include <string>
#include <vector>
#include <functional>
#include <opencv2/opencv.hpp>

// 1. 计算图片的 aHash (64位整数)
uint64_t computeHash(const cv::Mat& img) {
if (img.empty()) return 0;
cv::Mat resized, gray;
// 极速降采样至 8x8 并转灰度
cv::resize(img, resized, cv::Size(8, 8));
cv::cvtColor(resized, gray, cv::COLOR_BGR2GRAY);
cv::Scalar mean = cv::mean(gray);
uint64_t hash = 0;
int index = 0;
for (int i = 0; i < gray.rows; i++) {
for (int j = 0; j < gray.cols; j++) {
if (gray.at<uchar>(i, j) >= mean[0]) {
hash |= (1ULL << index);
}
index++;
}
}
return hash;
}

extern "C" JNIEXPORT jintArray JNICALL
Java_com_wefamily_papa_utils_OpenCVUtils_groupSimilarImages( 
JNIEnv* env, jobject /* this */, jobjectArray imagePaths, jint threshold) {

int n = env->GetArrayLength(imagePaths);
std::vector<uint64_t> hashes(n);

// 2. 依次读取图片并提取特征 (规避 OOM)
for (int i = 0; i < n; i++) {
jstring jpath = (jstring) env->GetObjectArrayElement(imagePaths, i);
const char* path = env->GetStringUTFChars(jpath, nullptr);
cv::Mat img = cv::imread(path, cv::IMREAD_COLOR);
hashes[i] = computeHash(img);
env->ReleaseStringUTFChars(jpath, path);
env->DeleteLocalRef(jpath);
}

// 3. 初始化并查集 (DSU) 进行聚类
std::vector<int> labels(n);
for(int i = 0; i < n; i++) labels[i] = i;

std::function<int(int)> find = [&](int i) {
return labels[i] == i ? i : (labels[i] = find(labels[i]));
};

// 4. 两两比对汉明距离,相似则合并分组
for(int i = 0; i < n; i++) {
for(int j = i + 1; j < n; j++) {
if (hashes[i] == 0 || hashes[j] == 0) continue;
int distance = __builtin_popcountll(hashes[i] ^ hashes[j]);
if (distance <= threshold) {
int rootI = find(i);
int rootJ = find(j);
if (rootI != rootJ) labels[rootI] = rootJ;
}
}
}

// 5. 组装结果返回给 Kotlin 层
jintArray result = env->NewIntArray(n);
jint* resPtr = env->GetIntArrayElements(result, nullptr);
for(int i = 0; i < n; i++) {
resPtr[i] = find(i); // 路径压缩,确保同组的 ID 一致
}
env->ReleaseIntArrayElements(result, resPtr, 0);

return result;
}

Step 3: Kotlin Integration

Map the native IntArray back into grouped lists of strings.

OpenCVUtils.kt
package com.wefamily.papa.utils

object OpenCVUtils {
init {
System.loadLibrary("image_matcher")
}

/** * 底层 JNI 接口 * @param imagePaths 图片的绝对路径数组 * @param threshold 汉明距离阈值 (建议值: 0-10,越小要求越相似) * @return 返回与数组长度一致的 IntArray,值为分组的 Group ID */
private external fun groupSimilarImages( 
imagePaths: Array<String>, 
threshold: Int
): IntArray

/** * 业务层调用的便捷封装 */
fun findDuplicates(paths: List<String>, threshold: Int = 5): List<List<String>> {
if (paths.isEmpty()) return emptyList()

// 1. 传入底层计算分组 ID
val groupIds = groupSimilarImages(paths.toTypedArray(), threshold)

// 2. 使用 Kotlin 高阶函数进行数据组合与过滤
return paths.zip(groupIds.toList())
.groupBy {  it.second  }       // 按 Group ID 分组
.values
.map {  group -> group.map {  it.first  } } // 提取出原路径
.filter {  it.size > 1  }   // 只保留发现相似/重复的组 (数量 > 1)
}
}

Step 4: Real-World Usage

// ViewModel 或 Repository 中的实际调用
val allPhotos = listOf( 
"/storage/emulated/0/DCIM/Camera/IMG_20260704_001.jpg",
"/storage/emulated/0/DCIM/Camera/IMG_20260704_002_burst.jpg",
"/storage/emulated/0/Download/meme.png"
)

// 开启协程在后台线程处理,防止阻塞主线程
viewModelScope.launch(Dispatchers.IO) {
val duplicateGroups = OpenCVUtils.findDuplicates(allPhotos, threshold = 5)

duplicateGroups.forEachIndexed {  index, group ->
Log.d("PapaGallery", "发现相似图片组 $index:")
group.forEach {  path ->
Log.d("PapaGallery", " -> $path")
}
}
}