From 04b84e188a26789cab8a309c59034cdf67bf0d18 Mon Sep 17 00:00:00 2001 From: TZGyn Date: Wed, 30 Sep 2026 11:44:26 +0800 Subject: [PATCH] Retry OCR on upscaled images when nothing is read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tiny UI text (around 7pt) reads as nothing at native size — verified locally. When a pass returns fewer than three characters, OCR retries at 2x and 3x and keeps the fullest result. The smoke test covers a tiny-text case so the fallback stays working. 💘 Generated with Crush Assisted-by: Crush:deepseek-v4-flash-0731 --- Sources/Kapture/App.swift | 21 +++++--- Sources/Kapture/OCRService.swift | 84 ++++++++++++++++++++++---------- build-app.sh | 4 +- 3 files changed, 73 insertions(+), 36 deletions(-) diff --git a/Sources/Kapture/App.swift b/Sources/Kapture/App.swift index 939a157..6951f31 100644 --- a/Sources/Kapture/App.swift +++ b/Sources/Kapture/App.swift @@ -123,8 +123,8 @@ enum LoginItem { } } -func renderTestImage(_ text: String, fontSize: CGFloat = 56) -> CGImage? { - let size = NSSize(width: 1900, height: 220) +func renderTestImage(_ text: String, fontSize: CGFloat = 56, width: CGFloat = 1900, height: CGFloat = 220) -> CGImage? { + let size = NSSize(width: width, height: height) guard let rep = NSBitmapImageRep( bitmapDataPlanes: nil, pixelsWide: Int(size.width), pixelsHigh: Int(size.height), bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, @@ -138,15 +138,20 @@ func renderTestImage(_ text: String, fontSize: CGFloat = 56) -> CGImage? { .font: NSFont.systemFont(ofSize: fontSize), .foregroundColor: NSColor.black, ] - text.draw(at: NSPoint(x: 60, y: 75), withAttributes: attrs) + // Centre the text vertically so tiny canvases still contain it. + let textHeight = ("Ag" as NSString).size(withAttributes: attrs).height + let y = max(4, (size.height - textHeight) / 2) + text.draw(at: NSPoint(x: 20, y: y), withAttributes: attrs) NSGraphicsContext.restoreGraphicsState() return rep.cgImage } func runOCRSelfTest() { - let cases: [(expected: String, contains: String)] = [ - ("Kapture OCR test 1234, hello world", "Kapture OCR test 1234"), - ("你好世界，这是一个测试", "你好世界"), + let cases: [(expected: String, contains: String, fontSize: CGFloat, width: CGFloat, height: CGFloat)] = [ + ("Kapture OCR test 1234, hello world", "Kapture OCR test 1234", 56, 1900, 220), + ("你好世界，这是一个测试", "你好世界", 56, 1900, 220), + // Tiny text: needs the upscale retry to read anything at all. + ("会话导出并发冲突报错 export fail", "导出", 7, 320, 40), ] var index = 0 @@ -154,9 +159,9 @@ func runOCRSelfTest() { guard index < cases.count else { exit(0) } let current = cases[index] index += 1 - guard let image = renderTestImage(current.expected) else { exit(1) } + guard let image = renderTestImage(current.expected, fontSize: current.fontSize, width: current.width, height: current.height) else { exit(1) } OCRService.shared.recognize(image) { recognized in - print("case: \(current.expected)") + print("case: \(current.expected) @\(Int(current.fontSize))pt") print("got: \(recognized)") let ok = recognized.contains(current.contains) print(ok ? "PASS" : "FAIL") diff --git a/Sources/Kapture/OCRService.swift b/Sources/Kapture/OCRService.swift index ca321ff..409c0c1 100644 --- a/Sources/Kapture/OCRService.swift +++ b/Sources/Kapture/OCRService.swift @@ -8,34 +8,66 @@ final class OCRService { func recognize(_ cgImage: CGImage, completion: @escaping (String) -> Void) { queue.async { - let request = VNRecognizeTextRequest() - request.recognitionLevel = .accurate - request.usesLanguageCorrection = true - request.recognitionLanguages = ["en-US", "zh-Hans", "zh-Hant", "ja-JP", "ko-KR"] - // Without this, Vision defaults to the first listed language and - // renders CJK as Latin garbage (verified: 你好世界 -> "177WR"). - if #available(macOS 13.0, *) { - request.automaticallyDetectsLanguage = true - } - - let handler = VNImageRequestHandler(cgImage: cgImage, options: [:]) - do { - try handler.perform([request]) - } catch { - completion("") + let text = Self.perform(cgImage) + // Tiny text (small UI labels) often reads as nothing at native + // size; retry upscaled and keep whichever pass read more. + guard text.count < 3 else { + completion(text) return } - - let observations = request.results ?? [] - let lines = observations - .sorted { a, b in - let ay = a.boundingBox.midY - let by = b.boundingBox.midY - if abs(ay - by) > 0.02 { return ay > by } - return a.boundingBox.minX < b.boundingBox.minX - } - .compactMap { $0.topCandidates(1).first?.string } - completion(lines.joined(separator: "\n")) + var best = text + for factor in [CGFloat(2), CGFloat(3)] { + guard let upscaled = Self.upscaled(cgImage, factor: factor) else { continue } + let candidate = Self.perform(upscaled) + if candidate.count > best.count { best = candidate } + if best.count >= 3 { break } + } + completion(best) } } + + private static func perform(_ cgImage: CGImage) -> String { + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + request.recognitionLanguages = ["en-US", "zh-Hans", "zh-Hant", "ja-JP", "ko-KR"] + // Without this, Vision defaults to the first listed language and + // renders CJK as Latin garbage (verified: 你好世界 -> "177WR"). + if #available(macOS 13.0, *) { + request.automaticallyDetectsLanguage = true + } + + let handler = VNImageRequestHandler(cgImage: cgImage, options: [:]) + do { + try handler.perform([request]) + } catch { + return "" + } + + let observations = request.results ?? [] + let lines = observations + .sorted { a, b in + let ay = a.boundingBox.midY + let by = b.boundingBox.midY + if abs(ay - by) > 0.02 { return ay > by } + return a.boundingBox.minX < b.boundingBox.minX + } + .compactMap { $0.topCandidates(1).first?.string } + return lines.joined(separator: "\n") + } + + private static func upscaled(_ cgImage: CGImage, factor: CGFloat) -> CGImage? { + let width = Int(CGFloat(cgImage.width) * factor) + let height = Int(CGFloat(cgImage.height) * factor) + guard width > 0, height > 0, width <= 20000, height <= 20000 else { return nil } + guard let ctx = CGContext( + data: nil, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { return nil } + ctx.interpolationQuality = .high + ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height))) + return ctx.makeImage() + } } diff --git a/build-app.sh b/build-app.sh index 564643e..61f6aba 100755 --- a/build-app.sh +++ b/build-app.sh @@ -108,9 +108,9 @@ cat > "$APP/Contents/Info.plist" <<'PLIST' CFBundleIconFile Icon CFBundleShortVersionString - 1.10.1 + 1.10.2 CFBundleVersion - 101 + 102 LSUIElement NSHighResolutionCapable