commit e814e63c0fa5ec1f7f4ce2f2f0bcaab451783ea2 Author: TZGyn Date: Sun Aug 9 18:15:21 2026 +0800 Add Kapture: menu-bar screenshot OCR app with Cmd+Shift+S hotkey diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bfc90bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +# SwiftPM +.build/ +.swiftpm/ + +# App bundle (build artifact, regenerate with ./build-app.sh) +Kapture.app/ + +# macOS +.DS_Store + +# Xcode +xcuserdata/ +DerivedData/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..7afb11c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + "configurations": [ + { + "type": "swift", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:kapture}", + "name": "Debug Kapture", + "target": "Kapture", + "configuration": "debug", + "preLaunchTask": "swift: Build Debug Kapture" + }, + { + "type": "swift", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:kapture}", + "name": "Release Kapture", + "target": "Kapture", + "configuration": "release", + "preLaunchTask": "swift: Build Release Kapture" + } + ] +} \ No newline at end of file diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..110f09d --- /dev/null +++ b/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "Kapture", + platforms: [.macOS(.v13)], + targets: [ + .executableTarget( + name: "Kapture", + path: "Sources/Kapture" + ) + ] +) diff --git a/README.md b/README.md new file mode 100644 index 0000000..b6c1461 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# Kapture + +A native macOS menu bar app that captures a screen region and extracts the text +from it using on-device Vision OCR. Press **⌘⇧S**, drag to select an area, and +the recognized text is copied to your clipboard and shown in a floating window. + +No network calls, no API keys — everything runs locally. + +## Features + +- Global hotkey **⌘⇧S** works from any app +- Region selection overlay (all screens, Escape cancels) +- OCR for English, Chinese (simplified/traditional), Japanese, Korean +- Text is auto-copied to the clipboard and shown in a floating window +- Extract text from an existing image file via the menu bar icon +- Menu bar icon with permission status + +## Build & run + +```bash +./build-app.sh # produces Kapture.app +open Kapture.app # or drag to /Applications +``` + +For development without the bundle: + +```bash +swift run Kapture # runs from terminal (will appear in Dock) +``` + +## First launch + +1. Click the menu bar icon → **Grant Screen Recording Permission…** + (required so the app can capture the screen; text extraction from files + works without it). +2. Press **⌘⇧S**, drag to select a region, release. +3. The extracted text lands on your clipboard and in the result window. + +Note: macOS may require re-granting Screen Recording permission after +rebuilding the app, since the binary identity changes. + +## Smoke test + +```bash +swift run Kapture --test-ocr # renders text, OCRs it, prints PASS/FAIL +``` + +## How it works + +| Concern | Tech | +|-------------|------| +| Hotkey | Carbon `RegisterEventHotKey` (⌘⇧S, global) | +| Selection | Borderless `NSPanel` overlay per screen, dim + clear-cut rectangle | +| Capture | `CGWindowListCreateImage` (retina-aware) | +| OCR | Vision `VNRecognizeTextRequest` (accurate, multi-language) | +| UI | SwiftUI `MenuBarExtra` + floating result window | + +## Layout + +``` +Package.swift SwiftPM manifest (macOS 13+) +build-app.sh builds a proper .app bundle + ad-hoc codesign +Sources/Kapture/ + App.swift app entry, menu bar UI, permission helper, OCR smoke test + HotkeyManager.swift global ⌘⇧S hotkey + OverlayView.swift selection overlay panel + drawing + CaptureController.swift selection flow, screen capture, coordinate mapping + OCRService.swift Vision text recognition + ResultWindow.swift floating result window + image-file extraction +``` diff --git a/Sources/Kapture/App.swift b/Sources/Kapture/App.swift new file mode 100644 index 0000000..bfe99b7 --- /dev/null +++ b/Sources/Kapture/App.swift @@ -0,0 +1,92 @@ +import SwiftUI +import AppKit +import CoreGraphics + +@main +struct KaptureApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate + + var body: some Scene { + MenuBarExtra("Kapture", systemImage: "text.viewfinder") { + MenuView() + } + .menuBarExtraStyle(.menu) + } +} + +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + if CommandLine.arguments.contains("--test-ocr") { + runOCRSelfTest() + return + } + HotkeyManager.shared.onTrigger = { CaptureController.shared.begin() } + HotkeyManager.shared.register() + } +} + +struct MenuView: View { + @State private var granted = PermissionManager.isGranted + + var body: some View { + Button("Capture & OCR") { CaptureController.shared.begin() } + .keyboardShortcut("s", modifiers: [.command, .shift]) + + Button("Extract from Image File…") { ResultWindow.shared.showOpenPanel() } + + Divider() + + if granted { + Text("Screen Recording: Granted") + } else { + Button("Grant Screen Recording Permission…") { + PermissionManager.request() + granted = PermissionManager.isGranted + } + } + + Divider() + + Button("Quit Kapture") { NSApp.terminate(nil) } + .keyboardShortcut("q") + } +} + +enum PermissionManager { + static var isGranted: Bool { CGPreflightScreenCaptureAccess() } + + static func request() { + CGRequestScreenCaptureAccess() + } +} + +func runOCRSelfTest() { + let text = "Kapture OCR test 1234, hello world" + let size = NSSize(width: 1600, height: 220) + guard let rep = NSBitmapImageRep( + bitmapDataPlanes: nil, pixelsWide: Int(size.width), pixelsHigh: Int(size.height), + bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, + colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0 + ) else { exit(1) } + + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: rep) + NSColor.white.setFill() + NSRect(origin: .zero, size: size).fill() + let attrs: [NSAttributedString.Key: Any] = [ + .font: NSFont.boldSystemFont(ofSize: 56), + .foregroundColor: NSColor.black, + ] + text.draw(at: NSPoint(x: 60, y: 75), withAttributes: attrs) + NSGraphicsContext.restoreGraphicsState() + + guard let cgImage = rep.cgImage else { exit(1) } + OCRService.shared.recognize(cgImage) { recognized in + print("expected: \(text)") + print("got: \(recognized)") + let ok = recognized == text + print(ok ? "PASS" : "FAIL") + exit(ok ? 0 : 1) + } + RunLoop.main.run() +} diff --git a/Sources/Kapture/CaptureController.swift b/Sources/Kapture/CaptureController.swift new file mode 100644 index 0000000..1f53842 --- /dev/null +++ b/Sources/Kapture/CaptureController.swift @@ -0,0 +1,102 @@ +import AppKit +import CoreGraphics + +final class CaptureController: NSObject { + static let shared = CaptureController() + + private var overlays: [NSWindow] = [] + private(set) var isSelecting = false + private var previousApp: NSRunningApplication? + + func begin() { + guard !isSelecting else { return } + guard PermissionManager.isGranted else { + PermissionManager.request() + return + } + isSelecting = true + previousApp = NSWorkspace.shared.frontmostApplication + if #available(macOS 14.0, *) { + NSApp.activate() + } else { + NSApp.activate(ignoringOtherApps: true) + } + overlays = NSScreen.screens.map { screen in + OverlayWindow( + screen: screen, + onComplete: { [weak self] rect in self?.finishSelection(rect) }, + onCancel: { [weak self] in self?.cancelSelection() } + ) + } + overlays.forEach { $0.makeKeyAndOrderFront(nil) } + } + + private func finishSelection(_ rect: NSRect) { + hideOverlays() + guard rect.width >= 2, rect.height >= 2 else { + cancelSelection() + return + } + DispatchQueue.main.async { [self] in + capture(rect) + } + } + + private func cancelSelection() { + hideOverlays() + isSelecting = false + restorePreviousApp() + } + + private func hideOverlays() { + overlays.forEach { $0.orderOut(nil) } + overlays = [] + } + + private func restorePreviousApp() { + previousApp?.activate(options: [.activateIgnoringOtherApps]) + previousApp = nil + } + + private func capture(_ rect: NSRect) { + guard let image = ScreenCapture.capture(rect) else { + isSelecting = false + restorePreviousApp() + ResultWindow.shared.show("Capture failed. Check Screen Recording permission.") + return + } + OCRService.shared.recognize(image) { [weak self] text in + DispatchQueue.main.async { + self?.isSelecting = false + self?.restorePreviousApp() + if text.isEmpty { + ResultWindow.shared.show("No text found in the selected area.") + } else { + let pasteboard = NSPasteboard.general + pasteboard.clearContents() + pasteboard.setString(text, forType: .string) + ResultWindow.shared.show(text) + } + } + } + } +} + +enum ScreenCapture { + static func capture(_ rect: NSRect) -> CGImage? { + guard let primary = NSScreen.screens.first(where: { $0.frame.origin == .zero }) + ?? NSScreen.screens.first else { return nil } + let cgRect = CGRect( + x: rect.minX, + y: primary.frame.height - rect.maxY, + width: rect.width, + height: rect.height + ) + return CGWindowListCreateImage( + cgRect, + .optionOnScreenOnly, + kCGNullWindowID, + [.bestResolution, .boundsIgnoreFraming] + ) + } +} diff --git a/Sources/Kapture/HotkeyManager.swift b/Sources/Kapture/HotkeyManager.swift new file mode 100644 index 0000000..500e724 --- /dev/null +++ b/Sources/Kapture/HotkeyManager.swift @@ -0,0 +1,34 @@ +import Carbon +import AppKit + +final class HotkeyManager { + static let shared = HotkeyManager() + + var onTrigger: (() -> Void)? + + private var hotKeyRef: EventHotKeyRef? + private var eventHandlerRef: EventHandlerRef? + private var hotKeyID = EventHotKeyID(signature: 0x4B505452, id: 1) + + private init() {} + + func register() { + var eventType = EventTypeSpec( + eventClass: OSType(kEventClassKeyboard), + eventKind: UInt32(kEventHotKeyPressed) + ) + InstallEventHandler(GetApplicationEventTarget(), { _, _, _ in + HotkeyManager.shared.onTrigger?() + return noErr + }, 1, &eventType, nil, &eventHandlerRef) + + RegisterEventHotKey( + UInt32(kVK_ANSI_S), + UInt32(cmdKey | shiftKey), + hotKeyID, + GetApplicationEventTarget(), + 0, + &hotKeyRef + ) + } +} diff --git a/Sources/Kapture/OCRService.swift b/Sources/Kapture/OCRService.swift new file mode 100644 index 0000000..e64d11b --- /dev/null +++ b/Sources/Kapture/OCRService.swift @@ -0,0 +1,36 @@ +import AppKit +import Vision + +final class OCRService { + static let shared = OCRService() + + private let queue = DispatchQueue(label: "dev.tzgyn.kapture.ocr", qos: .userInitiated) + + func recognize(_ cgImage: CGImage, completion: @escaping (String) -> Void) { + queue.async { + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + request.recognitionLanguages = ["en-US", "zh-Hans", "zh-Hant", "ja-JP", "ko-KR"] + + let handler = VNImageRequestHandler(cgImage: cgImage, options: [:]) + do { + try handler.perform([request]) + } catch { + completion("") + return + } + + let observations = request.results ?? [] + let lines = observations + .sorted { a, b in + let ay = a.boundingBox.midY + let by = b.boundingBox.midY + if abs(ay - by) > 0.02 { return ay > by } + return a.boundingBox.minX < b.boundingBox.minX + } + .compactMap { $0.topCandidates(1).first?.string } + completion(lines.joined(separator: "\n")) + } + } +} diff --git a/Sources/Kapture/OverlayView.swift b/Sources/Kapture/OverlayView.swift new file mode 100644 index 0000000..6b89d25 --- /dev/null +++ b/Sources/Kapture/OverlayView.swift @@ -0,0 +1,122 @@ +import AppKit + +final class OverlayWindow: NSPanel { + init(screen: NSScreen, onComplete: @escaping (NSRect) -> Void, onCancel: @escaping () -> Void) { + let content = OverlayView(onComplete: onComplete, onCancel: onCancel) + super.init( + contentRect: screen.frame, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + contentView = content + level = .screenSaver + collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary, .transient] + backgroundColor = .clear + isOpaque = false + hasShadow = false + isReleasedWhenClosed = false + acceptsMouseMovedEvents = true + } + + override var canBecomeKey: Bool { true } +} + +final class OverlayView: NSView { + private let onComplete: (NSRect) -> Void + private let onCancel: () -> Void + + private var startPoint: NSPoint? + private(set) var selection: NSRect? + + init(onComplete: @escaping (NSRect) -> Void, onCancel: @escaping () -> Void) { + self.onComplete = onComplete + self.onCancel = onCancel + super.init(frame: .zero) + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override var acceptsFirstResponder: Bool { true } + + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } + + override func resetCursorRects() { + addCursorRect(bounds, cursor: .crosshair) + } + + override func mouseDown(with event: NSEvent) { + let p = convert(event.locationInWindow, from: nil) + startPoint = p + selection = NSRect(origin: p, size: .zero) + needsDisplay = true + } + + override func mouseDragged(with event: NSEvent) { + guard let start = startPoint else { return } + let p = convert(event.locationInWindow, from: nil) + selection = NSRect( + x: min(start.x, p.x), + y: min(start.y, p.y), + width: abs(p.x - start.x), + height: abs(p.y - start.y) + ) + needsDisplay = true + } + + override func mouseUp(with event: NSEvent) { + guard let sel = selection else { return } + if sel.width >= 2, sel.height >= 2 { + onComplete(window!.convertToScreen(sel)) + } else { + onCancel() + } + } + + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 { + onCancel() + } else { + super.keyDown(with: event) + } + } + + override func draw(_ dirtyRect: NSRect) { + NSColor.black.withAlphaComponent(0.35).setFill() + NSBezierPath(rect: bounds).fill() + + guard let sel = selection, sel.width > 0, sel.height > 0 else { return } + + NSGraphicsContext.current?.saveGraphicsState() + NSGraphicsContext.current?.compositingOperation = .clear + NSColor.black.setFill() + NSBezierPath(rect: sel).fill() + NSGraphicsContext.current?.restoreGraphicsState() + + let border = NSBezierPath(rect: sel) + border.lineWidth = 1.5 + NSColor.systemOrange.setStroke() + border.stroke() + + let label = "\(Int(sel.width)) × \(Int(sel.height))" + let attrs: [NSAttributedString.Key: Any] = [ + .font: NSFont.monospacedDigitSystemFont(ofSize: 12, weight: .medium), + .foregroundColor: NSColor.white, + ] + let labelSize = label.size(withAttributes: attrs) + let labelRect = NSRect( + x: sel.maxX - labelSize.width - 8, + y: sel.minY - labelSize.height - 10, + width: labelSize.width + 8, + height: labelSize.height + 6 + ) + NSColor.black.withAlphaComponent(0.6).setFill() + NSBezierPath(roundedRect: labelRect, xRadius: 4, yRadius: 4).fill() + label.draw( + at: NSPoint(x: labelRect.minX + 4, y: labelRect.minY + 3), + withAttributes: attrs + ) + } +} diff --git a/Sources/Kapture/ResultWindow.swift b/Sources/Kapture/ResultWindow.swift new file mode 100644 index 0000000..fc9f84a --- /dev/null +++ b/Sources/Kapture/ResultWindow.swift @@ -0,0 +1,75 @@ +import SwiftUI +import AppKit +import UniformTypeIdentifiers + +final class ResultWindow { + static let shared = ResultWindow() + + private var window: NSWindow? + + func show(_ text: String) { + window?.close() + let hosting = NSHostingController(rootView: ResultView(text: text) { [weak self] in + self?.window?.close() + }) + let w = NSWindow(contentViewController: hosting) + w.title = "Kapture — Extracted Text" + w.level = .floating + w.styleMask = [.titled, .closable, .resizable] + w.setContentSize(NSSize(width: 520, height: 380)) + w.isReleasedWhenClosed = false + + let mouse = NSEvent.mouseLocation + if let screen = NSScreen.screens.first(where: { $0.frame.contains(mouse) }) ?? NSScreen.main { + let x = min(max(mouse.x - screen.frame.minX - 260, screen.frame.minX + 8), screen.frame.maxX - 528) + let y = min(max(mouse.y - screen.frame.minY + 60, screen.frame.minY + 8), screen.frame.maxY - 388) + w.setFrameOrigin(NSPoint(x: x, y: y)) + } + w.makeKeyAndOrderFront(nil) + window = w + } + + func showOpenPanel() { + let panel = NSOpenPanel() + panel.allowedContentTypes = [.image] + panel.allowsMultipleSelection = false + guard panel.runModal() == .OK, let url = panel.url else { return } + guard let image = NSImage(contentsOf: url), + let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else { + show("Could not load the selected image.") + return + } + OCRService.shared.recognize(cgImage) { text in + DispatchQueue.main.async { + self.show(text.isEmpty ? "No text found in the image." : text) + } + } + } +} + +struct ResultView: View { + let text: String + let onClose: () -> Void + + var body: some View { + VStack(spacing: 12) { + TextEditor(text: .constant(text)) + .font(.system(.body)) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.secondary.opacity(0.4)) + ) + HStack { + Button("Copy to Clipboard") { + let pasteboard = NSPasteboard.general + pasteboard.clearContents() + pasteboard.setString(text, forType: .string) + } + Spacer() + Button("Close") { onClose() } + } + } + .padding() + .frame(minWidth: 480, minHeight: 340) + } +} diff --git a/build-app.sh b/build-app.sh new file mode 100755 index 0000000..e602ea4 --- /dev/null +++ b/build-app.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" + +swift build -c release + +BIN=".build/release/Kapture" +APP="Kapture.app" +CERT="kapture-dev" + +if ! security find-identity -v -p codesigning 2>/dev/null | grep -q "\"$CERT\""; then + echo "Creating self-signed code signing identity '$CERT' (one-time)..." + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + cat > "$TMP/cert.cnf" <<'CNF' +[ req ] +distinguished_name = dn +x509_extensions = v3 +prompt = no + +[ dn ] +CN = kapture-dev + +[ v3 ] +basicConstraints = critical, CA:true +keyUsage = critical, digitalSignature +extendedKeyUsage = codeSigning +CNF + openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \ + -keyout "$TMP/key.pem" -out "$TMP/cert.pem" \ + -config "$TMP/cert.cnf" 2>/dev/null + openssl pkcs12 -export \ + -inkey "$TMP/key.pem" -in "$TMP/cert.pem" \ + -out "$TMP/cert.p12" -passout pass:kapture 2>/dev/null + security import "$TMP/cert.p12" -k "$HOME/Library/Keychains/login.keychain-db" \ + -P kapture -T /usr/bin/codesign 2>/dev/null +fi + +rm -rf "$APP" +mkdir -p "$APP/Contents/MacOS" +cp "$BIN" "$APP/Contents/MacOS/Kapture" + +cat > "$APP/Contents/Info.plist" <<'PLIST' + + + + + CFBundleExecutable + Kapture + CFBundleIdentifier + dev.tzgyn.kapture + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Kapture + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSUIElement + + NSHighResolutionCapable + + NSPrincipalClass + NSApplication + + +PLIST + +codesign --force --sign "$CERT" "$APP" + +echo "Built $APP (signed with stable identity '$CERT') — drag to /Applications or run: open $APP"