From 21ccd40a6b651e9de0f620a75da31cab5d7a9f39 Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Sun, 31 Dec 2023 14:36:23 +0100 Subject: [PATCH 01/42] Add setting option for partial OCR #38 --- rem/SettingsManager.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index 60abf7b..633c6c2 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -12,6 +12,7 @@ import SwiftUI struct AppSettings: Codable { var saveEverythingCopiedToClipboard: Bool var enableCmdScrollShortcut: Bool + var onlyOCRFrontmostWindow: Bool = false } // The settings manager handles saving and loading the settings @@ -51,6 +52,8 @@ struct SettingsView: View { .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { settingsManager.saveSettings() } Toggle("Allow opening / closing timeline with CMD + Scroll", isOn: $settingsManager.settings.enableCmdScrollShortcut) .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { settingsManager.saveSettings() } + Toggle("Only OCR region of active application window", isOn: $settingsManager.settings.onlyOCRFrontmostWindow) + .onChange(of: settingsManager.settings.onlyOCRFrontmostWindow) { settingsManager.saveSettings() } } } .padding() From a643c027384a78202c3c44ac72b9007eeec5f476 Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Sun, 31 Dec 2023 17:55:38 +0100 Subject: [PATCH 02/42] OCR for Active Window Only #38 - Added `WindowHelper` to get bounds for active window - Added `ImageUtil` to create a cropped NSImage based on the bounds of the active window - Changed `remApp` to use the aboce code to only OCR the active window iff the `onlyOCRFrontmostWindow` setting is `true` --- rem.xcodeproj/project.pbxproj | 12 +++++++ rem/ImageUtils.swift | 44 +++++++++++++++++++++++ rem/WindowHelper.swift | 66 +++++++++++++++++++++++++++++++++++ rem/remApp.swift | 25 ++++++++++--- 4 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 rem/ImageUtils.swift create mode 100644 rem/WindowHelper.swift diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 3822a7d..3641f24 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -31,6 +31,10 @@ 96DBA3C82B40164E0000CFBE /* (null) in Sources */ = {isa = PBXBuildFile; }; 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96DBA3E62B403ED90000CFBE /* Timings.swift */; }; 96F062182B35111D00695621 /* Search.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96F062172B35111D00695621 /* Search.swift */; }; + BF5FEB8E2B41AF6900744FC2 /* WindowHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */; }; + BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */; }; + BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; + BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -220,6 +224,8 @@ 96E66BB42B2F5745006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96E66BCC2B2F574D006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96F062172B35111D00695621 /* Search.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Search.swift; sourceTree = ""; }; + BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WindowHelper.swift; sourceTree = ""; }; + BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageUtils.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -287,6 +293,7 @@ 969F3F092B3B7F760085787B /* Info.plist */, 961C96122B2EB7DB0093F228 /* TimelineView.swift */, 961C95D92B2E19B30093F228 /* remApp.swift */, + BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */, 961C95DB2B2E19B30093F228 /* ContentView.swift */, 961C95DD2B2E19B40093F228 /* Assets.xcassets */, 961C95E22B2E19B40093F228 /* rem.entitlements */, @@ -296,6 +303,7 @@ 96B0DA372B3A02530030E8AE /* ClipboardManager.swift */, 96B0DA392B3A08280030E8AE /* TextMerger.swift */, 969F3F072B3B7C7C0085787B /* RemFileManager.swift */, + BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */, 969F3F0A2B3CB2110085787B /* Field.swift */, 969F3F0C2B3CCEC30085787B /* Ask.swift */, 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */, @@ -655,6 +663,7 @@ buildActionMask = 2147483647; files = ( 961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */, + BF5FEB8E2B41AF6900744FC2 /* WindowHelper.swift in Sources */, 961C96152B2EBEE50093F228 /* DB.swift in Sources */, 96B0DA3A2B3A08280030E8AE /* TextMerger.swift in Sources */, 96F062182B35111D00695621 /* Search.swift in Sources */, @@ -668,6 +677,7 @@ 969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */, 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */, 9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */, + BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -676,6 +686,8 @@ buildActionMask = 2147483647; files = ( 961C95EC2B2E19B40093F228 /* remTests.swift in Sources */, + BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */, + BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/rem/ImageUtils.swift b/rem/ImageUtils.swift new file mode 100644 index 0000000..1a2f452 --- /dev/null +++ b/rem/ImageUtils.swift @@ -0,0 +1,44 @@ +// +// ImageUtils.swift +// rem +// +// Created by Stefan Eletzhofer on 31.12.23. +// + +import Foundation +import Cocoa + +// Function to create a cropped NSImage from CGImage +func croppedImage(from cgImage: CGImage, frame: CGRect) -> NSImage? { + // Create a new NSImage with the CGImage + let nsImage = NSImage(cgImage: cgImage, size: NSZeroSize) + + // Begin a new image context to draw the cropped image + let imageRep = NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: Int(frame.width), + pixelsHigh: Int(frame.height), + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: NSColorSpaceName.deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: imageRep!) + + // Set the context to the portion of the image we want to draw + let context = NSGraphicsContext.current!.cgContext + context.draw(cgImage, in: CGRect(x: -frame.origin.x, y: frame.origin.y - (nsImage.size.height - frame.height), width: nsImage.size.width, height: nsImage.size.height)) + + NSGraphicsContext.restoreGraphicsState() + + // Create a new NSImage from the image representation + let croppedImage = NSImage(size: frame.size) + croppedImage.addRepresentation(imageRep!) + + return croppedImage +} diff --git a/rem/WindowHelper.swift b/rem/WindowHelper.swift new file mode 100644 index 0000000..17efbd6 --- /dev/null +++ b/rem/WindowHelper.swift @@ -0,0 +1,66 @@ +// +// WindowHelper.swift +// rem +// +// Created by Stefan Eletzhofer on 31.12.23. +// +// Adopted from https://stackoverflow.com/questions/72763902/swift-get-a-list-of-opened-app-windows-with-desktop-number-and-position-info +// + +import os +import Foundation +import CoreGraphics +import Cocoa + +class WindowHelper { + private let logger = Logger() + static let shared: WindowHelper = WindowHelper() + + func getActiveWindowInfo(forApp application: NSRunningApplication) -> NSDictionary? { + let windowsInfo = CGWindowListCopyWindowInfo(CGWindowListOption.optionOnScreenOnly, CGWindowID(0)) + + let pid = application.processIdentifier + logger.debug("Active application PID: \(pid)") + if pid > 0 { + let windowInfos = Array.fromCFArray(records: windowsInfo) ?? [] + + // it appears that the first window listed is the active window ... + for windowInfo in windowInfos.filter({$0["kCGWindowOwnerPID"] as! Int64 == pid}) { + return windowInfo + } + } + return nil + } + + func getActiveWindowBounds(forApp application: NSRunningApplication) -> CGRect? { + if + let wi : NSDictionary = getActiveWindowInfo(forApp: application), + let bounds : NSDictionary = wi["kCGWindowBounds"] as? NSDictionary, + let x = bounds["X"] as? Double, + let y = bounds["Y"] as? Double, + let width = bounds["Width"] as? Double, + let height = bounds["Height"] as? Double + { + return CGRect(x: x, y: y, width: width, height: height) + } + return nil + } + +} + +extension Array { + static func fromCFArray(records: CFArray?) -> Array? { + var result: [Element]? + if let records = records { + for i in 0.. Date: Sun, 31 Dec 2023 19:48:40 +0100 Subject: [PATCH 03/42] Remove ImageHelper As suggested by @jasonjmcghee. The cropping can be done using CGImage base methods. --- rem.xcodeproj/project.pbxproj | 4 --- rem/ImageHelper.swift | 49 ----------------------------------- rem/remApp.swift | 38 ++++++++++++++------------- 3 files changed, 20 insertions(+), 71 deletions(-) delete mode 100644 rem/ImageHelper.swift diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 3641f24..bf8bd68 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -16,7 +16,6 @@ 961C95F82B2E19B40093F228 /* remUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95F72B2E19B40093F228 /* remUITestsLaunchTests.swift */; }; 961C96132B2EB7DB0093F228 /* TimelineView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96122B2EB7DB0093F228 /* TimelineView.swift */; }; 961C96152B2EBEE50093F228 /* DB.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96142B2EBEE50093F228 /* DB.swift */; }; - 9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9670E1342B41683B005728F5 /* ImageHelper.swift */; }; 969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */; }; 969F3EFF2B3A8C4D0085787B /* HotKey in Frameworks */ = {isa = PBXBuildFile; productRef = 969F3EFE2B3A8C4D0085787B /* HotKey */; }; 969F3F082B3B7C7C0085787B /* RemFileManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969F3F072B3B7C7C0085787B /* RemFileManager.swift */; }; @@ -209,7 +208,6 @@ 961C960D2B2E73840093F228 /* ffmpeg */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.executable"; path = ffmpeg; sourceTree = ""; }; 961C96122B2EB7DB0093F228 /* TimelineView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimelineView.swift; sourceTree = ""; }; 961C96142B2EBEE50093F228 /* DB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DB.swift; sourceTree = ""; }; - 9670E1342B41683B005728F5 /* ImageHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = ""; }; 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsManager.swift; sourceTree = ""; }; 969F3F072B3B7C7C0085787B /* RemFileManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemFileManager.swift; sourceTree = ""; }; 969F3F092B3B7F760085787B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; @@ -308,7 +306,6 @@ 969F3F0C2B3CCEC30085787B /* Ask.swift */, 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */, 96DBA3E62B403ED90000CFBE /* Timings.swift */, - 9670E1342B41683B005728F5 /* ImageHelper.swift */, ); path = rem; sourceTree = ""; @@ -676,7 +673,6 @@ 969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */, 969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */, 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */, - 9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */, BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/rem/ImageHelper.swift b/rem/ImageHelper.swift deleted file mode 100644 index c6de7ec..0000000 --- a/rem/ImageHelper.swift +++ /dev/null @@ -1,49 +0,0 @@ -// -// ImageHelper.swift -// rem -// -// Created by Jason McGhee on 12/31/23. -// - -import Foundation -import os -import SwiftUI - -class ImageHelper { - private static let logger = Logger( - subsystem: Bundle.main.bundleIdentifier!, - category: String(describing: ImageHelper.self) - ) - - // Useful for debugging... - static func pngData(from nsImage: NSImage) -> Data? { - guard let tiffRepresentation = nsImage.tiffRepresentation, - let bitmapImage = NSBitmapImageRep(data: tiffRepresentation) else { - logger.error("Failed to get TIFF representation of NSImage") - return nil - } - - guard let pngData = bitmapImage.representation(using: .png, properties: [:]) else { - logger.error("Failed to convert NSImage to PNG") - return nil - } - - return pngData - } - - static func saveNSImage(image: NSImage, path: String) { - let pngData = pngData(from: image) - do { - if let savedir = RemFileManager.shared.getSaveDir() { - let outputPath = savedir.appendingPathComponent("\(path).png").path - let fileURL = URL(fileURLWithPath: outputPath) - try pngData?.write(to: fileURL) - logger.info("PNG file written successfully") - } else { - logger.error("Error writing PNG file") - } - } catch { - logger.error("Error writing PNG file: \(error)") - } - } -} diff --git a/rem/remApp.swift b/rem/remApp.swift index ce529a1..109f875 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -301,6 +301,23 @@ class AppDelegate: NSObject, NSApplicationDelegate { guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return } let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName) + if settingsManager.settings.onlyOCRFrontmostWindow { + // User wants to perform OCR on only active window. + if + // let window = shareableContent.windows.first(where: { $0.owningApplication?.processID == NSWorkspace.shared.frontmostApplication?.processIdentifier }), + let app = NSWorkspace.shared.frontmostApplication, + let bounds = WindowHelper.shared.getActiveWindowBounds(forApp: app), + let cropped = image.cropping(to: bounds) + { + logger.debug("bounds \(String(describing: bounds))") + logger.debug("cropped image \(String(describing: cropped))") + self.performOCR(frameId: frameId, on: cropped) + } + } else { + // default: User wants to perform OCR on full display. + self.performOCR(frameId: frameId, on: image) + } + await processScreenshot(frameId: frameId, image: image, frame: display.frame) screenshotQueue.asyncAfter(deadline: .now() + 2) { [weak self] in @@ -350,22 +367,6 @@ class AppDelegate: NSObject, NSApplicationDelegate { // } private func processScreenshot(frameId: Int64, image: CGImage, frame: CGRect) async { - var ocrFrame = frame - var nsImage = NSImage(cgImage: image, size: NSZeroSize) - if settingsManager.settings.onlyOCRFrontmostWindow { - if let app = NSWorkspace.shared.frontmostApplication { - if let bounds = WindowHelper.shared.getActiveWindowBounds(forApp: app) { - logger.debug("active window bounds = \(String(describing: bounds))") - ocrFrame = bounds - if let img = croppedImage(from: image, frame: ocrFrame) { - logger.debug("Cropped image to \(String(describing: ocrFrame)): \(img)") - nsImage = img - } - } - } - } - - self.performOCR(frameId: frameId, on: nsImage, frame: ocrFrame) let bitmapRep = NSBitmapImageRep(cgImage: image) guard let data = bitmapRep.representation(using: .png, properties: [:]) else { return } @@ -505,7 +506,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } - private func performOCR(frameId: Int64, on image: NSImage, frame: CGRect) { + private func performOCR(frameId: Int64, on image: CGImage) { ocrQueue.async { // Select only a region... / active window? // let invWidth = 1 / CGFloat(image.width) @@ -520,7 +521,8 @@ class AppDelegate: NSObject, NSApplicationDelegate { do { let configuration = ImageAnalyzer.Configuration([.text]) self.logger.debug("Start OCR ...") - let analysis = try await self.imageAnalyzer.analyze(image, orientation: CGImagePropertyOrientation.up, configuration: configuration) + let nsImage = NSImage(cgImage: image, size: NSZeroSize) + let analysis = try await self.imageAnalyzer.analyze(nsImage, orientation: CGImagePropertyOrientation.up, configuration: configuration) let textToAssociate = analysis.transcript self.logger.debug("Analysed text: \(textToAssociate)") var texts = [textToAssociate] From 8d358c02f9c820ac9fa997ffefee62142ef10e83 Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Sun, 31 Dec 2023 19:51:09 +0100 Subject: [PATCH 04/42] Fix compiler warnings. --- rem/TimelineView.swift | 2 +- rem/remApp.swift | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rem/TimelineView.swift b/rem/TimelineView.swift index 2de62e1..2e1110c 100644 --- a/rem/TimelineView.swift +++ b/rem/TimelineView.swift @@ -297,7 +297,7 @@ class TimelineViewModel: ObservableObject { func updateIndex(withDelta delta: Double) { // Logic to update the index based on the delta // This method will be called from AppDelegate - var nextValue = currentFrameContinuous - delta * speedFactor + let nextValue = currentFrameContinuous - delta * speedFactor let maxValue = Double(DatabaseManager.shared.getMaxFrame()) let clampedValue = min(max(1, nextValue), maxValue) self.currentFrameContinuous = clampedValue diff --git a/rem/remApp.swift b/rem/remApp.swift index 109f875..3545d54 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -395,7 +395,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { if let savedir = RemFileManager.shared.getSaveDir() { let outputPath = savedir.appendingPathComponent("output-\(Date().timeIntervalSince1970).mp4").path - DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) + let id = DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) // Setup the FFmpeg process for the chunk let ffmpegProcess = Process() @@ -431,7 +431,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } // Write the chunk data to the FFmpeg process - for (index, data) in chunk.enumerated() { + for (_, data) in chunk.enumerated() { do { try ffmpegInputPipe.fileHandleForWriting.write(contentsOf: data) } catch { From 6ad45ab3112471142dcf94c2642611b2cb9bd677 Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Sun, 31 Dec 2023 19:51:49 +0100 Subject: [PATCH 05/42] Remove dead comment / bit rot. --- rem/remApp.swift | 9 --------- 1 file changed, 9 deletions(-) diff --git a/rem/remApp.swift b/rem/remApp.swift index 3545d54..d8a8641 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -508,15 +508,6 @@ class AppDelegate: NSObject, NSApplicationDelegate { private func performOCR(frameId: Int64, on image: CGImage) { ocrQueue.async { - // Select only a region... / active window? -// let invWidth = 1 / CGFloat(image.width) -// let invHeight = 1 / CGFloat(image.height) -// let regionOfInterest = CGRect( -// x: min(max(0, frame.minX * invWidth), 1), -// y: min(max(0, frame.minY * invHeight), 1), -// width: min(max(0, frame.width * invWidth), 1), -// height: min(max(0, frame.height * invHeight), 1) -// ) Task { do { let configuration = ImageAnalyzer.Configuration([.text]) From b3c85e0d279fdf4709ea3c51491f62d1474dc09b Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Sun, 31 Dec 2023 20:03:01 +0100 Subject: [PATCH 06/42] Fix another warning. --- rem/remApp.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rem/remApp.swift b/rem/remApp.swift index d8a8641..358e028 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -395,7 +395,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { if let savedir = RemFileManager.shared.getSaveDir() { let outputPath = savedir.appendingPathComponent("output-\(Date().timeIntervalSince1970).mp4").path - let id = DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) + let _ = DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) // Setup the FFmpeg process for the chunk let ffmpegProcess = Process() From aecf872bcabfd9bffad7b30c82527d68ba162708 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 08:25:26 -0800 Subject: [PATCH 07/42] [rem-3]: try to get everything ffmpeg + app store to work --- rem.xcodeproj/project.pbxproj | 12 ++++++++++-- .../xcschemes/xcschememanagement.plist | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 3822a7d..da3dcbe 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -855,6 +855,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -891,6 +892,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -992,14 +994,17 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; ONLY_ACTIVE_ARCH = YES; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -1014,13 +1019,16 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; diff --git a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist index 9ac48dd..f44b8a3 100644 --- a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist @@ -28,12 +28,12 @@ ffmpegX.xcscheme_^#shared#^_ orderHint - 5 + 4 rem.xcscheme_^#shared#^_ orderHint - 4 + 5 From f064ee73bf6abe47ca2a2c9415ddb83a112528ed Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 09:58:22 -0800 Subject: [PATCH 08/42] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 91333e3..93db93b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![image](https://github.com/jasonjmcghee/rem/assets/1522149/bc7368dc-90b5-42a3-abba-9d365b368ddb) + # rem 🧠 Remember everything. (very alpha - [download anyway](https://github.com/jasonjmcghee/rem/releases)) From 238a1caaeffc1228fe9a72b6968ade6d05eadeb1 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 09:58:45 -0800 Subject: [PATCH 09/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93db93b..8a3b536 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![image](https://github.com/jasonjmcghee/rem/assets/1522149/bc7368dc-90b5-42a3-abba-9d365b368ddb) -# rem +# rem 🧠 Remember everything. (very alpha - [download anyway](https://github.com/jasonjmcghee/rem/releases)) From 8549525cade5c58837b79499e93f99081ec13a79 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 17:02:07 -0800 Subject: [PATCH 10/42] [rem-52]: fix compatability with 13 --- rem.xcodeproj/project.pbxproj | 6 ++---- .../jason.xcuserdatad/xcschemes/xcschememanagement.plist | 4 ++-- rem/Search.swift | 7 +------ rem/SettingsManager.swift | 4 ++-- rem/TimelineView.swift | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index da3dcbe..3e2851d 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -771,7 +771,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -830,7 +830,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -867,7 +867,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -904,7 +903,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist index f44b8a3..9ac48dd 100644 --- a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist @@ -28,12 +28,12 @@ ffmpegX.xcscheme_^#shared#^_ orderHint - 4 + 5 rem.xcscheme_^#shared#^_ orderHint - 5 + 4 diff --git a/rem/Search.swift b/rem/Search.swift index 9d706a2..b8b2197 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -57,7 +57,7 @@ struct SearchBar: View { onSearch() } } // Trigger search when user submits - .onChange(of: text) { + .onChange(of: text) { _ in debounceSearch.debounce { Task { onSearch() @@ -402,8 +402,3 @@ extension View { self.modifier(ScrollViewOffsetModifier(onBottomReached: action)) } } - -#Preview("Hello", traits: .defaultLayout) { - SearchView(onThumbnailClick: { _ in }) -} - diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index 60abf7b..227e0c9 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -48,9 +48,9 @@ struct SettingsView: View { .padding(.bottom) Form { Toggle("Remember everything copied to clipboard", isOn: $settingsManager.settings.saveEverythingCopiedToClipboard) - .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { _ in settingsManager.saveSettings() } Toggle("Allow opening / closing timeline with CMD + Scroll", isOn: $settingsManager.settings.enableCmdScrollShortcut) - .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { _ in settingsManager.saveSettings() } } } .padding() diff --git a/rem/TimelineView.swift b/rem/TimelineView.swift index 2de62e1..9bd57e9 100644 --- a/rem/TimelineView.swift +++ b/rem/TimelineView.swift @@ -46,7 +46,7 @@ struct TimelineView: View { ) .frame(width: frame.width, height: frame.height) .ignoresSafeArea(.all) - .onChange(of: viewModel.currentFrameIndex) { + .onChange(of: viewModel.currentFrameIndex) { _ in analyzeCurrentImage() } .onAppear { From 05fc0d8ce79e3ad6c23020ee7ccca25a74a02c3d Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 17:16:22 -0800 Subject: [PATCH 11/42] whoops, add missing changes in ffmpeg binary due to signing --- Resources/ffmpeg | Bin 1974216 -> 1992896 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/Resources/ffmpeg b/Resources/ffmpeg index 39be00942754b548fcadbf5a4c0a850e015a3816..6dec3d3544e0180d751c79b80adc2fca981a37b1 100755 GIT binary patch delta 2021 zcmX@Hzv95f$_nSOSP8fmjNNrGZ#xdw`v++EOojW?+J3g(glDAPr2IwmA?n z+X^7f&cI-|jDZoP!tVF4dpsa{3kD{j7DkX-#t5K90uZxLU+`Gg#({wkC;AZ8zD(TtXB)&UlYOEJh6mc;hIOo7w?axWC~ow#Vad6Ea@_+R%53xQtKv$3zp%~2>zEh^5;&$CrB)H6_0 zNX<*mPsz+nw^a%aanrR>vbW>1QFQTl4v7qKRVc{GEG|(92zB!DbXHK()z^0{D9B0G z*LMkVQ3&wy3=UBMnxU`n>Zhcjlu=SrV5P5LUS6)3SWr-(ld6}Tp9@l1tRGO6Uyxc< zQt1OUO&6#`FQp_!31|<*cog^8aVaR+q+}+SfEn4Tm3BZ=Kn4QMD^5)=Ey^sZ1aWnX z6Z2A%@+)lgL25vnN{UKT_3gMgZNQr9F?D6;WuyYNLv?V%Gyt{gLoBq>2Rq1)3m9HS zz+ek4@MdCU+~8)Q%p-^26^xAk4di*GkQ9l*+zPc7sN&Z>ZpP__meTs8CE~~`5rnSOSP8fmjNNrGZ#xdw`v++EOi>U|@n|geFdrU-x)`3Dd@& zfr*8Ife8p17~+9A0wh2Ez++h(^|Z9yg4A>dLkr0hqFp)s0VhK5ajCTYU(@V3ak0=05Nqdy8r+H From 3431d2a7acddf6eeeec19a429001fceafd94853f Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Tue, 2 Jan 2024 08:35:40 -0800 Subject: [PATCH 12/42] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 8a3b536..6d0fc93 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,6 @@ Please log any bugs / issues you find! Looking at this code and grimacing? Want to help turn this project into something awesome? Please contribute. I haven't written Swift since 2017. I'm sure you'll write better code than me. -Please look at https://github.com/jasonjmcghee/rem/issues/3 and if you _didn't_ experience anything let us know! -If you can help solve it, please do! - --- I think the idea of recording everything you see has the potential to change how we interact From 4cf1aebda2ce07e6acb3866321733d8afb0dbe1b Mon Sep 17 00:00:00 2001 From: drull Date: Sun, 31 Dec 2023 14:42:17 -0300 Subject: [PATCH 13/42] MacOS standard settings window. --- rem/remApp.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rem/remApp.swift b/rem/remApp.swift index 358e028..0b20c56 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -32,7 +32,7 @@ struct remApp: App { var body: some Scene { // Empty scene, as we are controlling everything through the AppDelegate - Settings { EmptyView() } + Settings { SettingsView(settingsManager: appDelegate.settingsManager) } } } @@ -190,7 +190,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { menu.addItem( withTitle: "Settings", action: #selector(self.openSettings), - keyEquivalent: "" + keyEquivalent: "," ) menu.addItem(NSMenuItem(title: "Quit", action: #selector(self.quitApp), keyEquivalent: "q")) self.statusBarItem.menu = menu From c1d19a48af63bc3fbf8fce6a73f3daa60d071b11 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Sun, 31 Dec 2023 10:44:49 -0800 Subject: [PATCH 14/42] Update README.md Removing contributors section from README for now as `contrib.rocks` is not updating and github exposes this on the right bar anyway --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 554c021..91333e3 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,6 @@ Also, that means there is no tracking / analytics of any kind, which means I don - [ ] Fine-grained purging / trimming / selecting recording - [ ] Multi-monitor support -## Contributors ✨ - - - - ### FAQ - Where is my data? - Click "Show Me My Data" in the tray / status icon menu From 0fdc27979c16661a716cc0811a68e03be93a4152 Mon Sep 17 00:00:00 2001 From: sudocurse <947518+sudocurse@users.noreply.github.com> Date: Sun, 31 Dec 2023 17:00:02 -0800 Subject: [PATCH 15/42] Prompt typo --- rem/ClipboardManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rem/ClipboardManager.swift b/rem/ClipboardManager.swift index a4d3665..8baaefb 100644 --- a/rem/ClipboardManager.swift +++ b/rem/ClipboardManager.swift @@ -39,7 +39,7 @@ class ClipboardManager { pasteboard.clearContents() let finalContents = string.isEmpty ? "No context. Is remembering disabled?" : """ - Below is the text that's been on my screen recently. ------------- \(string) ------------------ Above is the text that's been on my screen recently. Please answer whatever I ask using the provided information about what has been on the scren recently. Do not say anything else or give any other information. Only answer the query. --------------------------\n + Below is the text that's been on my screen recently. ------------- \(string) ------------------ Above is the text that's been on my screen recently. Please answer whatever I ask using the provided information about what has been on the screen recently. Do not say anything else or give any other information. Only answer the query. --------------------------\n """ pasteboard.setString(finalContents, forType: .string) From 6acc6861d2fc25fa1ae0a8dd247b580d0d4513ea Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 08:25:26 -0800 Subject: [PATCH 16/42] [rem-3]: try to get everything ffmpeg + app store to work --- rem.xcodeproj/project.pbxproj | 12 ++++++++++-- .../xcschemes/xcschememanagement.plist | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index bf8bd68..d1d4ac7 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -863,6 +863,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -899,6 +900,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -1000,14 +1002,17 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; ONLY_ACTIVE_ARCH = YES; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -1022,13 +1027,16 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; diff --git a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist index 9ac48dd..f44b8a3 100644 --- a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist @@ -28,12 +28,12 @@ ffmpegX.xcscheme_^#shared#^_ orderHint - 5 + 4 rem.xcscheme_^#shared#^_ orderHint - 4 + 5 From 3e29eedc5c92e2184737e1e4c9a5cf21050855cd Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 09:58:22 -0800 Subject: [PATCH 17/42] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 91333e3..93db93b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![image](https://github.com/jasonjmcghee/rem/assets/1522149/bc7368dc-90b5-42a3-abba-9d365b368ddb) + # rem 🧠 Remember everything. (very alpha - [download anyway](https://github.com/jasonjmcghee/rem/releases)) From 3bc0234d20d6a06e3eb1dc53216bf0f7cce333cb Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 09:58:45 -0800 Subject: [PATCH 18/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93db93b..8a3b536 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![image](https://github.com/jasonjmcghee/rem/assets/1522149/bc7368dc-90b5-42a3-abba-9d365b368ddb) -# rem +# rem 🧠 Remember everything. (very alpha - [download anyway](https://github.com/jasonjmcghee/rem/releases)) From 0e500d11c68218e22a06dc6df52bd7a4ea536e25 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 17:02:07 -0800 Subject: [PATCH 19/42] [rem-52]: fix compatability with 13 --- rem.xcodeproj/project.pbxproj | 6 ++---- .../jason.xcuserdatad/xcschemes/xcschememanagement.plist | 4 ++-- rem/Search.swift | 7 +------ rem/SettingsManager.swift | 6 +++--- rem/TimelineView.swift | 2 +- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index d1d4ac7..af8da7f 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -779,7 +779,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -838,7 +838,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -875,7 +875,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -912,7 +911,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist index f44b8a3..9ac48dd 100644 --- a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist @@ -28,12 +28,12 @@ ffmpegX.xcscheme_^#shared#^_ orderHint - 4 + 5 rem.xcscheme_^#shared#^_ orderHint - 5 + 4 diff --git a/rem/Search.swift b/rem/Search.swift index 9d706a2..b8b2197 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -57,7 +57,7 @@ struct SearchBar: View { onSearch() } } // Trigger search when user submits - .onChange(of: text) { + .onChange(of: text) { _ in debounceSearch.debounce { Task { onSearch() @@ -402,8 +402,3 @@ extension View { self.modifier(ScrollViewOffsetModifier(onBottomReached: action)) } } - -#Preview("Hello", traits: .defaultLayout) { - SearchView(onThumbnailClick: { _ in }) -} - diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index 633c6c2..84a84bd 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -49,11 +49,11 @@ struct SettingsView: View { .padding(.bottom) Form { Toggle("Remember everything copied to clipboard", isOn: $settingsManager.settings.saveEverythingCopiedToClipboard) - .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { _ in settingsManager.saveSettings() } Toggle("Allow opening / closing timeline with CMD + Scroll", isOn: $settingsManager.settings.enableCmdScrollShortcut) - .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { _ in settingsManager.saveSettings() } Toggle("Only OCR region of active application window", isOn: $settingsManager.settings.onlyOCRFrontmostWindow) - .onChange(of: settingsManager.settings.onlyOCRFrontmostWindow) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.onlyOCRFrontmostWindow) { _ in settingsManager.saveSettings() } } } .padding() diff --git a/rem/TimelineView.swift b/rem/TimelineView.swift index 2e1110c..28f38a4 100644 --- a/rem/TimelineView.swift +++ b/rem/TimelineView.swift @@ -46,7 +46,7 @@ struct TimelineView: View { ) .frame(width: frame.width, height: frame.height) .ignoresSafeArea(.all) - .onChange(of: viewModel.currentFrameIndex) { + .onChange(of: viewModel.currentFrameIndex) { _ in analyzeCurrentImage() } .onAppear { From 4ec58d39ee536d07870a96db32039a7accf33401 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Mon, 1 Jan 2024 17:16:22 -0800 Subject: [PATCH 20/42] whoops, add missing changes in ffmpeg binary due to signing --- Resources/ffmpeg | Bin 1974216 -> 1992896 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/Resources/ffmpeg b/Resources/ffmpeg index 39be00942754b548fcadbf5a4c0a850e015a3816..6dec3d3544e0180d751c79b80adc2fca981a37b1 100755 GIT binary patch delta 2021 zcmX@Hzv95f$_nSOSP8fmjNNrGZ#xdw`v++EOojW?+J3g(glDAPr2IwmA?n z+X^7f&cI-|jDZoP!tVF4dpsa{3kD{j7DkX-#t5K90uZxLU+`Gg#({wkC;AZ8zD(TtXB)&UlYOEJh6mc;hIOo7w?axWC~ow#Vad6Ea@_+R%53xQtKv$3zp%~2>zEh^5;&$CrB)H6_0 zNX<*mPsz+nw^a%aanrR>vbW>1QFQTl4v7qKRVc{GEG|(92zB!DbXHK()z^0{D9B0G z*LMkVQ3&wy3=UBMnxU`n>Zhcjlu=SrV5P5LUS6)3SWr-(ld6}Tp9@l1tRGO6Uyxc< zQt1OUO&6#`FQp_!31|<*cog^8aVaR+q+}+SfEn4Tm3BZ=Kn4QMD^5)=Ey^sZ1aWnX z6Z2A%@+)lgL25vnN{UKT_3gMgZNQr9F?D6;WuyYNLv?V%Gyt{gLoBq>2Rq1)3m9HS zz+ek4@MdCU+~8)Q%p-^26^xAk4di*GkQ9l*+zPc7sN&Z>ZpP__meTs8CE~~`5rnSOSP8fmjNNrGZ#xdw`v++EOi>U|@n|geFdrU-x)`3Dd@& zfr*8Ife8p17~+9A0wh2Ez++h(^|Z9yg4A>dLkr0hqFp)s0VhK5ajCTYU(@V3ak0=05Nqdy8r+H From dd1d46e93a9e90f473209b06d8d4a2e8a6b44f42 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Tue, 2 Jan 2024 08:35:40 -0800 Subject: [PATCH 21/42] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 8a3b536..6d0fc93 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,6 @@ Please log any bugs / issues you find! Looking at this code and grimacing? Want to help turn this project into something awesome? Please contribute. I haven't written Swift since 2017. I'm sure you'll write better code than me. -Please look at https://github.com/jasonjmcghee/rem/issues/3 and if you _didn't_ experience anything let us know! -If you can help solve it, please do! - --- I think the idea of recording everything you see has the potential to change how we interact From 8543334fcfb6b90a60254c6f6822438ee57836fe Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Tue, 2 Jan 2024 13:00:40 -0800 Subject: [PATCH 22/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d0fc93..1db6599 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ I think the idea of recording everything you see has the potential to change how with our computers, and believe it should be open source. Also, from a privacy / security perspective, this is like... pretty scary stuff, and I want the code open -so we know for certain that nothing is leaving your laptop. Even logging to Sentry has the potential to +so we know for certain that nothing is leaving your laptop. Even telemetry has the potential to leak private info. This is 100% local. Please, read the code yourself. From adb3570786578276cbc8b5ca4873ea1dff7b3c5c Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Tue, 2 Jan 2024 22:07:48 +0100 Subject: [PATCH 23/42] Resurrected accidentally removed file --- rem.xcodeproj/project.pbxproj | 8 ++++++ rem/ImageHelper.swift | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 rem/ImageHelper.swift diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index af8da7f..3b54bff 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -34,6 +34,9 @@ BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */; }; BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; + BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; + BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; + BF5FEBFD2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -224,6 +227,7 @@ 96F062172B35111D00695621 /* Search.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Search.swift; sourceTree = ""; }; BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WindowHelper.swift; sourceTree = ""; }; BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageUtils.swift; sourceTree = ""; }; + BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -290,6 +294,7 @@ children = ( 969F3F092B3B7F760085787B /* Info.plist */, 961C96122B2EB7DB0093F228 /* TimelineView.swift */, + BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */, 961C95D92B2E19B30093F228 /* remApp.swift */, BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */, 961C95DB2B2E19B30093F228 /* ContentView.swift */, @@ -674,6 +679,7 @@ 969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */, 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */, BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */, + BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -682,6 +688,7 @@ buildActionMask = 2147483647; files = ( 961C95EC2B2E19B40093F228 /* remTests.swift in Sources */, + BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */, BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */, BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */, ); @@ -693,6 +700,7 @@ files = ( 961C95F62B2E19B40093F228 /* remUITests.swift in Sources */, 961C95F82B2E19B40093F228 /* remUITestsLaunchTests.swift in Sources */, + BF5FEBFD2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/rem/ImageHelper.swift b/rem/ImageHelper.swift new file mode 100644 index 0000000..c6de7ec --- /dev/null +++ b/rem/ImageHelper.swift @@ -0,0 +1,49 @@ +// +// ImageHelper.swift +// rem +// +// Created by Jason McGhee on 12/31/23. +// + +import Foundation +import os +import SwiftUI + +class ImageHelper { + private static let logger = Logger( + subsystem: Bundle.main.bundleIdentifier!, + category: String(describing: ImageHelper.self) + ) + + // Useful for debugging... + static func pngData(from nsImage: NSImage) -> Data? { + guard let tiffRepresentation = nsImage.tiffRepresentation, + let bitmapImage = NSBitmapImageRep(data: tiffRepresentation) else { + logger.error("Failed to get TIFF representation of NSImage") + return nil + } + + guard let pngData = bitmapImage.representation(using: .png, properties: [:]) else { + logger.error("Failed to convert NSImage to PNG") + return nil + } + + return pngData + } + + static func saveNSImage(image: NSImage, path: String) { + let pngData = pngData(from: image) + do { + if let savedir = RemFileManager.shared.getSaveDir() { + let outputPath = savedir.appendingPathComponent("\(path).png").path + let fileURL = URL(fileURLWithPath: outputPath) + try pngData?.write(to: fileURL) + logger.info("PNG file written successfully") + } else { + logger.error("Error writing PNG file") + } + } catch { + logger.error("Error writing PNG file: \(error)") + } + } +} From 13d1859c9ac2baa09ada22f260ae402b897e5241 Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Tue, 2 Jan 2024 23:37:32 +0100 Subject: [PATCH 24/42] Fix image cropping and active window search Fix a bug where images would be cropped wrongly due to `CGImage` and `SCDisplay` size units count in pixels and points. We now determine the scale factor and update the crop rectangle. Fixed another bug where the active window would not be correctly found due to not filtering out off-screen windows. --- rem/ImageHelper.swift | 13 +++++++++++++ rem/ImageUtils.swift | 44 ------------------------------------------- rem/remApp.swift | 16 ++++++++-------- 3 files changed, 21 insertions(+), 52 deletions(-) delete mode 100644 rem/ImageUtils.swift diff --git a/rem/ImageHelper.swift b/rem/ImageHelper.swift index c6de7ec..574cbcf 100644 --- a/rem/ImageHelper.swift +++ b/rem/ImageHelper.swift @@ -46,4 +46,17 @@ class ImageHelper { logger.error("Error writing PNG file: \(error)") } } + + static func saveCGImage(image: CGImage, path: String) { + saveNSImage(image: NSImage(cgImage: image, size: NSZeroSize), path: path) + } + + static func cropImage(image: CGImage, frame: CGRect, scale: CGFloat) -> CGImage? { + let cropZone = CGRect( + x: frame.origin.x * scale, + y: frame.origin.y * scale, + width: frame.size.width * scale, + height: frame.size.height * scale) + return image.cropping(to: cropZone) + } } diff --git a/rem/ImageUtils.swift b/rem/ImageUtils.swift deleted file mode 100644 index 1a2f452..0000000 --- a/rem/ImageUtils.swift +++ /dev/null @@ -1,44 +0,0 @@ -// -// ImageUtils.swift -// rem -// -// Created by Stefan Eletzhofer on 31.12.23. -// - -import Foundation -import Cocoa - -// Function to create a cropped NSImage from CGImage -func croppedImage(from cgImage: CGImage, frame: CGRect) -> NSImage? { - // Create a new NSImage with the CGImage - let nsImage = NSImage(cgImage: cgImage, size: NSZeroSize) - - // Begin a new image context to draw the cropped image - let imageRep = NSBitmapImageRep( - bitmapDataPlanes: nil, - pixelsWide: Int(frame.width), - pixelsHigh: Int(frame.height), - bitsPerSample: 8, - samplesPerPixel: 4, - hasAlpha: true, - isPlanar: false, - colorSpaceName: NSColorSpaceName.deviceRGB, - bytesPerRow: 0, - bitsPerPixel: 0 - ) - - NSGraphicsContext.saveGraphicsState() - NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: imageRep!) - - // Set the context to the portion of the image we want to draw - let context = NSGraphicsContext.current!.cgContext - context.draw(cgImage, in: CGRect(x: -frame.origin.x, y: frame.origin.y - (nsImage.size.height - frame.height), width: nsImage.size.width, height: nsImage.size.height)) - - NSGraphicsContext.restoreGraphicsState() - - // Create a new NSImage from the image representation - let croppedImage = NSImage(size: frame.size) - croppedImage.addRepresentation(imageRep!) - - return croppedImage -} diff --git a/rem/remApp.swift b/rem/remApp.swift index 0b20c56..54c5daf 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -303,14 +303,15 @@ class AppDelegate: NSObject, NSApplicationDelegate { if settingsManager.settings.onlyOCRFrontmostWindow { // User wants to perform OCR on only active window. + + // We need to determine the scale factor for cropping. CGImage is + // measured in pixels, display sizes are measured in points. + let scale = max(CGFloat(image.width) / CGFloat(display.width), CGFloat(image.height) / CGFloat(display.height)) + if - // let window = shareableContent.windows.first(where: { $0.owningApplication?.processID == NSWorkspace.shared.frontmostApplication?.processIdentifier }), - let app = NSWorkspace.shared.frontmostApplication, - let bounds = WindowHelper.shared.getActiveWindowBounds(forApp: app), - let cropped = image.cropping(to: bounds) + let window = shareableContent.windows.first(where: { $0.isOnScreen && $0.owningApplication?.processID == NSWorkspace.shared.frontmostApplication?.processIdentifier }), + let cropped = ImageHelper.cropImage(image: image, frame: window.frame, scale: scale) { - logger.debug("bounds \(String(describing: bounds))") - logger.debug("cropped image \(String(describing: cropped))") self.performOCR(frameId: frameId, on: cropped) } } else { @@ -511,11 +512,10 @@ class AppDelegate: NSObject, NSApplicationDelegate { Task { do { let configuration = ImageAnalyzer.Configuration([.text]) - self.logger.debug("Start OCR ...") let nsImage = NSImage(cgImage: image, size: NSZeroSize) let analysis = try await self.imageAnalyzer.analyze(nsImage, orientation: CGImagePropertyOrientation.up, configuration: configuration) let textToAssociate = analysis.transcript - self.logger.debug("Analysed text: \(textToAssociate)") + // self.logger.debug("Analysed text: \(textToAssociate)") var texts = [textToAssociate] if self.settingsManager.settings.saveEverythingCopiedToClipboard { let newClipboardText = ClipboardManager.shared.getClipboardIfChanged() ?? "" From 6989143ca5fd988ce881b5c6e13f719b5316d603 Mon Sep 17 00:00:00 2001 From: drull Date: Tue, 2 Jan 2024 19:40:42 -0300 Subject: [PATCH 25/42] Filter and search by app name --- rem/DB.swift | 20 ++++++++++++++++++-- rem/Search.swift | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index f815da3..eb10a57 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -244,14 +244,30 @@ class DatabaseManager { } func search(searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = allText + var query = allText .join(frames, on: frames[id] == allText[frameId]) .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .filter(text.match("*\(searchText)*")) .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .limit(limit, offset: offset) var results: [(Int64, String, String?, Date, String, Int64)] = [] + + if searchText.hasPrefix("!") { + let components = searchText.components(separatedBy: " ") + let appName = components.first?.dropFirst() ?? "" + let textToSearch = components.dropFirst().joined(separator: " ") + + if textToSearch.isEmpty { + query = query.filter(frames[activeApplicationName].like("%\(appName)%")) + } else { + query = query.filter(frames[activeApplicationName].like("%\(appName)%") && text.like("%\(textToSearch)%")) + } + } else { + //normal search + //uncommenting the following line allows to search both text and app names at the same time + query = query.filter(text.like("%\(searchText)%")) // || frames[activeApplicationName].like("%\(searchText)%")) + } + do { for row in try db.prepare(query) { let frameId = row[allText[frameId]] diff --git a/rem/Search.swift b/rem/Search.swift index b8b2197..087c77b 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -136,6 +136,7 @@ class SearchResult: ObservableObject, Identifiable { // .replacingOccurrences(of: "\\s+", with: "\\s*", options: .regularExpression) .replacingOccurrences(of: "(", with: "\\(") .replacingOccurrences(of: ")", with: "\\)") + .replacingOccurrences(of: #"!([^ ]*) "#, with: "", options: .regularExpression) if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive), let match = regex.firstMatch(in: text, options: [], range: NSRange(location: 0, length: text.utf16.count)) { From a289f4ebef6addb57df3b5b6b7413ee7b6490d1e Mon Sep 17 00:00:00 2001 From: Stefan Eletzhofer Date: Tue, 2 Jan 2024 23:44:01 +0100 Subject: [PATCH 26/42] Remove unneeded code The active window selection is better done using the ScreenCaptureKit results. --- rem/WindowHelper.swift | 66 ------------------------------------------ 1 file changed, 66 deletions(-) delete mode 100644 rem/WindowHelper.swift diff --git a/rem/WindowHelper.swift b/rem/WindowHelper.swift deleted file mode 100644 index 17efbd6..0000000 --- a/rem/WindowHelper.swift +++ /dev/null @@ -1,66 +0,0 @@ -// -// WindowHelper.swift -// rem -// -// Created by Stefan Eletzhofer on 31.12.23. -// -// Adopted from https://stackoverflow.com/questions/72763902/swift-get-a-list-of-opened-app-windows-with-desktop-number-and-position-info -// - -import os -import Foundation -import CoreGraphics -import Cocoa - -class WindowHelper { - private let logger = Logger() - static let shared: WindowHelper = WindowHelper() - - func getActiveWindowInfo(forApp application: NSRunningApplication) -> NSDictionary? { - let windowsInfo = CGWindowListCopyWindowInfo(CGWindowListOption.optionOnScreenOnly, CGWindowID(0)) - - let pid = application.processIdentifier - logger.debug("Active application PID: \(pid)") - if pid > 0 { - let windowInfos = Array.fromCFArray(records: windowsInfo) ?? [] - - // it appears that the first window listed is the active window ... - for windowInfo in windowInfos.filter({$0["kCGWindowOwnerPID"] as! Int64 == pid}) { - return windowInfo - } - } - return nil - } - - func getActiveWindowBounds(forApp application: NSRunningApplication) -> CGRect? { - if - let wi : NSDictionary = getActiveWindowInfo(forApp: application), - let bounds : NSDictionary = wi["kCGWindowBounds"] as? NSDictionary, - let x = bounds["X"] as? Double, - let y = bounds["Y"] as? Double, - let width = bounds["Width"] as? Double, - let height = bounds["Height"] as? Double - { - return CGRect(x: x, y: y, width: width, height: height) - } - return nil - } - -} - -extension Array { - static func fromCFArray(records: CFArray?) -> Array? { - var result: [Element]? - if let records = records { - for i in 0.. Date: Tue, 2 Jan 2024 20:51:59 -0800 Subject: [PATCH 27/42] clean up project.pbxproj --- rem.xcodeproj/project.pbxproj | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 3b54bff..4ad89c0 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -30,10 +30,6 @@ 96DBA3C82B40164E0000CFBE /* (null) in Sources */ = {isa = PBXBuildFile; }; 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96DBA3E62B403ED90000CFBE /* Timings.swift */; }; 96F062182B35111D00695621 /* Search.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96F062172B35111D00695621 /* Search.swift */; }; - BF5FEB8E2B41AF6900744FC2 /* WindowHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */; }; - BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */; }; - BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; - BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */; }; BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; BF5FEBFD2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; @@ -225,8 +221,6 @@ 96E66BB42B2F5745006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96E66BCC2B2F574D006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96F062172B35111D00695621 /* Search.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Search.swift; sourceTree = ""; }; - BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WindowHelper.swift; sourceTree = ""; }; - BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageUtils.swift; sourceTree = ""; }; BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -296,7 +290,6 @@ 961C96122B2EB7DB0093F228 /* TimelineView.swift */, BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */, 961C95D92B2E19B30093F228 /* remApp.swift */, - BF5FEB902B41D08F00744FC2 /* ImageUtils.swift */, 961C95DB2B2E19B30093F228 /* ContentView.swift */, 961C95DD2B2E19B40093F228 /* Assets.xcassets */, 961C95E22B2E19B40093F228 /* rem.entitlements */, @@ -306,7 +299,6 @@ 96B0DA372B3A02530030E8AE /* ClipboardManager.swift */, 96B0DA392B3A08280030E8AE /* TextMerger.swift */, 969F3F072B3B7C7C0085787B /* RemFileManager.swift */, - BF5FEB8D2B41AF6900744FC2 /* WindowHelper.swift */, 969F3F0A2B3CB2110085787B /* Field.swift */, 969F3F0C2B3CCEC30085787B /* Ask.swift */, 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */, @@ -665,7 +657,6 @@ buildActionMask = 2147483647; files = ( 961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */, - BF5FEB8E2B41AF6900744FC2 /* WindowHelper.swift in Sources */, 961C96152B2EBEE50093F228 /* DB.swift in Sources */, 96B0DA3A2B3A08280030E8AE /* TextMerger.swift in Sources */, 96F062182B35111D00695621 /* Search.swift in Sources */, @@ -678,7 +669,6 @@ 969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */, 969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */, 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */, - BF5FEB912B41D08F00744FC2 /* ImageUtils.swift in Sources */, BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -689,8 +679,6 @@ files = ( 961C95EC2B2E19B40093F228 /* remTests.swift in Sources */, BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */, - BF5FEB8F2B41AF6900744FC2 /* WindowHelper.swift in Sources */, - BF5FEB922B41D08F00744FC2 /* ImageUtils.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; From b031892506b1586fbb389e3eb7a6d879e2f1ecb7 Mon Sep 17 00:00:00 2001 From: drull Date: Wed, 3 Jan 2024 10:47:47 -0300 Subject: [PATCH 28/42] Add picker and get apps names function --- rem/DB.swift | 38 +++++++++++++------------- rem/Search.swift | 69 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index eb10a57..585fd0a 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -244,30 +244,14 @@ class DatabaseManager { } func search(searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - var query = allText + let query = allText .join(frames, on: frames[id] == allText[frameId]) .join(videoChunks, on: frames[chunkId] == videoChunks[id]) + .filter(text.match("*\(searchText)*")) .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .limit(limit, offset: offset) var results: [(Int64, String, String?, Date, String, Int64)] = [] - - if searchText.hasPrefix("!") { - let components = searchText.components(separatedBy: " ") - let appName = components.first?.dropFirst() ?? "" - let textToSearch = components.dropFirst().joined(separator: " ") - - if textToSearch.isEmpty { - query = query.filter(frames[activeApplicationName].like("%\(appName)%")) - } else { - query = query.filter(frames[activeApplicationName].like("%\(appName)%") && text.like("%\(textToSearch)%")) - } - } else { - //normal search - //uncommenting the following line allows to search both text and app names at the same time - query = query.filter(text.like("%\(searchText)%")) // || frames[activeApplicationName].like("%\(searchText)%")) - } - do { for row in try db.prepare(query) { let frameId = row[allText[frameId]] @@ -307,6 +291,24 @@ class DatabaseManager { return results } + func getAllApplicationNames() -> [String] { + var applicationNames: [String] = [] + + do { + let distinctAppsQuery = frames.select(distinct: activeApplicationName) + for row in try db.prepare(distinctAppsQuery) { + if let appName = row[activeApplicationName] { + applicationNames.append(appName) + } + } + } catch { + print("Error fetching application names: \(error)") + } + + return applicationNames + } + + func getImage(index: Int64, maxSize: CGSize? = nil) -> CGImage? { guard let frameData = DatabaseManager.shared.getFrame(forIndex: index) else { return nil } diff --git a/rem/Search.swift b/rem/Search.swift index 087c77b..c0c7c14 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -29,11 +29,14 @@ struct SearchBar: View { var onSearch: () -> Void @Namespace var nspace @FocusState var focused: Bool? - var debounceSearch = Debouncer(delay: 0.3) - + var applicationNameFilter: [String] + @Binding var selectedAppFilterIndex: Int + @Binding var selectedFilterApp: String + var body: some View { HStack { + // Search TextField TextField("Search", text: $text, prompt: Text("Search for something...")) .prefersDefaultFocus(in: nspace) .textFieldStyle(.plain) @@ -56,22 +59,37 @@ struct SearchBar: View { Task { onSearch() } - } // Trigger search when user submits + } .onChange(of: text) { _ in debounceSearch.debounce { Task { onSearch() } } - } // Trigger search when text changes + } .onAppear { self.focused = true } .padding(.horizontal, 10) + + // Application Filter Dropdown + Picker("Select App", selection: $selectedAppFilterIndex) { + ForEach(applicationNameFilter.indices, id: \.self) { index in + Text(applicationNameFilter[index]) + .tag(index) + } + } + .pickerStyle(.menu) + .onChange(of: selectedAppFilterIndex) { _ in + selectedFilterApp = applicationNameFilter[selectedAppFilterIndex] + onSearch() + } + .frame(width: 200) // Adjust width as needed } } } + struct VisualEffectView: NSViewRepresentable { var material: NSVisualEffectView.Material var blendingMode: NSVisualEffectView.BlendingMode @@ -256,12 +274,20 @@ struct ResultsView: View { @State private var searchResults: [SearchResult] = [] @State var limit: Int = 27 @State var offset: Int = 0 - - var onThumbnailClick: (Int64) -> Void // Closure to handle thumbnail click + @State var selectedFilterApp: String = "" + @State var selectedFilterAppIndex: Int = 0 - var body: some View { - VStack { - SearchBar(text: $searchText, onSearch: performSearch) + var onThumbnailClick: (Int64) -> Void + + var body: some View { + VStack { + SearchBar( + text: $searchText, + onSearch: performSearch, + applicationNameFilter: getAppFilterData(), + selectedAppFilterIndex: $selectedFilterAppIndex, + selectedFilterApp: $selectedFilterApp + ) ScrollView { LazyVGrid(columns: Array(repeating: GridItem(.flexible()), count: 3), spacing: 20) { @@ -302,15 +328,34 @@ struct ResultsView: View { } private func getSearchResults() -> [SearchResult] { + var results: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] = [] + if searchText.isEmpty { - let recentResults = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) - return mapResultsToSearchResult(recentResults) + results = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) } else { - let results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) + results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) + } + + if selectedFilterAppIndex == 0 { return mapResultsToSearchResult(results) + } else { + let filteredResults = results.filter { result in + if let appName = result.applicationName { + return appName.lowercased() == selectedFilterApp.lowercased() + } + return false + } + return mapResultsToSearchResult(filteredResults) } } + private func getAppFilterData() -> [String] { + var appFilters = ["All apps"] + let allAppNames = DatabaseManager.shared.getAllApplicationNames() + appFilters.append(contentsOf: allAppNames) + return appFilters + } + private func mapResultsToSearchResult(_ data: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)]) -> [SearchResult] { let searchResults = data.map { item in SearchResult(frameId: item.frameId, applicationName: item.applicationName, fullText: item.fullText?.split(separator: "\n").joined(separator: " "), searchText: searchText, timestamp: item.timestamp) From 0e5c6484b69476037f842f54ff6f62310ea1a587 Mon Sep 17 00:00:00 2001 From: drull Date: Wed, 3 Jan 2024 11:10:45 -0300 Subject: [PATCH 29/42] Add debouncer --- rem/Search.swift | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rem/Search.swift b/rem/Search.swift index c0c7c14..45f4e69 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -72,7 +72,6 @@ struct SearchBar: View { } .padding(.horizontal, 10) - // Application Filter Dropdown Picker("Select App", selection: $selectedAppFilterIndex) { ForEach(applicationNameFilter.indices, id: \.self) { index in Text(applicationNameFilter[index]) @@ -81,10 +80,14 @@ struct SearchBar: View { } .pickerStyle(.menu) .onChange(of: selectedAppFilterIndex) { _ in - selectedFilterApp = applicationNameFilter[selectedAppFilterIndex] - onSearch() + debounceSearch.debounce { + Task { + selectedFilterApp = applicationNameFilter[selectedAppFilterIndex] + onSearch() + } + } } - .frame(width: 200) // Adjust width as needed + .frame(width: 200) } } } From 15109b07fa485e0e0719e422e5abcfd8b54355f8 Mon Sep 17 00:00:00 2001 From: drull Date: Wed, 3 Jan 2024 11:36:48 -0300 Subject: [PATCH 30/42] extract picker to it own struct --- rem/Search.swift | 49 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/rem/Search.swift b/rem/Search.swift index 45f4e69..4db24c1 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -31,7 +31,7 @@ struct SearchBar: View { @FocusState var focused: Bool? var debounceSearch = Debouncer(delay: 0.3) var applicationNameFilter: [String] - @Binding var selectedAppFilterIndex: Int + @Binding var selectedFilterAppIndex: Int @Binding var selectedFilterApp: String var body: some View { @@ -72,23 +72,40 @@ struct SearchBar: View { } .padding(.horizontal, 10) - Picker("Select App", selection: $selectedAppFilterIndex) { - ForEach(applicationNameFilter.indices, id: \.self) { index in - Text(applicationNameFilter[index]) - .tag(index) - } + FilterPicker( + applicationNameFilter: applicationNameFilter, + selectedFilterAppIndex: $selectedFilterAppIndex, + selectedFilterApp: $selectedFilterApp, + debounceSearch: debounceSearch, + onSearch: onSearch + ) + } + } +} + +struct FilterPicker: View { + var applicationNameFilter: [String] + @Binding var selectedFilterAppIndex: Int + @Binding var selectedFilterApp: String + var debounceSearch: Debouncer + var onSearch: () -> Void + + var body: some View { + Picker("Select App", selection: $selectedFilterAppIndex) { + ForEach(applicationNameFilter.indices, id: \.self) { index in + Text(applicationNameFilter[index]) + .tag(index) } - .pickerStyle(.menu) - .onChange(of: selectedAppFilterIndex) { _ in - debounceSearch.debounce { - Task { - selectedFilterApp = applicationNameFilter[selectedAppFilterIndex] - onSearch() - } - } + } + .pickerStyle(.menu) + .onChange(of: selectedFilterAppIndex) { newIndex in + guard newIndex >= 0 && newIndex < applicationNameFilter.count else { + return } - .frame(width: 200) + selectedFilterApp = applicationNameFilter[selectedFilterAppIndex] + onSearch() } + .frame(width: 200) } } @@ -288,7 +305,7 @@ struct ResultsView: View { text: $searchText, onSearch: performSearch, applicationNameFilter: getAppFilterData(), - selectedAppFilterIndex: $selectedFilterAppIndex, + selectedFilterAppIndex: $selectedFilterAppIndex, selectedFilterApp: $selectedFilterApp ) From 851e8987e4492dbb0ce31cbb3ebc09b6629b463a Mon Sep 17 00:00:00 2001 From: drull Date: Wed, 3 Jan 2024 15:42:35 -0300 Subject: [PATCH 31/42] New table for unique app names --- rem/DB.swift | 63 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index 585fd0a..2533230 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -26,6 +26,8 @@ class DatabaseManager { private let videoChunks = Table("video_chunks") private let frames = Table("frames") + private let uniqueAppNames = Table("unique_application_names") + let allText = VirtualTable("allText") private let id = Expression("id") @@ -59,6 +61,7 @@ class DatabaseManager { try db.run(videoChunks.drop(ifExists: true)) try db.run(frames.drop(ifExists: true)) try db.run(allText.drop(ifExists: true)) + try db.run(uniqueAppNames.drop(ifExists: true)) } catch { print("Failed to delete tables") } @@ -83,6 +86,11 @@ class DatabaseManager { t.column(activeApplicationName) }) + try! db.run(uniqueAppNames.create(ifNotExists: true) { t in + t.column(id, primaryKey: .autoincrement) + t.column(activeApplicationName, unique: true) + }) + let config = FTS4Config() .column(frameId, [.unindexed]) .column(text) @@ -137,14 +145,42 @@ class DatabaseManager { } func insertFrame(activeApplicationName: String?) -> Int64 { - // logger.debug("inserting frame: \(self.lastFrameId + 1) at offset: \(self.currentFrameOffset)") let insert = frames.insert(chunkId <- currentChunkId, timestamp <- Date(), offsetIndex <- currentFrameOffset, self.activeApplicationName <- activeApplicationName) let id = try! db.run(insert) currentFrameOffset += 1 lastFrameId = id + + if let appName = activeApplicationName { + //will check if the app name is already in the database. + insertUniqueApplicationNamesIfNeeded(appName) + } + return id } + private func insertUniqueApplicationNamesIfNeeded(_ appName: String) { + let query = uniqueAppNames.filter(activeApplicationName == appName) + + do { + let count = try db.scalar(query.count) + if count == 0 { + print("insert") + insertUniqueApplicationNames(appName) + } + } catch { + print("Error checking existence of app name: \(error)") + } + } + + func insertUniqueApplicationNames(_ appName: String) { + let insert = uniqueAppNames.insert(activeApplicationName <- appName) + do { + try db.run(insert) + } catch { + print("Error inserting unique application name: \(error)") + } + } + func insertTextForFrame(frameId: Int64, text: String) { let insert = allText.insert(self.frameId <- frameId, self.text <- text) try! db.run(insert) @@ -292,23 +328,22 @@ class DatabaseManager { } func getAllApplicationNames() -> [String] { - var applicationNames: [String] = [] - - do { - let distinctAppsQuery = frames.select(distinct: activeApplicationName) - for row in try db.prepare(distinctAppsQuery) { - if let appName = row[activeApplicationName] { - applicationNames.append(appName) - } + var applicationNames: [String] = [] + + do { + let distinctAppsQuery = uniqueAppNames.select(activeApplicationName) + for row in try db.prepare(distinctAppsQuery) { + if let appName = row[activeApplicationName] { + applicationNames.append(appName) } - } catch { - print("Error fetching application names: \(error)") } - - return applicationNames + } catch { + print("Error fetching application names: \(error)") } - + return applicationNames + } + func getImage(index: Int64, maxSize: CGSize? = nil) -> CGImage? { guard let frameData = DatabaseManager.shared.getFrame(forIndex: index) else { return nil } From d5329aeffb60b5954c351254cfe0e6fb97e8a2a6 Mon Sep 17 00:00:00 2001 From: drull Date: Wed, 3 Jan 2024 15:43:14 -0300 Subject: [PATCH 32/42] Add state wrapper --- rem/Search.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rem/Search.swift b/rem/Search.swift index 4db24c1..a2c60b7 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -30,7 +30,7 @@ struct SearchBar: View { @Namespace var nspace @FocusState var focused: Bool? var debounceSearch = Debouncer(delay: 0.3) - var applicationNameFilter: [String] + @State var applicationNameFilter: [String] @Binding var selectedFilterAppIndex: Int @Binding var selectedFilterApp: String From a70cbb56b81b71b0722afa439d42eb873e85e137 Mon Sep 17 00:00:00 2001 From: drull Date: Thu, 4 Jan 2024 10:48:03 -0300 Subject: [PATCH 33/42] Renaming variable & updateFilterApp function --- rem/Search.swift | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/rem/Search.swift b/rem/Search.swift index a2c60b7..47204a3 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -30,9 +30,9 @@ struct SearchBar: View { @Namespace var nspace @FocusState var focused: Bool? var debounceSearch = Debouncer(delay: 0.3) - @State var applicationNameFilter: [String] @Binding var selectedFilterAppIndex: Int @Binding var selectedFilterApp: String + @State private var applicationFilterArray: [String] = [] var body: some View { HStack { @@ -73,7 +73,7 @@ struct SearchBar: View { .padding(.horizontal, 10) FilterPicker( - applicationNameFilter: applicationNameFilter, + applicationFilterArray: applicationFilterArray, selectedFilterAppIndex: $selectedFilterAppIndex, selectedFilterApp: $selectedFilterApp, debounceSearch: debounceSearch, @@ -84,7 +84,7 @@ struct SearchBar: View { } struct FilterPicker: View { - var applicationNameFilter: [String] + @State var applicationFilterArray: [String] @Binding var selectedFilterAppIndex: Int @Binding var selectedFilterApp: String var debounceSearch: Debouncer @@ -92,21 +92,30 @@ struct FilterPicker: View { var body: some View { Picker("Select App", selection: $selectedFilterAppIndex) { - ForEach(applicationNameFilter.indices, id: \.self) { index in - Text(applicationNameFilter[index]) + ForEach(applicationFilterArray.indices, id: \.self) { index in + Text(applicationFilterArray[index]) .tag(index) } } + .onHover(perform: { hovering in + updateAppFilterData() + }) .pickerStyle(.menu) .onChange(of: selectedFilterAppIndex) { newIndex in - guard newIndex >= 0 && newIndex < applicationNameFilter.count else { + guard newIndex >= 0 && newIndex < applicationFilterArray.count else { return } - selectedFilterApp = applicationNameFilter[selectedFilterAppIndex] + selectedFilterApp = applicationFilterArray[selectedFilterAppIndex] onSearch() } .frame(width: 200) } + private func updateAppFilterData() { + var appFilters = ["All apps"] + let allAppNames = DatabaseManager.shared.getAllApplicationNames() + appFilters.append(contentsOf: allAppNames) + applicationFilterArray = appFilters + } } @@ -304,7 +313,6 @@ struct ResultsView: View { SearchBar( text: $searchText, onSearch: performSearch, - applicationNameFilter: getAppFilterData(), selectedFilterAppIndex: $selectedFilterAppIndex, selectedFilterApp: $selectedFilterApp ) @@ -369,13 +377,6 @@ struct ResultsView: View { } } - private func getAppFilterData() -> [String] { - var appFilters = ["All apps"] - let allAppNames = DatabaseManager.shared.getAllApplicationNames() - appFilters.append(contentsOf: allAppNames) - return appFilters - } - private func mapResultsToSearchResult(_ data: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)]) -> [SearchResult] { let searchResults = data.map { item in SearchResult(frameId: item.frameId, applicationName: item.applicationName, fullText: item.fullText?.split(separator: "\n").joined(separator: " "), searchText: searchText, timestamp: item.timestamp) From e28f3bacf45c456f0ee26382edd4d089b209ba76 Mon Sep 17 00:00:00 2001 From: drull Date: Thu, 4 Jan 2024 11:27:44 -0300 Subject: [PATCH 34/42] Optimizing search function --- rem/DB.swift | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ rem/Search.swift | 25 ++++++++++++------------ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index 2533230..78e5467 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -303,6 +303,55 @@ class DatabaseManager { } return results } + func searchFilteredByAppName(appName: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + let query = frames + .join(videoChunks, on: frames[chunkId] == videoChunks[id]) + .filter(frames[activeApplicationName].lowercaseString == appName.lowercased()) // Apply filter by application name + .select(frames[id], frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) + .limit(limit, offset: offset) + + var results: [(Int64, String, String?, Date, String, Int64)] = [] + do { + for row in try db.prepare(query) { + let frameId = row[frames[id]] + let applicationName = row[frames[activeApplicationName]] + let timestamp = row[frames[timestamp]] + let filePath = row[videoChunks[filePath]] + let offsetIndex = row[frames[offsetIndex]] + results.append((frameId, "", applicationName, timestamp, filePath, offsetIndex)) + } + } catch { + print("Search error: \(error)") + } + return results + } + + func searchFilteredByAppNameAndText(appName: String, searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + let query = allText + .join(frames, on: frames[id] == allText[frameId]) + .join(videoChunks, on: frames[chunkId] == videoChunks[id]) + .filter(text.match("*\(searchText)*") && frames[activeApplicationName].lowercaseString == appName.lowercased()) // Apply filter by application name and search text + .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) + .limit(limit, offset: offset) + + var results: [(Int64, String, String?, Date, String, Int64)] = [] + do { + for row in try db.prepare(query) { + let frameId = row[allText[frameId]] + let matchedText = row[text] + let applicationName = row[frames[activeApplicationName]] + let timestamp = row[frames[timestamp]] + let filePath = row[videoChunks[filePath]] + let offsetIndex = row[frames[offsetIndex]] + results.append((frameId, matchedText, applicationName, timestamp, filePath, offsetIndex)) + } + } catch { + print("Search error: \(error)") + } + return results + } + + func getRecentResults(limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { let query = frames diff --git a/rem/Search.swift b/rem/Search.swift index 47204a3..613580f 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -358,24 +358,23 @@ struct ResultsView: View { private func getSearchResults() -> [SearchResult] { var results: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] = [] - if searchText.isEmpty { - results = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) - } else { - results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) - } - if selectedFilterAppIndex == 0 { - return mapResultsToSearchResult(results) + if searchText.isEmpty { + results = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) + } else { + results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) + } } else { - let filteredResults = results.filter { result in - if let appName = result.applicationName { - return appName.lowercased() == selectedFilterApp.lowercased() - } - return false + if searchText.isEmpty { + results = DatabaseManager.shared.searchFilteredByAppName(appName: selectedFilterApp, limit: limit, offset: offset) + } else { + results = DatabaseManager.shared.searchFilteredByAppNameAndText(appName: selectedFilterApp, searchText: searchText, limit: limit, offset: offset) } - return mapResultsToSearchResult(filteredResults) } + + return mapResultsToSearchResult(results) } + private func mapResultsToSearchResult(_ data: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)]) -> [SearchResult] { let searchResults = data.map { item in From 800a6194f3fb2a218400e74d57cbe1027b2713d3 Mon Sep 17 00:00:00 2001 From: drull Date: Thu, 4 Jan 2024 11:55:27 -0300 Subject: [PATCH 35/42] Using the new table in the queries --- rem/DB.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index 78e5467..0f3259f 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -303,10 +303,12 @@ class DatabaseManager { } return results } + func searchFilteredByAppName(appName: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { let query = frames .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .filter(frames[activeApplicationName].lowercaseString == appName.lowercased()) // Apply filter by application name + .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) + .filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) .select(frames[id], frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .limit(limit, offset: offset) @@ -330,7 +332,8 @@ class DatabaseManager { let query = allText .join(frames, on: frames[id] == allText[frameId]) .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .filter(text.match("*\(searchText)*") && frames[activeApplicationName].lowercaseString == appName.lowercased()) // Apply filter by application name and search text + .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) + .filter(text.match("*\(searchText)*") && uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .limit(limit, offset: offset) @@ -351,7 +354,6 @@ class DatabaseManager { return results } - func getRecentResults(limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { let query = frames From f85445c5ef6d15b7f1a2dd22ea72425800169b57 Mon Sep 17 00:00:00 2001 From: drull Date: Thu, 4 Jan 2024 21:08:14 -0300 Subject: [PATCH 36/42] Merging search functions --- rem/DB.swift | 70 +++++++++++------------------------------------- rem/Search.swift | 5 ++-- 2 files changed, 18 insertions(+), 57 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index 0f3259f..a9e57a6 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -164,7 +164,6 @@ class DatabaseManager { do { let count = try db.scalar(query.count) if count == 0 { - print("insert") insertUniqueApplicationNames(appName) } } catch { @@ -279,63 +278,27 @@ class DatabaseManager { return 0 } - func search(searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = allText + func search(appName: String = "", searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + var partialQuery = allText .join(frames, on: frames[id] == allText[frameId]) .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .filter(text.match("*\(searchText)*")) - .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) - .limit(limit, offset: offset) - - var results: [(Int64, String, String?, Date, String, Int64)] = [] - do { - for row in try db.prepare(query) { - let frameId = row[allText[frameId]] - let matchedText = row[text] - let applicationName = row[frames[activeApplicationName]] - let timestamp = row[frames[timestamp]] - let filePath = row[videoChunks[filePath]] - let offsetIndex = row[frames[offsetIndex]] - results.append((frameId, matchedText, applicationName, timestamp, filePath, offsetIndex)) - } - } catch { - print("Search error: \(error)") + + if !appName.isEmpty { + partialQuery = partialQuery.join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) } - return results - } - - func searchFilteredByAppName(appName: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = frames - .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) - .filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) - .select(frames[id], frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) - .limit(limit, offset: offset) - - var results: [(Int64, String, String?, Date, String, Int64)] = [] - do { - for row in try db.prepare(query) { - let frameId = row[frames[id]] - let applicationName = row[frames[activeApplicationName]] - let timestamp = row[frames[timestamp]] - let filePath = row[videoChunks[filePath]] - let offsetIndex = row[frames[offsetIndex]] - results.append((frameId, "", applicationName, timestamp, filePath, offsetIndex)) - } - } catch { - print("Search error: \(error)") + + var query = partialQuery + .filter(text.match("*\(searchText)*")) + + if !appName.isEmpty && searchText.isEmpty { + query = partialQuery + .filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) + } else if !appName.isEmpty && !searchText.isEmpty { + query = query.filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) } - return results - } - func searchFilteredByAppNameAndText(appName: String, searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = allText - .join(frames, on: frames[id] == allText[frameId]) - .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) - .filter(text.match("*\(searchText)*") && uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) - .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) - .limit(limit, offset: offset) + query = query.select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) + .limit(limit, offset: offset) var results: [(Int64, String, String?, Date, String, Int64)] = [] do { @@ -353,7 +316,6 @@ class DatabaseManager { } return results } - func getRecentResults(limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { let query = frames diff --git a/rem/Search.swift b/rem/Search.swift index 613580f..1416688 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -183,7 +183,6 @@ class SearchResult: ObservableObject, Identifiable { // .replacingOccurrences(of: "\\s+", with: "\\s*", options: .regularExpression) .replacingOccurrences(of: "(", with: "\\(") .replacingOccurrences(of: ")", with: "\\)") - .replacingOccurrences(of: #"!([^ ]*) "#, with: "", options: .regularExpression) if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive), let match = regex.firstMatch(in: text, options: [], range: NSRange(location: 0, length: text.utf16.count)) { @@ -366,9 +365,9 @@ struct ResultsView: View { } } else { if searchText.isEmpty { - results = DatabaseManager.shared.searchFilteredByAppName(appName: selectedFilterApp, limit: limit, offset: offset) + results = DatabaseManager.shared.search(appName: selectedFilterApp, searchText: "", limit: limit, offset: offset) } else { - results = DatabaseManager.shared.searchFilteredByAppNameAndText(appName: selectedFilterApp, searchText: searchText, limit: limit, offset: offset) + results = DatabaseManager.shared.search(appName: selectedFilterApp, searchText: searchText, limit: limit, offset: offset) } } From 58208df9970c8641c9e62cbc0859769439de20d0 Mon Sep 17 00:00:00 2001 From: drull Date: Thu, 4 Jan 2024 21:45:46 -0300 Subject: [PATCH 37/42] Modifying getRecentResults function --- rem/DB.swift | 10 ++++++++-- rem/Search.swift | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rem/DB.swift b/rem/DB.swift index a9e57a6..edea328 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -317,12 +317,18 @@ class DatabaseManager { return results } - func getRecentResults(limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = frames + func getRecentResults(selectedFilterApp: String = "", limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + var query = frames .join(videoChunks, on: frames[chunkId] == videoChunks[id]) .select(frames[id], frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .order(frames[timestamp].desc) .limit(limit, offset: offset) + + if !selectedFilterApp.isEmpty { + query = query + .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) + .filter(uniqueAppNames[activeApplicationName].lowercaseString == selectedFilterApp.lowercased()) + } var results: [(Int64, String?, String?, Date, String, Int64)] = [] do { diff --git a/rem/Search.swift b/rem/Search.swift index 1416688..b4addef 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -365,7 +365,7 @@ struct ResultsView: View { } } else { if searchText.isEmpty { - results = DatabaseManager.shared.search(appName: selectedFilterApp, searchText: "", limit: limit, offset: offset) + results = DatabaseManager.shared.getRecentResults(selectedFilterApp: selectedFilterApp, limit: limit, offset: offset) } else { results = DatabaseManager.shared.search(appName: selectedFilterApp, searchText: searchText, limit: limit, offset: offset) } From 2fb948103eede97e5067ac7e90c7e14edf3394e3 Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Thu, 4 Jan 2024 21:55:50 -0800 Subject: [PATCH 38/42] minor tweaks to unify spacing and improve search visibility --- rem/Search.swift | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/rem/Search.swift b/rem/Search.swift index 4db24c1..9df2084 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -35,7 +35,7 @@ struct SearchBar: View { @Binding var selectedFilterApp: String var body: some View { - HStack { + HStack(spacing: 16) { // Search TextField TextField("Search", text: $text, prompt: Text("Search for something...")) .prefersDefaultFocus(in: nspace) @@ -55,6 +55,10 @@ struct SearchBar: View { .padding(.leading, 12) } ) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(Color(white: 0.3), lineWidth: 1) + ) .onSubmit { Task { onSearch() @@ -70,8 +74,7 @@ struct SearchBar: View { .onAppear { self.focused = true } - .padding(.horizontal, 10) - + FilterPicker( applicationNameFilter: applicationNameFilter, selectedFilterAppIndex: $selectedFilterAppIndex, @@ -79,7 +82,7 @@ struct SearchBar: View { debounceSearch: debounceSearch, onSearch: onSearch ) - } + }.padding(.horizontal, 16) } } @@ -91,21 +94,23 @@ struct FilterPicker: View { var onSearch: () -> Void var body: some View { - Picker("Select App", selection: $selectedFilterAppIndex) { - ForEach(applicationNameFilter.indices, id: \.self) { index in - Text(applicationNameFilter[index]) - .tag(index) + VStack(alignment: .leading) { + Picker("Application", selection: $selectedFilterAppIndex) { + ForEach(applicationNameFilter.indices, id: \.self) { index in + Text(applicationNameFilter[index]) + .tag(index) + } } - } - .pickerStyle(.menu) - .onChange(of: selectedFilterAppIndex) { newIndex in - guard newIndex >= 0 && newIndex < applicationNameFilter.count else { - return + .pickerStyle(.menu) + .onChange(of: selectedFilterAppIndex) { newIndex in + guard newIndex >= 0 && newIndex < applicationNameFilter.count else { + return + } + selectedFilterApp = applicationNameFilter[selectedFilterAppIndex] + onSearch() } - selectedFilterApp = applicationNameFilter[selectedFilterAppIndex] - onSearch() + .frame(width: 200) } - .frame(width: 200) } } From abc4b0515c74a8e412032e2d06d093fae0c7d8bd Mon Sep 17 00:00:00 2001 From: Jason McGhee Date: Thu, 4 Jan 2024 23:17:51 -0800 Subject: [PATCH 39/42] [rem-59]: Adds 'Fast OCR' (lower accuracy, more efficient) and modifies our approach to be more efficent --- rem/SettingsManager.swift | 5 +- rem/remApp.swift | 120 +++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 62 deletions(-) diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index 84a84bd..a69cbd3 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -13,6 +13,7 @@ struct AppSettings: Codable { var saveEverythingCopiedToClipboard: Bool var enableCmdScrollShortcut: Bool var onlyOCRFrontmostWindow: Bool = false + var fastOCR: Bool = false } // The settings manager handles saving and loading the settings @@ -52,8 +53,10 @@ struct SettingsView: View { .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { _ in settingsManager.saveSettings() } Toggle("Allow opening / closing timeline with CMD + Scroll", isOn: $settingsManager.settings.enableCmdScrollShortcut) .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { _ in settingsManager.saveSettings() } - Toggle("Only OCR region of active application window", isOn: $settingsManager.settings.onlyOCRFrontmostWindow) + Toggle("Only OCR region of active application window (more efficient)", isOn: $settingsManager.settings.onlyOCRFrontmostWindow) .onChange(of: settingsManager.settings.onlyOCRFrontmostWindow) { _ in settingsManager.saveSettings() } + Toggle("Use faster, but lower accuracy OCR (more efficient)", isOn: $settingsManager.settings.fastOCR) + .onChange(of: settingsManager.settings.fastOCR) { _ in settingsManager.saveSettings() } } } .padding() diff --git a/rem/remApp.swift b/rem/remApp.swift index 54c5daf..e8df928 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -5,14 +5,14 @@ // Created by Jason McGhee on 12/16/23. // -import SwiftUI import AppKit +import CoreGraphics +import os import ScreenCaptureKit +import ScriptingBridge +import SwiftUI import Vision import VisionKit -import CoreGraphics -import ScriptingBridge -import os final class MainWindow: NSWindow { override var canBecomeKey: Bool { @@ -100,16 +100,16 @@ class AppDelegate: NSObject, NSApplicationDelegate { // Setup Menu setupMenu() - self.observer = self.statusBarItem.button?.observe(\.effectiveAppearance) { _, _ in + observer = statusBarItem.button?.observe(\.effectiveAppearance) { _, _ in self.setupMenu() } // Monitor for scroll events - NSEvent.addGlobalMonitorForEvents(matching: .scrollWheel) { [weak self] (event) in + NSEvent.addGlobalMonitorForEvents(matching: .scrollWheel) { [weak self] event in self?.handleGlobalScrollEvent(event) } - NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] (event) in + NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] event in if (self?.searchViewWindow?.isVisible ?? false) && event.keyCode == 53 { DispatchQueue.main.async { [weak self] in self?.closeSearchView() @@ -123,7 +123,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] (event) in + NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in if event.modifierFlags.contains([.command, .shift]) && event.keyCode == 3 { DispatchQueue.main.async { [weak self] in self?.closeTimelineView() @@ -145,7 +145,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { return event } - NSEvent.addLocalMonitorForEvents(matching: .scrollWheel) { [weak self] (event) in + NSEvent.addLocalMonitorForEvents(matching: .scrollWheel) { [weak self] event in if self?.isTimelineOpen() ?? false { if !event.modifierFlags.contains(.command) && event.scrollingDeltaX != 0 { self?.timelineView?.viewModel.updateIndex(withDelta: event.scrollingDeltaX) @@ -204,11 +204,11 @@ class AppDelegate: NSObject, NSApplicationDelegate { } @objc func toggleTimeline() { - if self.isTimelineOpen() { - self.closeTimelineView() + if isTimelineOpen() { + closeTimelineView() } else { let frame = DatabaseManager.shared.getMaxFrame() - self.showTimelineView(with: frame) + showTimelineView(with: frame) } } @@ -219,14 +219,20 @@ class AppDelegate: NSObject, NSApplicationDelegate { contentRect: NSRect(x: 0, y: 0, width: 400, height: 300), styleMask: [.titled, .closable], backing: .buffered, - defer: false) + defer: false + ) settingsViewWindow?.isReleasedWhenClosed = false settingsViewWindow?.center() settingsViewWindow?.contentView = NSHostingView(rootView: settingsView) settingsViewWindow?.makeKeyAndOrderFront(nil) - } else if (!(settingsViewWindow?.isVisible ?? false)) { + DispatchQueue.main.async { + self.settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + } + } else if !(settingsViewWindow?.isVisible ?? false) { settingsViewWindow?.makeKeyAndOrderFront(nil) - settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + DispatchQueue.main.async { + self.settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + } } } @@ -242,17 +248,17 @@ class AppDelegate: NSObject, NSApplicationDelegate { if response == .alertFirstButtonReturn { alert.window.close() - self.forgetEverything() + forgetEverything() } else { alert.window.close() } } private func handleGlobalScrollEvent(_ event: NSEvent) { - guard settingsManager.settings.enableCmdScrollShortcut else { return} + guard settingsManager.settings.enableCmdScrollShortcut else { return } guard event.modifierFlags.contains(.command) else { return } - if event.scrollingDeltaY < 0 && !self.isTimelineOpen() { // Check if scroll up + if event.scrollingDeltaY < 0 && !isTimelineOpen() { // Check if scroll up DispatchQueue.main.async { [weak self] in self?.showTimelineView(with: DatabaseManager.shared.getMaxFrame()) } @@ -269,7 +275,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - func startScreenCapture() async { + func startScreenCapture() async { do { let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) @@ -299,6 +305,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { // Do we want to record the timeline being searched? guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return } + let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName) if settingsManager.settings.onlyOCRFrontmostWindow { @@ -388,7 +395,6 @@ class AppDelegate: NSObject, NSApplicationDelegate { strongSelf.processChunk(tempBuffer) } } - } private func processChunk(_ chunk: [Data]) { @@ -497,7 +503,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - self.timelineView?.viewModel.setIndexToLatest() + timelineView?.viewModel.setIndexToLatest() setupMenu() } @@ -506,57 +512,48 @@ class AppDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.terminate(self) } - private func performOCR(frameId: Int64, on image: CGImage) { ocrQueue.async { Task { - do { - let configuration = ImageAnalyzer.Configuration([.text]) - let nsImage = NSImage(cgImage: image, size: NSZeroSize) - let analysis = try await self.imageAnalyzer.analyze(nsImage, orientation: CGImagePropertyOrientation.up, configuration: configuration) - let textToAssociate = analysis.transcript - // self.logger.debug("Analysed text: \(textToAssociate)") - var texts = [textToAssociate] + let request = VNRecognizeTextRequest { request, error in + guard let observations = request.results as? [VNRecognizedTextObservation], error == nil else { + print("OCR error: \(error?.localizedDescription ?? "Unknown error")") + return + } + + let topK = 1 + let recognizedStrings = observations.compactMap { observation in + observation.topCandidates(topK).first?.string + // observation.topCandidates(1).first?.boundingBox(for: str.startIndex.. Date: Thu, 4 Jan 2024 23:20:58 -0800 Subject: [PATCH 40/42] turn on efficient settings by default --- rem/SettingsManager.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index a69cbd3..8862b38 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -12,8 +12,8 @@ import SwiftUI struct AppSettings: Codable { var saveEverythingCopiedToClipboard: Bool var enableCmdScrollShortcut: Bool - var onlyOCRFrontmostWindow: Bool = false - var fastOCR: Bool = false + var onlyOCRFrontmostWindow: Bool = true + var fastOCR: Bool = true } // The settings manager handles saving and loading the settings From 5095d9c453eefbe1e0032dd27544532472d44922 Mon Sep 17 00:00:00 2001 From: Pedro Caetano Date: Fri, 5 Jan 2024 09:55:57 -0300 Subject: [PATCH 41/42] Seed the uniqueAppNames table if empty Co-authored-by: Jason McGhee --- rem/DB.swift | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/rem/DB.swift b/rem/DB.swift index edea328..07a237a 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -90,7 +90,24 @@ class DatabaseManager { t.column(id, primaryKey: .autoincrement) t.column(activeApplicationName, unique: true) }) - + // Seed the `uniqueAppNames` table if empty + do { + if try db.scalar(uniqueAppNames.count) == 0 { + let query = frames.select(distinct: activeApplicationName) + var appNames: [String] = [] + for row in try db.prepare(query) { + if let appName = row[activeApplicationName] { + appNames.append(appName) + } + } + let insert = uniqueAppNames.insertMany( + appNames.map { name in [activeApplicationName <- name] } + ) + try db.run(insert) + } + } catch { + print("Error seeding database with app names: \(error)") + } let config = FTS4Config() .column(frameId, [.unindexed]) .column(text) From 75e0f13ee1ab76afd13aa24fd0883d77d4704411 Mon Sep 17 00:00:00 2001 From: drull Date: Fri, 5 Jan 2024 10:07:04 -0300 Subject: [PATCH 42/42] Update when hovering and appearing --- rem/Search.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rem/Search.swift b/rem/Search.swift index fcf3ed5..6430396 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -104,6 +104,9 @@ struct FilterPicker: View { .onHover(perform: { hovering in updateAppFilterData() }) + .onAppear{ + updateAppFilterData() + } .pickerStyle(.menu) .onChange(of: selectedFilterAppIndex) { newIndex in guard newIndex >= 0 && newIndex < applicationFilterArray.count else {