diff --git a/README.md b/README.md index 91333e3..1db6599 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# rem +![image](https://github.com/jasonjmcghee/rem/assets/1522149/bc7368dc-90b5-42a3-abba-9d365b368ddb) + +# rem 🧠 Remember everything. (very alpha - [download anyway](https://github.com/jasonjmcghee/rem/releases)) @@ -19,16 +21,13 @@ Please log any bugs / issues you find! Looking at this code and grimacing? Want to help turn this project into something awesome? Please contribute. I haven't written Swift since 2017. I'm sure you'll write better code than me. -Please look at https://github.com/jasonjmcghee/rem/issues/3 and if you _didn't_ experience anything let us know! -If you can help solve it, please do! - --- I think the idea of recording everything you see has the potential to change how we interact with our computers, and believe it should be open source. Also, from a privacy / security perspective, this is like... pretty scary stuff, and I want the code open -so we know for certain that nothing is leaving your laptop. Even logging to Sentry has the potential to +so we know for certain that nothing is leaving your laptop. Even telemetry has the potential to leak private info. This is 100% local. Please, read the code yourself. diff --git a/Resources/ffmpeg b/Resources/ffmpeg index 39be009..6dec3d3 100755 Binary files a/Resources/ffmpeg and b/Resources/ffmpeg differ diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 3822a7d..4ad89c0 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -16,7 +16,6 @@ 961C95F82B2E19B40093F228 /* remUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95F72B2E19B40093F228 /* remUITestsLaunchTests.swift */; }; 961C96132B2EB7DB0093F228 /* TimelineView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96122B2EB7DB0093F228 /* TimelineView.swift */; }; 961C96152B2EBEE50093F228 /* DB.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96142B2EBEE50093F228 /* DB.swift */; }; - 9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9670E1342B41683B005728F5 /* ImageHelper.swift */; }; 969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */; }; 969F3EFF2B3A8C4D0085787B /* HotKey in Frameworks */ = {isa = PBXBuildFile; productRef = 969F3EFE2B3A8C4D0085787B /* HotKey */; }; 969F3F082B3B7C7C0085787B /* RemFileManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969F3F072B3B7C7C0085787B /* RemFileManager.swift */; }; @@ -31,6 +30,9 @@ 96DBA3C82B40164E0000CFBE /* (null) in Sources */ = {isa = PBXBuildFile; }; 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96DBA3E62B403ED90000CFBE /* Timings.swift */; }; 96F062182B35111D00695621 /* Search.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96F062172B35111D00695621 /* Search.swift */; }; + BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; + BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; + BF5FEBFD2B44B26800744FC2 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -205,7 +207,6 @@ 961C960D2B2E73840093F228 /* ffmpeg */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.executable"; path = ffmpeg; sourceTree = ""; }; 961C96122B2EB7DB0093F228 /* TimelineView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimelineView.swift; sourceTree = ""; }; 961C96142B2EBEE50093F228 /* DB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DB.swift; sourceTree = ""; }; - 9670E1342B41683B005728F5 /* ImageHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = ""; }; 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsManager.swift; sourceTree = ""; }; 969F3F072B3B7C7C0085787B /* RemFileManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemFileManager.swift; sourceTree = ""; }; 969F3F092B3B7F760085787B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; @@ -220,6 +221,7 @@ 96E66BB42B2F5745006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96E66BCC2B2F574D006E1E97 /* SQLite.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = SQLite.xcodeproj; path = ./SQLite.swift/SQLite.xcodeproj; sourceTree = ""; }; 96F062172B35111D00695621 /* Search.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Search.swift; sourceTree = ""; }; + BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -286,6 +288,7 @@ children = ( 969F3F092B3B7F760085787B /* Info.plist */, 961C96122B2EB7DB0093F228 /* TimelineView.swift */, + BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */, 961C95D92B2E19B30093F228 /* remApp.swift */, 961C95DB2B2E19B30093F228 /* ContentView.swift */, 961C95DD2B2E19B40093F228 /* Assets.xcassets */, @@ -300,7 +303,6 @@ 969F3F0C2B3CCEC30085787B /* Ask.swift */, 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */, 96DBA3E62B403ED90000CFBE /* Timings.swift */, - 9670E1342B41683B005728F5 /* ImageHelper.swift */, ); path = rem; sourceTree = ""; @@ -667,7 +669,7 @@ 969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */, 969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */, 96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */, - 9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */, + BF5FEBFB2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -676,6 +678,7 @@ buildActionMask = 2147483647; files = ( 961C95EC2B2E19B40093F228 /* remTests.swift in Sources */, + BF5FEBFC2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -685,6 +688,7 @@ files = ( 961C95F62B2E19B40093F228 /* remUITests.swift in Sources */, 961C95F82B2E19B40093F228 /* remUITestsLaunchTests.swift in Sources */, + BF5FEBFD2B44B26800744FC2 /* ImageHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -771,7 +775,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -830,7 +834,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = NO; @@ -855,6 +859,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -866,7 +871,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -891,6 +895,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"rem/Preview Content\""; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -902,7 +907,6 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -992,14 +996,17 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; ONLY_ACTIVE_ARCH = YES; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -1014,13 +1021,16 @@ ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; CODE_SIGN_ENTITLEMENTS = ffmpegX/ffmpegX.entitlements; "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_INJECT_BASE_ENTITLEMENTS = NO; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = XK2A33Z623; + ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 14.2; + NEW_SETTING = ""; + OTHER_CODE_SIGN_FLAGS = "$(inherited) -i $(PRODUCT_BUNDLE_IDENTIFIER)"; PRODUCT_BUNDLE_IDENTIFIER = today.jason.rem.ffmpegX; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; diff --git a/rem/DB.swift b/rem/DB.swift index f815da3..07a237a 100644 --- a/rem/DB.swift +++ b/rem/DB.swift @@ -26,6 +26,8 @@ class DatabaseManager { private let videoChunks = Table("video_chunks") private let frames = Table("frames") + private let uniqueAppNames = Table("unique_application_names") + let allText = VirtualTable("allText") private let id = Expression("id") @@ -59,6 +61,7 @@ class DatabaseManager { try db.run(videoChunks.drop(ifExists: true)) try db.run(frames.drop(ifExists: true)) try db.run(allText.drop(ifExists: true)) + try db.run(uniqueAppNames.drop(ifExists: true)) } catch { print("Failed to delete tables") } @@ -83,6 +86,28 @@ class DatabaseManager { t.column(activeApplicationName) }) + try! db.run(uniqueAppNames.create(ifNotExists: true) { t in + t.column(id, primaryKey: .autoincrement) + t.column(activeApplicationName, unique: true) + }) + // Seed the `uniqueAppNames` table if empty + do { + if try db.scalar(uniqueAppNames.count) == 0 { + let query = frames.select(distinct: activeApplicationName) + var appNames: [String] = [] + for row in try db.prepare(query) { + if let appName = row[activeApplicationName] { + appNames.append(appName) + } + } + let insert = uniqueAppNames.insertMany( + appNames.map { name in [activeApplicationName <- name] } + ) + try db.run(insert) + } + } catch { + print("Error seeding database with app names: \(error)") + } let config = FTS4Config() .column(frameId, [.unindexed]) .column(text) @@ -137,14 +162,41 @@ class DatabaseManager { } func insertFrame(activeApplicationName: String?) -> Int64 { - // logger.debug("inserting frame: \(self.lastFrameId + 1) at offset: \(self.currentFrameOffset)") let insert = frames.insert(chunkId <- currentChunkId, timestamp <- Date(), offsetIndex <- currentFrameOffset, self.activeApplicationName <- activeApplicationName) let id = try! db.run(insert) currentFrameOffset += 1 lastFrameId = id + + if let appName = activeApplicationName { + //will check if the app name is already in the database. + insertUniqueApplicationNamesIfNeeded(appName) + } + return id } + private func insertUniqueApplicationNamesIfNeeded(_ appName: String) { + let query = uniqueAppNames.filter(activeApplicationName == appName) + + do { + let count = try db.scalar(query.count) + if count == 0 { + insertUniqueApplicationNames(appName) + } + } catch { + print("Error checking existence of app name: \(error)") + } + } + + func insertUniqueApplicationNames(_ appName: String) { + let insert = uniqueAppNames.insert(activeApplicationName <- appName) + do { + try db.run(insert) + } catch { + print("Error inserting unique application name: \(error)") + } + } + func insertTextForFrame(frameId: Int64, text: String) { let insert = allText.insert(self.frameId <- frameId, self.text <- text) try! db.run(insert) @@ -243,13 +295,27 @@ class DatabaseManager { return 0 } - func search(searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = allText + func search(appName: String = "", searchText: String, limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + var partialQuery = allText .join(frames, on: frames[id] == allText[frameId]) .join(videoChunks, on: frames[chunkId] == videoChunks[id]) - .filter(text.match("*\(searchText)*")) - .select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) - .limit(limit, offset: offset) + + if !appName.isEmpty { + partialQuery = partialQuery.join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) + } + + var query = partialQuery + .filter(text.match("*\(searchText)*")) + + if !appName.isEmpty && searchText.isEmpty { + query = partialQuery + .filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) + } else if !appName.isEmpty && !searchText.isEmpty { + query = query.filter(uniqueAppNames[activeApplicationName].lowercaseString == appName.lowercased()) + } + + query = query.select(allText[frameId], text, frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) + .limit(limit, offset: offset) var results: [(Int64, String, String?, Date, String, Int64)] = [] do { @@ -268,12 +334,18 @@ class DatabaseManager { return results } - func getRecentResults(limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { - let query = frames + func getRecentResults(selectedFilterApp: String = "", limit: Int = 9, offset: Int = 0) -> [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] { + var query = frames .join(videoChunks, on: frames[chunkId] == videoChunks[id]) .select(frames[id], frames[activeApplicationName], frames[timestamp], videoChunks[filePath], frames[offsetIndex]) .order(frames[timestamp].desc) .limit(limit, offset: offset) + + if !selectedFilterApp.isEmpty { + query = query + .join(uniqueAppNames, on: uniqueAppNames[activeApplicationName] == frames[activeApplicationName]) + .filter(uniqueAppNames[activeApplicationName].lowercaseString == selectedFilterApp.lowercased()) + } var results: [(Int64, String?, String?, Date, String, Int64)] = [] do { @@ -291,6 +363,23 @@ class DatabaseManager { return results } + func getAllApplicationNames() -> [String] { + var applicationNames: [String] = [] + + do { + let distinctAppsQuery = uniqueAppNames.select(activeApplicationName) + for row in try db.prepare(distinctAppsQuery) { + if let appName = row[activeApplicationName] { + applicationNames.append(appName) + } + } + } catch { + print("Error fetching application names: \(error)") + } + + return applicationNames + } + func getImage(index: Int64, maxSize: CGSize? = nil) -> CGImage? { guard let frameData = DatabaseManager.shared.getFrame(forIndex: index) else { return nil } diff --git a/rem/ImageHelper.swift b/rem/ImageHelper.swift index c6de7ec..574cbcf 100644 --- a/rem/ImageHelper.swift +++ b/rem/ImageHelper.swift @@ -46,4 +46,17 @@ class ImageHelper { logger.error("Error writing PNG file: \(error)") } } + + static func saveCGImage(image: CGImage, path: String) { + saveNSImage(image: NSImage(cgImage: image, size: NSZeroSize), path: path) + } + + static func cropImage(image: CGImage, frame: CGRect, scale: CGFloat) -> CGImage? { + let cropZone = CGRect( + x: frame.origin.x * scale, + y: frame.origin.y * scale, + width: frame.size.width * scale, + height: frame.size.height * scale) + return image.cropping(to: cropZone) + } } diff --git a/rem/Search.swift b/rem/Search.swift index 9d706a2..6430396 100644 --- a/rem/Search.swift +++ b/rem/Search.swift @@ -29,11 +29,14 @@ struct SearchBar: View { var onSearch: () -> Void @Namespace var nspace @FocusState var focused: Bool? - var debounceSearch = Debouncer(delay: 0.3) - + @Binding var selectedFilterAppIndex: Int + @Binding var selectedFilterApp: String + @State private var applicationFilterArray: [String] = [] + var body: some View { - HStack { + HStack(spacing: 16) { + // Search TextField TextField("Search", text: $text, prompt: Text("Search for something...")) .prefersDefaultFocus(in: nspace) .textFieldStyle(.plain) @@ -52,26 +55,78 @@ struct SearchBar: View { .padding(.leading, 12) } ) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(Color(white: 0.3), lineWidth: 1) + ) .onSubmit { Task { onSearch() } - } // Trigger search when user submits - .onChange(of: text) { + } + .onChange(of: text) { _ in debounceSearch.debounce { Task { onSearch() } } - } // Trigger search when text changes + } .onAppear { self.focused = true } - .padding(.horizontal, 10) + + FilterPicker( + applicationFilterArray: applicationFilterArray, + selectedFilterAppIndex: $selectedFilterAppIndex, + selectedFilterApp: $selectedFilterApp, + debounceSearch: debounceSearch, + onSearch: onSearch + ) + }.padding(.horizontal, 16) + } +} + +struct FilterPicker: View { + @State var applicationFilterArray: [String] + @Binding var selectedFilterAppIndex: Int + @Binding var selectedFilterApp: String + var debounceSearch: Debouncer + var onSearch: () -> Void + + var body: some View { + VStack(alignment: .leading) { + Picker("Application", selection: $selectedFilterAppIndex) { + ForEach(applicationFilterArray.indices, id: \.self) { index in + Text(applicationFilterArray[index]) + .tag(index) + } + } + .onHover(perform: { hovering in + updateAppFilterData() + }) + .onAppear{ + updateAppFilterData() + } + .pickerStyle(.menu) + .onChange(of: selectedFilterAppIndex) { newIndex in + guard newIndex >= 0 && newIndex < applicationFilterArray.count else { + return + } + selectedFilterApp = applicationFilterArray[selectedFilterAppIndex] + onSearch() + } + .frame(width: 200) } } + private func updateAppFilterData() { + var appFilters = ["All apps"] + let allAppNames = DatabaseManager.shared.getAllApplicationNames() + appFilters.append(contentsOf: allAppNames) + applicationFilterArray = appFilters + } } + struct VisualEffectView: NSViewRepresentable { var material: NSVisualEffectView.Material var blendingMode: NSVisualEffectView.BlendingMode @@ -255,12 +310,19 @@ struct ResultsView: View { @State private var searchResults: [SearchResult] = [] @State var limit: Int = 27 @State var offset: Int = 0 - - var onThumbnailClick: (Int64) -> Void // Closure to handle thumbnail click + @State var selectedFilterApp: String = "" + @State var selectedFilterAppIndex: Int = 0 - var body: some View { - VStack { - SearchBar(text: $searchText, onSearch: performSearch) + var onThumbnailClick: (Int64) -> Void + + var body: some View { + VStack { + SearchBar( + text: $searchText, + onSearch: performSearch, + selectedFilterAppIndex: $selectedFilterAppIndex, + selectedFilterApp: $selectedFilterApp + ) ScrollView { LazyVGrid(columns: Array(repeating: GridItem(.flexible()), count: 3), spacing: 20) { @@ -301,14 +363,25 @@ struct ResultsView: View { } private func getSearchResults() -> [SearchResult] { - if searchText.isEmpty { - let recentResults = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) - return mapResultsToSearchResult(recentResults) + var results: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)] = [] + + if selectedFilterAppIndex == 0 { + if searchText.isEmpty { + results = DatabaseManager.shared.getRecentResults(limit: limit, offset: offset) + } else { + results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) + } } else { - let results = DatabaseManager.shared.search(searchText: searchText, limit: limit, offset: offset) - return mapResultsToSearchResult(results) + if searchText.isEmpty { + results = DatabaseManager.shared.getRecentResults(selectedFilterApp: selectedFilterApp, limit: limit, offset: offset) + } else { + results = DatabaseManager.shared.search(appName: selectedFilterApp, searchText: searchText, limit: limit, offset: offset) + } } + + return mapResultsToSearchResult(results) } + private func mapResultsToSearchResult(_ data: [(frameId: Int64, fullText: String?, applicationName: String?, timestamp: Date, filePath: String, offsetIndex: Int64)]) -> [SearchResult] { let searchResults = data.map { item in @@ -402,8 +475,3 @@ extension View { self.modifier(ScrollViewOffsetModifier(onBottomReached: action)) } } - -#Preview("Hello", traits: .defaultLayout) { - SearchView(onThumbnailClick: { _ in }) -} - diff --git a/rem/SettingsManager.swift b/rem/SettingsManager.swift index 60abf7b..8862b38 100644 --- a/rem/SettingsManager.swift +++ b/rem/SettingsManager.swift @@ -12,6 +12,8 @@ import SwiftUI struct AppSettings: Codable { var saveEverythingCopiedToClipboard: Bool var enableCmdScrollShortcut: Bool + var onlyOCRFrontmostWindow: Bool = true + var fastOCR: Bool = true } // The settings manager handles saving and loading the settings @@ -48,9 +50,13 @@ struct SettingsView: View { .padding(.bottom) Form { Toggle("Remember everything copied to clipboard", isOn: $settingsManager.settings.saveEverythingCopiedToClipboard) - .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.saveEverythingCopiedToClipboard) { _ in settingsManager.saveSettings() } Toggle("Allow opening / closing timeline with CMD + Scroll", isOn: $settingsManager.settings.enableCmdScrollShortcut) - .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { settingsManager.saveSettings() } + .onChange(of: settingsManager.settings.enableCmdScrollShortcut) { _ in settingsManager.saveSettings() } + Toggle("Only OCR region of active application window (more efficient)", isOn: $settingsManager.settings.onlyOCRFrontmostWindow) + .onChange(of: settingsManager.settings.onlyOCRFrontmostWindow) { _ in settingsManager.saveSettings() } + Toggle("Use faster, but lower accuracy OCR (more efficient)", isOn: $settingsManager.settings.fastOCR) + .onChange(of: settingsManager.settings.fastOCR) { _ in settingsManager.saveSettings() } } } .padding() diff --git a/rem/TimelineView.swift b/rem/TimelineView.swift index 2de62e1..28f38a4 100644 --- a/rem/TimelineView.swift +++ b/rem/TimelineView.swift @@ -46,7 +46,7 @@ struct TimelineView: View { ) .frame(width: frame.width, height: frame.height) .ignoresSafeArea(.all) - .onChange(of: viewModel.currentFrameIndex) { + .onChange(of: viewModel.currentFrameIndex) { _ in analyzeCurrentImage() } .onAppear { @@ -297,7 +297,7 @@ class TimelineViewModel: ObservableObject { func updateIndex(withDelta delta: Double) { // Logic to update the index based on the delta // This method will be called from AppDelegate - var nextValue = currentFrameContinuous - delta * speedFactor + let nextValue = currentFrameContinuous - delta * speedFactor let maxValue = Double(DatabaseManager.shared.getMaxFrame()) let clampedValue = min(max(1, nextValue), maxValue) self.currentFrameContinuous = clampedValue diff --git a/rem/remApp.swift b/rem/remApp.swift index 5921fdd..e8df928 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -5,13 +5,14 @@ // Created by Jason McGhee on 12/16/23. // -import SwiftUI import AppKit +import CoreGraphics +import os import ScreenCaptureKit +import ScriptingBridge +import SwiftUI import Vision import VisionKit -import CoreGraphics -import os final class MainWindow: NSWindow { override var canBecomeKey: Bool { @@ -99,16 +100,16 @@ class AppDelegate: NSObject, NSApplicationDelegate { // Setup Menu setupMenu() - self.observer = self.statusBarItem.button?.observe(\.effectiveAppearance) { _, _ in + observer = statusBarItem.button?.observe(\.effectiveAppearance) { _, _ in self.setupMenu() } // Monitor for scroll events - NSEvent.addGlobalMonitorForEvents(matching: .scrollWheel) { [weak self] (event) in + NSEvent.addGlobalMonitorForEvents(matching: .scrollWheel) { [weak self] event in self?.handleGlobalScrollEvent(event) } - NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] (event) in + NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] event in if (self?.searchViewWindow?.isVisible ?? false) && event.keyCode == 53 { DispatchQueue.main.async { [weak self] in self?.closeSearchView() @@ -122,7 +123,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] (event) in + NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in if event.modifierFlags.contains([.command, .shift]) && event.keyCode == 3 { DispatchQueue.main.async { [weak self] in self?.closeTimelineView() @@ -144,7 +145,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { return event } - NSEvent.addLocalMonitorForEvents(matching: .scrollWheel) { [weak self] (event) in + NSEvent.addLocalMonitorForEvents(matching: .scrollWheel) { [weak self] event in if self?.isTimelineOpen() ?? false { if !event.modifierFlags.contains(.command) && event.scrollingDeltaX != 0 { self?.timelineView?.viewModel.updateIndex(withDelta: event.scrollingDeltaX) @@ -203,11 +204,11 @@ class AppDelegate: NSObject, NSApplicationDelegate { } @objc func toggleTimeline() { - if self.isTimelineOpen() { - self.closeTimelineView() + if isTimelineOpen() { + closeTimelineView() } else { let frame = DatabaseManager.shared.getMaxFrame() - self.showTimelineView(with: frame) + showTimelineView(with: frame) } } @@ -218,14 +219,20 @@ class AppDelegate: NSObject, NSApplicationDelegate { contentRect: NSRect(x: 0, y: 0, width: 400, height: 300), styleMask: [.titled, .closable], backing: .buffered, - defer: false) + defer: false + ) settingsViewWindow?.isReleasedWhenClosed = false settingsViewWindow?.center() settingsViewWindow?.contentView = NSHostingView(rootView: settingsView) settingsViewWindow?.makeKeyAndOrderFront(nil) - } else if (!(settingsViewWindow?.isVisible ?? false)) { + DispatchQueue.main.async { + self.settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + } + } else if !(settingsViewWindow?.isVisible ?? false) { settingsViewWindow?.makeKeyAndOrderFront(nil) - settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + DispatchQueue.main.async { + self.settingsViewWindow?.orderFrontRegardless() // Ensure it comes to the front + } } } @@ -241,17 +248,17 @@ class AppDelegate: NSObject, NSApplicationDelegate { if response == .alertFirstButtonReturn { alert.window.close() - self.forgetEverything() + forgetEverything() } else { alert.window.close() } } private func handleGlobalScrollEvent(_ event: NSEvent) { - guard settingsManager.settings.enableCmdScrollShortcut else { return} + guard settingsManager.settings.enableCmdScrollShortcut else { return } guard event.modifierFlags.contains(.command) else { return } - if event.scrollingDeltaY < 0 && !self.isTimelineOpen() { // Check if scroll up + if event.scrollingDeltaY < 0 && !isTimelineOpen() { // Check if scroll up DispatchQueue.main.async { [weak self] in self?.showTimelineView(with: DatabaseManager.shared.getMaxFrame()) } @@ -268,7 +275,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - func startScreenCapture() async { + func startScreenCapture() async { do { let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) @@ -298,8 +305,27 @@ class AppDelegate: NSObject, NSApplicationDelegate { // Do we want to record the timeline being searched? guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return } + let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName) + if settingsManager.settings.onlyOCRFrontmostWindow { + // User wants to perform OCR on only active window. + + // We need to determine the scale factor for cropping. CGImage is + // measured in pixels, display sizes are measured in points. + let scale = max(CGFloat(image.width) / CGFloat(display.width), CGFloat(image.height) / CGFloat(display.height)) + + if + let window = shareableContent.windows.first(where: { $0.isOnScreen && $0.owningApplication?.processID == NSWorkspace.shared.frontmostApplication?.processIdentifier }), + let cropped = ImageHelper.cropImage(image: image, frame: window.frame, scale: scale) + { + self.performOCR(frameId: frameId, on: cropped) + } + } else { + // default: User wants to perform OCR on full display. + self.performOCR(frameId: frameId, on: image) + } + await processScreenshot(frameId: frameId, image: image, frame: display.frame) screenshotQueue.asyncAfter(deadline: .now() + 2) { [weak self] in @@ -349,7 +375,6 @@ class AppDelegate: NSObject, NSApplicationDelegate { // } private func processScreenshot(frameId: Int64, image: CGImage, frame: CGRect) async { - self.performOCR(frameId: frameId, on: image, frame: frame) let bitmapRep = NSBitmapImageRep(cgImage: image) guard let data = bitmapRep.representation(using: .png, properties: [:]) else { return } @@ -370,7 +395,6 @@ class AppDelegate: NSObject, NSApplicationDelegate { strongSelf.processChunk(tempBuffer) } } - } private func processChunk(_ chunk: [Data]) { @@ -378,7 +402,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { if let savedir = RemFileManager.shared.getSaveDir() { let outputPath = savedir.appendingPathComponent("output-\(Date().timeIntervalSince1970).mp4").path - DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) + let _ = DatabaseManager.shared.startNewVideoChunk(filePath: outputPath) // Setup the FFmpeg process for the chunk let ffmpegProcess = Process() @@ -414,7 +438,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } // Write the chunk data to the FFmpeg process - for (index, data) in chunk.enumerated() { + for (_, data) in chunk.enumerated() { do { try ffmpegInputPipe.fileHandleForWriting.write(contentsOf: data) } catch { @@ -479,7 +503,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { } } - self.timelineView?.viewModel.setIndexToLatest() + timelineView?.viewModel.setIndexToLatest() setupMenu() } @@ -488,65 +512,48 @@ class AppDelegate: NSObject, NSApplicationDelegate { NSApplication.shared.terminate(self) } - - private func performOCR(frameId: Int64, on image: CGImage, frame: CGRect) { + private func performOCR(frameId: Int64, on image: CGImage) { ocrQueue.async { - // Select only a region... / active window? -// let invWidth = 1 / CGFloat(image.width) -// let invHeight = 1 / CGFloat(image.height) -// let regionOfInterest = CGRect( -// x: min(max(0, frame.minX * invWidth), 1), -// y: min(max(0, frame.minY * invHeight), 1), -// width: min(max(0, frame.width * invWidth), 1), -// height: min(max(0, frame.height * invHeight), 1) -// ) Task { - do { - let configuration = ImageAnalyzer.Configuration([.text]) - let nsImage = NSImage(cgImage: image, size: NSSize(width: image.width, height: image.height)) - let analysis = try await self.imageAnalyzer.analyze(nsImage, orientation: CGImagePropertyOrientation.up, configuration: configuration) - let textToAssociate = analysis.transcript - var texts = [textToAssociate] + let request = VNRecognizeTextRequest { request, error in + guard let observations = request.results as? [VNRecognizedTextObservation], error == nil else { + print("OCR error: \(error?.localizedDescription ?? "Unknown error")") + return + } + + let topK = 1 + let recognizedStrings = observations.compactMap { observation in + observation.topCandidates(topK).first?.string + // observation.topCandidates(1).first?.boundingBox(for: str.startIndex..