From 57f6c6727a89b4bfb174dd2ea55365b08c8ee7b8 Mon Sep 17 00:00:00 2001 From: Jayshankar Shahu Date: Fri, 24 Oct 2025 22:04:07 +0530 Subject: [PATCH 1/3] feat: Show notification when human input is required but browser in not in focus --- manifest.json | 3 +- src/background/index.ts | 99 +++++++++++++++------- src/sidepanel/App.tsx | 29 ++++++- src/sidepanel/hooks/usePushNotification.ts | 36 ++++++++ 4 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 src/sidepanel/hooks/usePushNotification.ts diff --git a/manifest.json b/manifest.json index 257f80ac..00ff0f2c 100644 --- a/manifest.json +++ b/manifest.json @@ -18,7 +18,8 @@ "browserOS", "webNavigation", "downloads", - "audioCapture" + "audioCapture", + "notifications" ], "update_url": "https://cdn.browseros.com/extensions/update-manifest.xml", "host_permissions": [ diff --git a/src/background/index.ts b/src/background/index.ts index 85104eea..1fe8a729 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -55,12 +55,12 @@ function registerHandlers(): void { MessageType.EXECUTE_QUERY, (msg, port) => executionHandler.handleExecuteQuery(msg, port) ) - + messageRouter.registerHandler( MessageType.CANCEL_TASK, (msg, port) => executionHandler.handleCancelTask(msg, port) ) - + messageRouter.registerHandler( MessageType.RESET_CONVERSATION, (msg, port) => executionHandler.handleResetConversation(msg, port) @@ -87,49 +87,49 @@ function registerHandlers(): void { MessageType.GET_LLM_PROVIDERS, (msg, port) => providersHandler.handleGetProviders(msg, port) ) - + messageRouter.registerHandler( MessageType.SAVE_LLM_PROVIDERS, (msg, port) => providersHandler.handleSaveProviders(msg, port) ) - + // MCP handlers messageRouter.registerHandler( MessageType.GET_MCP_SERVERS, (msg, port) => mcpHandler.handleGetMCPServers(msg, port) ) - + messageRouter.registerHandler( MessageType.CONNECT_MCP_SERVER, (msg, port) => mcpHandler.handleConnectMCPServer(msg, port) ) - + messageRouter.registerHandler( MessageType.DISCONNECT_MCP_SERVER, (msg, port) => mcpHandler.handleDisconnectMCPServer(msg, port) ) - + messageRouter.registerHandler( MessageType.CALL_MCP_TOOL, (msg, port) => mcpHandler.handleCallMCPTool(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_INSTALL_SERVER, (msg, port) => mcpHandler.handleInstallServer(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_DELETE_SERVER, (msg, port) => mcpHandler.handleDeleteServer(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_GET_INSTALLED_SERVERS, (msg, port) => mcpHandler.handleGetInstalledServers(msg, port) ) - - + + // Plan generation handlers (for AI plan generation in newtab) messageRouter.registerHandler( MessageType.GENERATE_PLAN, @@ -215,7 +215,7 @@ function registerHandlers(): void { Logging.log(logMsg.source || 'Unknown', logMsg.message, logMsg.level || 'info') } ) - + // Metrics handler messageRouter.registerHandler( MessageType.LOG_METRIC, @@ -224,7 +224,7 @@ function registerHandlers(): void { Logging.logMetric(event, properties) } ) - + // Heartbeat handler - acknowledge heartbeats to keep connection alive messageRouter.registerHandler( MessageType.HEARTBEAT, @@ -237,7 +237,7 @@ function registerHandlers(): void { }) } ) - + // Panel close handler messageRouter.registerHandler( MessageType.CLOSE_PANEL, @@ -290,33 +290,33 @@ function registerHandlers(): void { */ function handlePortConnection(port: chrome.runtime.Port): void { const portId = portManager.registerPort(port) - + // Handle sidepanel connections if (port.name === 'sidepanel') { isPanelOpen = true Logging.log('Background', `Side panel connected`) Logging.logMetric('side_panel_opened', { source: 'port_connection' }) } - + // Register with logging system Logging.registerPort(port.name, port) - + // Set up message listener port.onMessage.addListener((message: PortMessage) => { messageRouter.routeMessage(message, port) }) - + // Set up disconnect listener port.onDisconnect.addListener(() => { portManager.unregisterPort(port) - + // Update panel state if this was the sidepanel if (port.name === 'sidepanel') { isPanelOpen = false Logging.log('Background', `Side panel disconnected`) Logging.logMetric('side_panel_closed', { source: 'port_disconnection' }) } - + // Unregister from logging Logging.unregisterPort(port.name) }) @@ -327,9 +327,9 @@ function handlePortConnection(port: chrome.runtime.Port): void { */ async function toggleSidePanel(tabId: number): Promise { if (isPanelToggling) return - + isPanelToggling = true - + try { if (isPanelOpen) { // Signal sidepanel to close itself @@ -345,7 +345,7 @@ async function toggleSidePanel(tabId: number): Promise { } } catch (error) { Logging.log('Background', `Error toggling side panel: ${error}`, 'error') - + // Try fallback with windowId if (!isPanelOpen) { try { @@ -366,6 +366,45 @@ async function toggleSidePanel(tabId: number): Promise { } } +/** + * Register notification interaction handlers ( notification click , button click etc. ) + */ +function registerNotificationListeners() { + + // event listener to listen for detecting when browser is opened/resumed + chrome.windows.onFocusChanged.addListener((windowId) => { + + // windowId is not none that means window is focues + if( windowId !== chrome.windows.WINDOW_ID_NONE ) { + + //clear all notifications because browser is in focus now + chrome.notifications.getAll((notifications) => { + + Object.keys(notifications).forEach(id => { + chrome.notifications.clear(id); + }) + + }) + + } + + }); + + //handle click of notification + chrome.notifications.onClicked.addListener((noticationId) => { + + // clear notification + chrome.notifications.clear(noticationId); + + chrome.windows.getCurrent((window) => { + //open browser window + chrome.windows.update(window.id!, { focused: true }); + }); + + }); + +} + /** * Handle extension installation */ @@ -420,9 +459,11 @@ function initialize(): void { // Register all handlers registerHandlers() + registerNotificationListeners(); + // Set up port connection listener chrome.runtime.onConnect.addListener(handlePortConnection) - + // Set up extension icon click handler chrome.action.onClicked.addListener(async (tab) => { Logging.log('Background', 'Extension icon clicked') @@ -430,7 +471,7 @@ function initialize(): void { await toggleSidePanel(tab.id) } }) - + // Set up keyboard shortcut handler chrome.commands.onCommand.addListener(async (command) => { if (command === 'toggle-panel') { @@ -441,13 +482,13 @@ function initialize(): void { } } }) - + // Clean up on tab removal chrome.tabs.onRemoved.addListener(async (tabId) => { // With singleton execution, just log the tab removal Logging.log('Background', `Tab ${tabId} removed`) }) - + // Handle messages from newtab only chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { if (message.type === 'NEWTAB_EXECUTE_QUERY') { @@ -455,7 +496,7 @@ function initialize(): void { return true // Keep message channel open for async response } }) - + Logging.log('Background', 'Nxtscape extension initialized successfully') } diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index c6fae829..0814bb5e 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -12,6 +12,7 @@ import { Header } from './components/Header' import { ModeToggle } from './components/ModeToggle' import { useChatStore } from './stores/chatStore' import './styles.css' +import { usePushNotification } from './hooks/usePushNotification' /** * Root component for sidepanel v2 @@ -34,7 +35,10 @@ export function App() { const { teachModeState, abortTeachExecution } = useTeachModeStore(state => ({ teachModeState: state.mode, abortTeachExecution: state.abortExecution - })) + })); + + // Get Push notification function for calling when human-input is needed + const { sendNotification } = usePushNotification(); // Check if any execution is running (chat or teach mode) const isExecuting = isProcessing || teachModeState === 'executing' @@ -96,6 +100,25 @@ export function App() { useEffect(() => { announcer.announce(connected ? 'Extension connected' : 'Extension disconnected') }, [connected, announcer]) + + // show push notification if human input is needed and browser is hidden + useEffect(() => { + + // document.hidden may incorrect values in some linux distros , but it works in most + if (humanInputRequest && document.hidden) { + + sendNotification({ + title: "Human input needed", + message: humanInputRequest.prompt, + type: 'basic', + iconUrl: 'assets/icon48.png', + isClickable: true, + requireInteraction: true, + }); + + } + + }, [humanInputRequest]); return ( - + {humanInputRequest && ( + /> )} diff --git a/src/sidepanel/hooks/usePushNotification.ts b/src/sidepanel/hooks/usePushNotification.ts new file mode 100644 index 00000000..eae4c81b --- /dev/null +++ b/src/sidepanel/hooks/usePushNotification.ts @@ -0,0 +1,36 @@ +import { useCallback } from 'react' + +/** +* Custom hook for sending web push notifications + */ +export function usePushNotification() { + + /** + * Can be called for sending notifications + */ + const sendNotification = useCallback(async (options: chrome.notifications.NotificationOptions ): Promise => { + + return new Promise(async (resolve, reject) => { + + try { + + const noticationId = `notification-${Date.now()}`; + + chrome.notifications.create( noticationId , options, (notificationId) => { + resolve(notificationId); + }); + + } catch (err: any) { + + reject(err); + + } + + }) + + }, []); + + return { + sendNotification, + } +} From ca93ef26833d96e568d01144db8d6a1663089e9d Mon Sep 17 00:00:00 2001 From: Jayshankar Date: Wed, 29 Oct 2025 18:41:51 +0530 Subject: [PATCH 2/3] fix: Improve notification handling in background script --- src/background/index.ts | 141 ++++++++++++++------- src/sidepanel/App.tsx | 2 +- src/sidepanel/hooks/usePushNotification.ts | 30 +++-- 3 files changed, 112 insertions(+), 61 deletions(-) diff --git a/src/background/index.ts b/src/background/index.ts index 1fe8a729..9b05e002 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -55,12 +55,12 @@ function registerHandlers(): void { MessageType.EXECUTE_QUERY, (msg, port) => executionHandler.handleExecuteQuery(msg, port) ) - + messageRouter.registerHandler( MessageType.CANCEL_TASK, (msg, port) => executionHandler.handleCancelTask(msg, port) ) - + messageRouter.registerHandler( MessageType.RESET_CONVERSATION, (msg, port) => executionHandler.handleResetConversation(msg, port) @@ -87,49 +87,49 @@ function registerHandlers(): void { MessageType.GET_LLM_PROVIDERS, (msg, port) => providersHandler.handleGetProviders(msg, port) ) - + messageRouter.registerHandler( MessageType.SAVE_LLM_PROVIDERS, (msg, port) => providersHandler.handleSaveProviders(msg, port) ) - + // MCP handlers messageRouter.registerHandler( MessageType.GET_MCP_SERVERS, (msg, port) => mcpHandler.handleGetMCPServers(msg, port) ) - + messageRouter.registerHandler( MessageType.CONNECT_MCP_SERVER, (msg, port) => mcpHandler.handleConnectMCPServer(msg, port) ) - + messageRouter.registerHandler( MessageType.DISCONNECT_MCP_SERVER, (msg, port) => mcpHandler.handleDisconnectMCPServer(msg, port) ) - + messageRouter.registerHandler( MessageType.CALL_MCP_TOOL, (msg, port) => mcpHandler.handleCallMCPTool(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_INSTALL_SERVER, (msg, port) => mcpHandler.handleInstallServer(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_DELETE_SERVER, (msg, port) => mcpHandler.handleDeleteServer(msg, port) ) - + messageRouter.registerHandler( MessageType.MCP_GET_INSTALLED_SERVERS, (msg, port) => mcpHandler.handleGetInstalledServers(msg, port) ) - - + + // Plan generation handlers (for AI plan generation in newtab) messageRouter.registerHandler( MessageType.GENERATE_PLAN, @@ -215,7 +215,7 @@ function registerHandlers(): void { Logging.log(logMsg.source || 'Unknown', logMsg.message, logMsg.level || 'info') } ) - + // Metrics handler messageRouter.registerHandler( MessageType.LOG_METRIC, @@ -224,7 +224,7 @@ function registerHandlers(): void { Logging.logMetric(event, properties) } ) - + // Heartbeat handler - acknowledge heartbeats to keep connection alive messageRouter.registerHandler( MessageType.HEARTBEAT, @@ -237,7 +237,7 @@ function registerHandlers(): void { }) } ) - + // Panel close handler messageRouter.registerHandler( MessageType.CLOSE_PANEL, @@ -290,33 +290,33 @@ function registerHandlers(): void { */ function handlePortConnection(port: chrome.runtime.Port): void { const portId = portManager.registerPort(port) - + // Handle sidepanel connections if (port.name === 'sidepanel') { isPanelOpen = true Logging.log('Background', `Side panel connected`) Logging.logMetric('side_panel_opened', { source: 'port_connection' }) } - + // Register with logging system Logging.registerPort(port.name, port) - + // Set up message listener port.onMessage.addListener((message: PortMessage) => { messageRouter.routeMessage(message, port) }) - + // Set up disconnect listener port.onDisconnect.addListener(() => { portManager.unregisterPort(port) - + // Update panel state if this was the sidepanel if (port.name === 'sidepanel') { isPanelOpen = false Logging.log('Background', `Side panel disconnected`) Logging.logMetric('side_panel_closed', { source: 'port_disconnection' }) } - + // Unregister from logging Logging.unregisterPort(port.name) }) @@ -327,9 +327,9 @@ function handlePortConnection(port: chrome.runtime.Port): void { */ async function toggleSidePanel(tabId: number): Promise { if (isPanelToggling) return - + isPanelToggling = true - + try { if (isPanelOpen) { // Signal sidepanel to close itself @@ -345,7 +345,7 @@ async function toggleSidePanel(tabId: number): Promise { } } catch (error) { Logging.log('Background', `Error toggling side panel: ${error}`, 'error') - + // Try fallback with windowId if (!isPanelOpen) { try { @@ -371,36 +371,84 @@ async function toggleSidePanel(tabId: number): Promise { */ function registerNotificationListeners() { - // event listener to listen for detecting when browser is opened/resumed - chrome.windows.onFocusChanged.addListener((windowId) => { + // key: notificationId , value: windowId + const windowIds = new Map(); - // windowId is not none that means window is focues - if( windowId !== chrome.windows.WINDOW_ID_NONE ) { + // key: windowId , value: notificationId[] + const notificationIds = new Map>(); - //clear all notifications because browser is in focus now - chrome.notifications.getAll((notifications) => { + const getNotificationIds = (windowId: number): Array => { - Object.keys(notifications).forEach(id => { - chrome.notifications.clear(id); - }) + if( !notificationIds.has(windowId) ) { + return []; + } - }) + return notificationIds.get(windowId)!; + + } + + // event listener to listen for detecting when browser is opened/resumed + chrome.windows.onFocusChanged.addListener(async (windowId) => { + // windowId is not none when all chrome windows + // are out of focus that means no notification needs to be cleared + if (windowId == chrome.windows.WINDOW_ID_NONE) { + return; } - + + const data = getNotificationIds(windowId); + + data.forEach( notificationId => { + chrome.notifications.clear(notificationId); + } ) + + notificationIds.delete(windowId); + }); //handle click of notification - chrome.notifications.onClicked.addListener((noticationId) => { + chrome.notifications.onClicked.addListener((notificationId) => { // clear notification - chrome.notifications.clear(noticationId); - chrome.windows.getCurrent((window) => { + const windowId = windowIds.get(notificationId); + + if( windowId ) { + //open browser window - chrome.windows.update(window.id!, { focused: true }); - }); + chrome.windows.update( windowId , { focused: true }); + windowIds.delete(notificationId); + // Not clearing `notificationId` from `notficationIds` map here becaue + // the above code will open browser window and it will be cleared in the + // `onFocusChange` handler + } else { + console.info("window if not found for notification" , notificationId); + } + + chrome.notifications.clear(notificationId); + + }); + + //listener for sending notification + chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { + if (request.action === "send-notification") { + const windowId = request.windowId; + + //setting window if against notification so that it can be used to clear notification when the window opens + chrome.notifications.create(request.options, async (notificationId) => { + if (windowId) { + let existingNotificationIds = getNotificationIds(windowId!); + existingNotificationIds.push(notificationId); + + notificationIds.set(windowId!, existingNotificationIds); + windowIds.set(notificationId, windowId!); + console.log("setting window id", notificationId, windowId, windowIds); + } + sendResponse(notificationId); + }); + } + return true; }); } @@ -463,7 +511,7 @@ function initialize(): void { // Set up port connection listener chrome.runtime.onConnect.addListener(handlePortConnection) - + // Set up extension icon click handler chrome.action.onClicked.addListener(async (tab) => { Logging.log('Background', 'Extension icon clicked') @@ -471,7 +519,7 @@ function initialize(): void { await toggleSidePanel(tab.id) } }) - + // Set up keyboard shortcut handler chrome.commands.onCommand.addListener(async (command) => { if (command === 'toggle-panel') { @@ -482,13 +530,13 @@ function initialize(): void { } } }) - + // Clean up on tab removal chrome.tabs.onRemoved.addListener(async (tabId) => { // With singleton execution, just log the tab removal Logging.log('Background', `Tab ${tabId} removed`) }) - + // Handle messages from newtab only chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { if (message.type === 'NEWTAB_EXECUTE_QUERY') { @@ -496,10 +544,9 @@ function initialize(): void { return true // Keep message channel open for async response } }) - + Logging.log('Background', 'Nxtscape extension initialized successfully') } // Initialize the extension -initialize() - +initialize() \ No newline at end of file diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index 0814bb5e..db2caa80 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -111,7 +111,7 @@ export function App() { title: "Human input needed", message: humanInputRequest.prompt, type: 'basic', - iconUrl: 'assets/icon48.png', + iconUrl: chrome.runtime.getURL('assets/icon48.png'), isClickable: true, requireInteraction: true, }); diff --git a/src/sidepanel/hooks/usePushNotification.ts b/src/sidepanel/hooks/usePushNotification.ts index eae4c81b..a41e5a3f 100644 --- a/src/sidepanel/hooks/usePushNotification.ts +++ b/src/sidepanel/hooks/usePushNotification.ts @@ -7,25 +7,29 @@ export function usePushNotification() { /** * Can be called for sending notifications + * + * options parameter is `chrome.notifications.NotificationOptions` because `chrome.notifications.create` function expects it like that */ - const sendNotification = useCallback(async (options: chrome.notifications.NotificationOptions ): Promise => { + const sendNotification = useCallback(async (options: chrome.notifications.NotificationOptions): Promise => { return new Promise(async (resolve, reject) => { - try { - - const noticationId = `notification-${Date.now()}`; - - chrome.notifications.create( noticationId , options, (notificationId) => { - resolve(notificationId); + const { id: windowId } = await chrome.windows.getCurrent(); + + chrome.runtime.sendMessage({ + action: "send-notification", + options, + windowId + }, (response) => { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + return; + } + resolve(response); }); - - } catch (err: any) { - - reject(err); - + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))); } - }) }, []); From 0f67b1650e526f9a54a3524425cb8588cdaa5469 Mon Sep 17 00:00:00 2001 From: Jayshankar Date: Wed, 29 Oct 2025 19:57:54 +0530 Subject: [PATCH 3/3] fix: Use window.focused for checking is window is not focus ( not document.hidden ) and some refactoring changes --- src/background/index.ts | 11 ++----- src/sidepanel/App.tsx | 37 ++++++++++++++-------- src/sidepanel/hooks/usePushNotification.ts | 11 +++---- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/background/index.ts b/src/background/index.ts index 9b05e002..8c24b5cd 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -398,9 +398,7 @@ function registerNotificationListeners() { const data = getNotificationIds(windowId); - data.forEach( notificationId => { - chrome.notifications.clear(notificationId); - } ) + data.forEach(notificationId => chrome.notifications.clear(notificationId)); notificationIds.delete(windowId); @@ -409,11 +407,10 @@ function registerNotificationListeners() { //handle click of notification chrome.notifications.onClicked.addListener((notificationId) => { - // clear notification const windowId = windowIds.get(notificationId); - if( windowId ) { + if( windowId !== undefined ) { //open browser window chrome.windows.update( windowId , { focused: true }); @@ -422,10 +419,9 @@ function registerNotificationListeners() { // Not clearing `notificationId` from `notficationIds` map here becaue // the above code will open browser window and it will be cleared in the // `onFocusChange` handler - } else { - console.info("window if not found for notification" , notificationId); } + // clear notification chrome.notifications.clear(notificationId); }); @@ -443,7 +439,6 @@ function registerNotificationListeners() { notificationIds.set(windowId!, existingNotificationIds); windowIds.set(notificationId, windowId!); - console.log("setting window id", notificationId, windowId, windowIds); } sendResponse(notificationId); }); diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index db2caa80..68b0d48d 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -103,20 +103,31 @@ export function App() { // show push notification if human input is needed and browser is hidden useEffect(() => { - - // document.hidden may incorrect values in some linux distros , but it works in most - if (humanInputRequest && document.hidden) { - - sendNotification({ - title: "Human input needed", - message: humanInputRequest.prompt, - type: 'basic', - iconUrl: chrome.runtime.getURL('assets/icon48.png'), - isClickable: true, - requireInteraction: true, - }); - } + (async () => { + + const { id: windowId } = await chrome.windows.getCurrent(); + + if( windowId === undefined ) { + return; + } + + const window = await chrome.windows.get(windowId); + + if (humanInputRequest && !window.focused) { + + sendNotification({ + title: "Human input needed", + message: humanInputRequest.prompt, + type: 'basic', + iconUrl: chrome.runtime.getURL('assets/icon48.png'), + isClickable: true, + requireInteraction: true, + }); + + } + + })() }, [humanInputRequest]); diff --git a/src/sidepanel/hooks/usePushNotification.ts b/src/sidepanel/hooks/usePushNotification.ts index a41e5a3f..29be108e 100644 --- a/src/sidepanel/hooks/usePushNotification.ts +++ b/src/sidepanel/hooks/usePushNotification.ts @@ -15,12 +15,11 @@ export function usePushNotification() { return new Promise(async (resolve, reject) => { try { const { id: windowId } = await chrome.windows.getCurrent(); - - chrome.runtime.sendMessage({ - action: "send-notification", - options, - windowId - }, (response) => { + + // Not checking windowId for undefined here because it is already being checked in the background script + // And sending the notification has more priority than saving windowId + + chrome.runtime.sendMessage({ action: "send-notification", options, windowId }, (response) => { if (chrome.runtime.lastError) { reject(new Error(chrome.runtime.lastError.message)); return;