From fe1290dd942dfd61d225956ee92cd10ddf9517a9 Mon Sep 17 00:00:00 2001 From: n0nag0n Date: Sat, 29 Mar 2025 08:06:46 -0600 Subject: [PATCH 1/5] fixed 500 error and corrected request sort order --- src/apm/presenter/SqlitePresenter.php | 41 ++++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/apm/presenter/SqlitePresenter.php b/src/apm/presenter/SqlitePresenter.php index 3b0c004..f1769d0 100644 --- a/src/apm/presenter/SqlitePresenter.php +++ b/src/apm/presenter/SqlitePresenter.php @@ -126,28 +126,29 @@ public function getDashboardData(string $threshold, string $range = 'last_hour') $p95 = $this->calculatePercentile($times, 95); $p99 = $this->calculatePercentile($times, 99); - // Graph Data (aggregated) - $stmt = $this->db->prepare('SELECT timestamp, total_time FROM apm_requests WHERE timestamp >= ? ORDER BY timestamp'); - $stmt->execute([$threshold]); - $requestData = $stmt->fetchAll(); - $aggregatedData = []; + // Graph Data (aggregated) - Optimized for large datasets + // Use SQLite's strftime to group by time intervals directly in SQL + $intervalSeconds = $interval; + $stmt = $this->db->prepare(" + SELECT + (strftime('%s', timestamp) / ?) * ? as time_bucket, + AVG(total_time) as average_time, + COUNT(*) as request_count + FROM apm_requests + WHERE timestamp >= ? + GROUP BY time_bucket + ORDER BY time_bucket + "); + $stmt->execute([$intervalSeconds, $intervalSeconds, $threshold]); + $aggregatedData = $stmt->fetchAll(); - // Use the same interval for consistent visualization - foreach ($requestData as $row) { - $timestamp = strtotime($row['timestamp']); - $bucket = floor($timestamp / $interval) * $interval; - if (!isset($aggregatedData[$bucket])) { - $aggregatedData[$bucket] = ['sum' => 0, 'count' => 0]; - } - $aggregatedData[$bucket]['sum'] += $row['total_time']; - $aggregatedData[$bucket]['count']++; - } - $chartData = array_map(function($bucket, $data) { + $chartData = array_map(function($row) { return [ - 'timestamp' => date('Y-m-d H:i:s', (int) $bucket), - 'average_time' => $data['sum'] / $data['count'], + 'timestamp' => date('Y-m-d H:i:s', (int) $row['time_bucket']), + 'average_time' => $row['average_time'], + 'request_count' => $row['request_count'] ]; - }, array_keys($aggregatedData), $aggregatedData); + }, $aggregatedData); return [ 'slowRequests' => $slowRequests, @@ -320,7 +321,7 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri // Get the actual request data $requestQuery = "SELECT request_id, timestamp, request_url, total_time, response_code, is_bot, ip, user_agent, host, session_id FROM apm_requests - WHERE request_id IN ($placeholders) ORDER BY timestamp DESC"; + WHERE request_id IN ($placeholders) ORDER BY id DESC"; $stmt = $this->db->prepare($requestQuery); $stmt->execute($paginatedRequestIds); $requests = $stmt->fetchAll(PDO::FETCH_ASSOC); From 2406c3f474ca0e0a0cb5acdc298315db08963e56 Mon Sep 17 00:00:00 2001 From: n0nag0n Date: Mon, 31 Mar 2025 19:24:58 -0600 Subject: [PATCH 2/5] allow masking of IPs on the dashboard --- src/apm/presenter/SqlitePresenter.php | 49 +++++++++++++++++++++++++++ src/commands/InitCommand.php | 8 +++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/apm/presenter/SqlitePresenter.php b/src/apm/presenter/SqlitePresenter.php index f1769d0..2c247fa 100644 --- a/src/apm/presenter/SqlitePresenter.php +++ b/src/apm/presenter/SqlitePresenter.php @@ -330,6 +330,11 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri foreach ($requests as &$request) { $details = $this->getRequestDetails($request['request_id']); $request = array_merge($request, $details); + + // Mask IP address if the option is enabled + if ($this->shouldMaskIpAddresses()) { + $request['ip'] = $this->maskIpAddress($request['ip']); + } } unset($request); @@ -454,6 +459,11 @@ public function getRequestDetails(string $requestId): array $request['custom_events'] = $events; + // Mask IP address if the option is enabled + if ($this->shouldMaskIpAddresses() && isset($request['ip'])) { + $request['ip'] = $this->maskIpAddress($request['ip']); + } + return $request; } @@ -490,4 +500,43 @@ protected function databaseSupportsJson(): bool return false; } } + + /** + * Check if IP addresses should be masked + * + * @return bool + */ + protected function shouldMaskIpAddresses(): bool + { + // Assuming the configuration is accessible via $this->config + return $this->config['apm']['mask_ip_addresses'] ?? false; + } + + /** + * Mask an IP address + * + * @param string $ip + * @return string + */ + protected function maskIpAddress(string $ip): string + { + + // Check if it's ipv4 or ipv6 with a '.' or a ':' + $ipCharacter = strpos($ip, '.') !== false ? '.' : ':'; + + $parts = explode($ipCharacter, $ip); + if (count($parts) === 4) { + // replace the last part of the IP address with 'x' for as many digits as there are + $parts[3] = str_repeat('x', strlen($parts[3])); + } + + // account for ipv6 addresses + if (count($parts) > 4) { + // replace the last part of the IP address with 'x' for as many digits as there are + $parts[count($parts) - 1] = str_repeat('x', strlen($parts[count($parts) - 1])); + } + + // Join the parts back together + return implode('.', $parts); + } } diff --git a/src/commands/InitCommand.php b/src/commands/InitCommand.php index 8d4852f..08138c1 100644 --- a/src/commands/InitCommand.php +++ b/src/commands/InitCommand.php @@ -161,8 +161,12 @@ protected function runStorageConfigWalkthrough(string $configFile): void $config['apm'] = []; } - $config['apm'] = $apmConfig; - + // Add option for masking IP addresses + $maskIp = $io->prompt('Do you want to mask IP addresses in the dashboard? (y/n)', 'n'); + $apmConfig['mask_ip_addresses'] = strtolower($maskIp) === 'y'; + + $config['apm'] = $apmConfig; + $json = json_encode($config, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); file_put_contents($configFile, $json); From 0c3cd25c8c5594520022560c421ede700877db5a Mon Sep 17 00:00:00 2001 From: n0nag0n Date: Mon, 31 Mar 2025 20:19:51 -0600 Subject: [PATCH 3/5] initial changes to better custom event searching --- dashboard/index.php | 8 + dashboard/js/script.js | 257 +++++++++++++++--- dashboard/views/dashboard.php | 53 +++- .../migration/sqlite/0004-custom-events.sql | 31 +++ src/apm/presenter/PresenterInterface.php | 8 + src/apm/presenter/SqlitePresenter.php | 181 +++++++++++- src/apm/writer/SqliteWriter.php | 31 ++- 7 files changed, 520 insertions(+), 49 deletions(-) create mode 100644 src/apm/migration/sqlite/0004-custom-events.sql diff --git a/dashboard/index.php b/dashboard/index.php index 6a35a39..c770a0f 100644 --- a/dashboard/index.php +++ b/dashboard/index.php @@ -62,4 +62,12 @@ function calculateThreshold($range) { $app->json($data); }); +// New endpoint to retrieve available event keys for filtering +$app->route('GET /apm/data/event-keys', function() use ($app, $presenter) { + $range = Flight::request()->query['range'] ?? 'last_hour'; + $threshold = calculateThreshold($range); + $eventKeys = $presenter->getEventKeys($threshold); + $app->json(['event_keys' => $eventKeys]); +}); + $app->start(); diff --git a/dashboard/js/script.js b/dashboard/js/script.js index f4fdbe2..359e140 100644 --- a/dashboard/js/script.js +++ b/dashboard/js/script.js @@ -119,7 +119,12 @@ function loadRequestLogData() { // Add each active filter to the query params Object.entries(activeFilters).forEach(([key, value]) => { if (value !== null && value !== '') { - queryParams.append(key, value); + // Special handling for event_keys which needs to be JSON + if (key === 'event_keys') { + queryParams.append(key, JSON.stringify(value)); + } else { + queryParams.append(key, value); + } } }); @@ -684,6 +689,140 @@ function setupFilterHandlers() { const clearFiltersBtn = document.getElementById('clear-filters'); const activeFiltersContainer = document.getElementById('active-filters'); const activeFiltersList = document.getElementById('active-filters-list'); + const addEventFilterBtn = document.getElementById('add-event-filter'); + const eventFiltersContainer = document.getElementById('event-filters-container'); + const noEventFiltersMsg = document.getElementById('no-event-filters-msg'); + const eventFilterTemplate = document.getElementById('event-filter-template'); + + // Store available keys for reuse + let availableEventKeys = []; + + // Define operators directly instead of loading them + const availableOperators = [ + { id: 'contains', name: 'Contains', desc: 'Value contains the text (case-insensitive)' }, + { id: 'exact', name: 'Equals', desc: 'Value exactly matches the text' }, + { id: 'starts_with', name: 'Starts with', desc: 'Value starts with the text' }, + { id: 'ends_with', name: 'Ends with', desc: 'Value ends with the text' }, + { id: 'greater_than', name: '>', desc: 'Value is greater than (numeric comparison)' }, + { id: 'less_than', name: '<', desc: 'Value is less than (numeric comparison)' }, + { id: 'greater_than_equal', name: '>=', desc: 'Value is greater than or equal to (numeric comparison)' }, + { id: 'less_than_equal', name: '<=', desc: 'Value is less than or equal to (numeric comparison)' } + ]; + + // Load event keys for dropdowns + function loadEventKeys() { + const range = rangeSelector.value; + fetch(`/apm/data/event-keys?range=${range}`) + .then(response => { + if (!response.ok) throw new Error('Network response was not ok'); + return response.json(); + }) + .then(data => { + availableEventKeys = data.event_keys || []; + updateEventKeyDropdowns(); + }) + .catch(error => console.error('Error loading event keys:', error)); + } + + // Update all event key dropdowns with available options + function updateEventKeyDropdowns() { + document.querySelectorAll('.event-key-select').forEach(select => { + const currentValue = select.value; + + // Clear existing options except the first one + while (select.options.length > 1) { + select.remove(1); + } + + // Add options for each key + if (availableEventKeys.length) { + availableEventKeys.forEach(key => { + const option = document.createElement('option'); + option.value = key; + option.textContent = key; + select.appendChild(option); + }); + } + + // Restore previous value if it exists + if (currentValue && availableEventKeys.includes(currentValue)) { + select.value = currentValue; + } + }); + } + + // Update all operator dropdowns (now uses hardcoded values) + function updateOperatorDropdowns() { + document.querySelectorAll('.event-operator-select').forEach(select => { + const currentValue = select.value || 'contains'; + + // Clear existing options + select.innerHTML = ''; + + // Add options for each operator + availableOperators.forEach(op => { + const option = document.createElement('option'); + option.value = op.id; + option.textContent = op.name; + option.title = op.desc; + select.appendChild(option); + }); + + // Restore previous value + select.value = currentValue; + }); + } + + // Create a new event filter row + function addEventFilterRow(keyValue = '', operator = 'contains', valueValue = '') { + // Hide the "no filters" message + noEventFiltersMsg.style.display = 'none'; + + // Clone the template + const clone = document.importNode(eventFilterTemplate.content, true); + const row = clone.querySelector('.event-filter-row'); + + // Set initial values if provided + const keySelect = row.querySelector('.event-key-select'); + const operatorSelect = row.querySelector('.event-operator-select'); + const valueInput = row.querySelector('.event-value-input'); + + // Populate dropdowns + updateEventKeyDropdowns(); + updateOperatorDropdowns(); + + // Set values + if (keyValue) keySelect.value = keyValue; + if (operator) operatorSelect.value = operator; + if (valueValue) valueInput.value = valueValue; + + // Add remove handler + row.querySelector('.remove-event-filter').addEventListener('click', function() { + row.remove(); + + // Show message if no filters remain + if (eventFiltersContainer.querySelectorAll('.event-filter-row').length === 0) { + noEventFiltersMsg.style.display = 'block'; + } + }); + + // Append to container + eventFiltersContainer.appendChild(row); + + return row; + } + + // Add event filter button click + addEventFilterBtn.addEventListener('click', () => { + addEventFilterRow(); + }); + + // Initial load of event keys and populate operator dropdowns + loadEventKeys(); + updateOperatorDropdowns(); + + // Reload event keys when range changes + rangeSelector.addEventListener('change', loadEventKeys); // Show/hide exact code input based on selection filterResponseCode.addEventListener('change', () => { @@ -708,6 +847,18 @@ function setupFilterHandlers() { const sessionId = document.getElementById('filter-session-id').value.trim(); const userAgent = document.getElementById('filter-user-agent').value.trim(); + // Collect event filter data + const eventFilters = []; + document.querySelectorAll('.event-filter-row').forEach(row => { + const key = row.querySelector('.event-key-select').value; + const operator = row.querySelector('.event-operator-select').value; + const value = row.querySelector('.event-value-input').value.trim(); + + if (key || value) { + eventFilters.push({ key, operator, value }); + } + }); + // Clear previous filters activeFilters = {}; @@ -731,6 +882,11 @@ function setupFilterHandlers() { if (sessionId) activeFilters.session_id = sessionId; if (userAgent) activeFilters.user_agent = userAgent; + // Add event filters if any exist + if (eventFilters.length > 0) { + activeFilters.event_keys = eventFilters; + } + // Update UI to show active filters updateActiveFiltersDisplay(); @@ -758,6 +914,10 @@ function setupFilterHandlers() { document.getElementById('filter-session-id').value = ''; document.getElementById('filter-user-agent').value = ''; + // Clear event filters + eventFiltersContainer.innerHTML = ''; + noEventFiltersMsg.style.display = 'block'; + // Hide the exact code input exactCodeContainer.style.display = 'none'; @@ -790,56 +950,77 @@ function setupFilterHandlers() { switch (key) { case 'url': - filterLabel = `URL: ${value}`; - break; case 'request_id': - filterLabel = `Request ID: ${value}`; - break; case 'response_code': - filterLabel = `Code: ${value}`; - break; - case 'response_code_prefix': - filterLabel = `Code: ${value}xx`; - break; - case 'is_bot': - filterLabel = `Bot: ${value === '1' ? 'Yes' : 'No'}`; - break; case 'custom_event_type': - filterLabel = `Event: ${value}`; - break; case 'min_time': - filterLabel = `Min Time: ${value}ms`; - break; - // Add new metadata filter labels case 'ip': - filterLabel = `IP: ${value}`; - break; case 'host': - filterLabel = `Host: ${value}`; - break; case 'session_id': - filterLabel = `Session: ${value}`; - break; case 'user_agent': - filterLabel = `User Agent: ${value}`; + // ...existing code for these cases... break; + + case 'event_keys': + // For each event key filter, create a separate badge + value.forEach((filter, index) => { + const opDisplay = availableOperators.find(op => op.id === filter.operator)?.name || filter.operator; + const filterKey = `event_keys_${index}`; + const display = filter.key ? + `${filter.key} ${opDisplay} ${filter.value}` : + `Any Key ${opDisplay} ${filter.value}`; + + const badge = document.createElement('span'); + badge.className = 'badge bg-info me-2 mb-1'; + badge.innerHTML = `Event: ${display} `; + + // Add click event to remove this specific event filter + badge.querySelector('i').addEventListener('click', function() { + const index = parseInt(this.dataset.filterIndex); + activeFilters.event_keys.splice(index, 1); + + if (activeFilters.event_keys.length === 0) { + delete activeFilters.event_keys; + } + + updateActiveFiltersDisplay(); + currentPage = 1; + loadRequestLogData(); + }); + + activeFiltersList.appendChild(badge); + }); + return; // Skip the default badge creation for this key + default: filterLabel = `${key}: ${value}`; } - const badge = document.createElement('span'); - badge.className = 'badge bg-info me-2 mb-1'; - badge.innerHTML = `${filterLabel} `; - - // Add click event to remove individual filter - badge.querySelector('i').addEventListener('click', function() { - delete activeFilters[this.dataset.filter]; - updateActiveFiltersDisplay(); - currentPage = 1; - loadRequestLogData(); - }); - - activeFiltersList.appendChild(badge); + if (filterLabel) { // Only create badge if we have a label + const badge = document.createElement('span'); + badge.className = 'badge bg-info me-2 mb-1'; + badge.innerHTML = `${filterLabel} `; + + // Add click event to remove individual filter + badge.querySelector('i').addEventListener('click', function() { + delete activeFilters[this.dataset.filter]; + updateActiveFiltersDisplay(); + currentPage = 1; + loadRequestLogData(); + }); + + activeFiltersList.appendChild(badge); + } + }); + } + + // If we have event filters in active filters, recreate the UI for them + if (activeFilters.event_keys && Array.isArray(activeFilters.event_keys) && activeFilters.event_keys.length > 0) { + noEventFiltersMsg.style.display = 'none'; + eventFiltersContainer.innerHTML = ''; + + activeFilters.event_keys.forEach(filter => { + addEventFilterRow(filter.key, filter.operator, filter.value); }); } } diff --git a/dashboard/views/dashboard.php b/dashboard/views/dashboard.php index a234007..527031b 100644 --- a/dashboard/views/dashboard.php +++ b/dashboard/views/dashboard.php @@ -183,10 +183,55 @@ -
- - -
+ + + +
+
+
+ Event Data Filters + +
+
+ +
+ No event data filters. Click "Add Filter" to add one. +
+
+
+
+ + + + +
+ + +
diff --git a/src/apm/migration/sqlite/0004-custom-events.sql b/src/apm/migration/sqlite/0004-custom-events.sql new file mode 100644 index 0000000..5a66143 --- /dev/null +++ b/src/apm/migration/sqlite/0004-custom-events.sql @@ -0,0 +1,31 @@ + +-- Add a deprecated comment to the apm_custom_events.event_data column +ALTER TABLE apm_custom_events +CHANGE COLUMN event_data event_data TEXT COMMENT 'Deprecated: Use apm_custom_event_data instead.'; + + +-- Create a new table for custom event key-value data +CREATE TABLE IF NOT EXISTS apm_custom_event_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + custom_event_id INTEGER NOT NULL, + request_id TEXT NOT NULL, + json_key TEXT NOT NULL, + json_value TEXT, + FOREIGN KEY (custom_event_id) REFERENCES apm_custom_events(id) ON DELETE CASCADE, + FOREIGN KEY (request_id) REFERENCES apm_requests(request_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_apm_custom_event_data_event_id ON apm_custom_event_data(custom_event_id); +CREATE INDEX IF NOT EXISTS idx_apm_custom_event_data_request_id ON apm_custom_event_data(request_id); +CREATE INDEX IF NOT EXISTS idx_apm_custom_event_data_key ON apm_custom_event_data(json_key); + +-- Migrate existing event_data JSON into the new apm_custom_event_data table +INSERT INTO apm_custom_event_data (custom_event_id, request_id, json_key, json_value) +SELECT + id AS custom_event_id, + request_id, + json_each.json_key AS json_key, + json_each.json_value AS json_value +FROM + apm_custom_events, + json_each(apm_custom_events.event_data); diff --git a/src/apm/presenter/PresenterInterface.php b/src/apm/presenter/PresenterInterface.php index a97c20a..fc5c9bc 100644 --- a/src/apm/presenter/PresenterInterface.php +++ b/src/apm/presenter/PresenterInterface.php @@ -30,4 +30,12 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri * @return array Request details including middleware, queries, errors, cache operations, and custom events */ public function getRequestDetails(string $requestId): array; + + /** + * Get available event keys for search filters + * + * @param string $threshold Timestamp threshold + * @return array List of unique event keys + */ + public function getEventKeys(string $threshold): array; } diff --git a/src/apm/presenter/SqlitePresenter.php b/src/apm/presenter/SqlitePresenter.php index 2c247fa..3fddec1 100644 --- a/src/apm/presenter/SqlitePresenter.php +++ b/src/apm/presenter/SqlitePresenter.php @@ -175,7 +175,7 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri // Check if database supports JSON functions $hasJsonFunctions = $this->databaseSupportsJson(); - + // Extract filter parameters from the search string or use directly provided parameters $url = $_GET['url'] ?? ''; $responseCode = $_GET['response_code'] ?? ''; @@ -191,6 +191,9 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri $sessionId = $_GET['session_id'] ?? ''; $userAgent = $_GET['user_agent'] ?? ''; + // Enhanced custom event data filters - multiple keys and values with operators + $eventKeys = isset($_GET['event_keys']) ? json_decode($_GET['event_keys'], true) : []; + // Build main query with conditions for URL and response code $conditions = ['timestamp >= ?']; $params = [$threshold]; @@ -280,8 +283,105 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri $customEventRequestIds = $stmt->fetchAll(PDO::FETCH_COLUMN); } + // Get request IDs from custom event data if key/value filters are set + $customEventDataRequestIds = []; + if (!empty($eventKeys)) { + // Build complex query for multiple key/value pairs + $keyValueConditions = []; + $keyValueParams = []; + $keyValueGroups = []; + + // Group number to match pairs of key/value conditions + $groupNum = 0; + + foreach ($eventKeys as $filter) { + $key = $filter['key'] ?? ''; + $operator = $filter['operator'] ?? 'contains'; + $value = $filter['value'] ?? ''; + + if (empty($key) && empty($value)) { + continue; + } + + $groupConditions = []; + + // Add key condition if provided + if (!empty($key)) { + $groupConditions[] = "json_key = ?"; + $keyValueParams[] = $key; + } + + // Add value condition if provided + if (!empty($value)) { + switch ($operator) { + case 'exact': + $groupConditions[] = "json_value = ?"; + $keyValueParams[] = $value; + break; + case 'contains': + $groupConditions[] = "json_value LIKE ?"; + $keyValueParams[] = "%$value%"; + break; + case 'starts_with': + $groupConditions[] = "json_value LIKE ?"; + $keyValueParams[] = "$value%"; + break; + case 'ends_with': + $groupConditions[] = "json_value LIKE ?"; + $keyValueParams[] = "%$value"; + break; + case 'greater_than': + $groupConditions[] = "CAST(json_value AS NUMERIC) > ?"; + $keyValueParams[] = $value; + break; + case 'less_than': + $groupConditions[] = "CAST(json_value AS NUMERIC) < ?"; + $keyValueParams[] = $value; + break; + case 'greater_than_equal': + $groupConditions[] = "CAST(json_value AS NUMERIC) >= ?"; + $keyValueParams[] = $value; + break; + case 'less_than_equal': + $groupConditions[] = "CAST(json_value AS NUMERIC) <= ?"; + $keyValueParams[] = $value; + break; + } + } + + if (!empty($groupConditions)) { + $keyValueGroups[] = "(" . implode(" AND ", $groupConditions) . ")"; + $groupNum++; + } + } + + if (!empty($keyValueGroups)) { + // For "AND" logic between pairs (all conditions must match) + $keyValueQuery = " + SELECT request_id + FROM apm_requests + WHERE request_id IN ( + SELECT request_id + FROM apm_custom_event_data + WHERE " . implode(" OR ", $keyValueGroups) . " + GROUP BY request_id + HAVING COUNT(DISTINCT json_key) >= ? + ) + LIMIT ? + "; + + // Add the count of distinct conditions to ensure all match + $keyValueParams[] = count(array_filter($eventKeys, fn($f) => !empty($f['key']) || !empty($f['value']))); + $keyValueParams[] = $maxRequests; + + $stmt = $this->db->prepare($keyValueQuery); + $stmt->execute($keyValueParams); + $customEventDataRequestIds = $stmt->fetchAll(PDO::FETCH_COLUMN); + } + } + // Merge request IDs from different sources - $allRequestIds = array_merge($mainRequestIds, $customEventRequestIds); + $allRequestIds = array_merge($mainRequestIds, $customEventRequestIds, $customEventDataRequestIds); $uniqueRequestIds = array_unique($allRequestIds); // If we have no matching requests, return empty result @@ -442,18 +542,48 @@ public function getRequestDetails(string $requestId): array $stmt->execute([$requestId]); $request['cache'] = $stmt->fetchAll(PDO::FETCH_ASSOC); - // Custom Events - $stmt = $this->db->prepare('SELECT timestamp, event_type, event_data FROM apm_custom_events WHERE request_id = ?'); + // Custom Events - Use both tables for comprehensive data + $stmt = $this->db->prepare('SELECT id, timestamp, event_type, event_data FROM apm_custom_events WHERE request_id = ?'); $stmt->execute([$requestId]); $customEvents = $stmt->fetchAll(PDO::FETCH_ASSOC); // Process custom events data $events = []; foreach ($customEvents as $event) { + $eventData = json_decode($event['event_data'], true); + + // If the custom event data table exists, retrieve the key-value pairs + $dataStmt = $this->db->prepare(' + SELECT json_key, json_value FROM apm_custom_event_data + WHERE custom_event_id = ? AND request_id = ? + '); + $dataStmt->execute([$event['id'], $requestId]); + $keyValuePairs = $dataStmt->fetchAll(PDO::FETCH_KEY_PAIR); + + // For each key-value pair, try to decode JSON values + foreach ($keyValuePairs as $key => $value) { + // Try to decode the value if it looks like JSON + if (is_string($value) && + ((str_starts_with($value, '{') && str_ends_with($value, '}')) || + (str_starts_with($value, '[') && str_ends_with($value, ']')))) { + try { + $decodedValue = json_decode($value, true); + if (json_last_error() === JSON_ERROR_NONE) { + $keyValuePairs[$key] = $decodedValue; + } + } catch (\Exception $e) { + // Keep the original value if decoding fails + } + } + } + + // Use key-value pairs if available, otherwise fall back to the JSON data + $eventData = !empty($keyValuePairs) ? $keyValuePairs : $eventData; + $events[] = [ 'timestamp' => $event['timestamp'], 'type' => $event['event_type'], - 'data' => json_decode($event['event_data'], true) + 'data' => $eventData ]; } @@ -539,4 +669,45 @@ protected function maskIpAddress(string $ip): string // Join the parts back together return implode('.', $parts); } + + /** + * Get available event keys for search filters + * + * @param string $threshold Timestamp threshold + * @return array List of unique event keys + */ + public function getEventKeys(string $threshold): array + { + try { + $stmt = $this->db->prepare(" + SELECT DISTINCT json_key + FROM apm_custom_event_data + WHERE request_id IN (SELECT request_id FROM apm_requests WHERE timestamp >= ?) + ORDER BY json_key + "); + $stmt->execute([$threshold]); + return $stmt->fetchAll(PDO::FETCH_COLUMN); + } catch (\Exception $e) { + return []; + } + } + + /** + * Get operator options for event value filtering + * + * @return array List of available operators + */ + public function getEventValueOperators(): array + { + return [ + ['id' => 'contains', 'name' => 'Contains', 'desc' => 'Value contains the text (case-insensitive)'], + ['id' => 'exact', 'name' => 'Equals', 'desc' => 'Value exactly matches the text'], + ['id' => 'starts_with', 'name' => 'Starts with', 'desc' => 'Value starts with the text'], + ['id' => 'ends_with', 'name' => 'Ends with', 'desc' => 'Value ends with the text'], + ['id' => 'greater_than', 'name' => '>', 'desc' => 'Value is greater than (numeric comparison)'], + ['id' => 'less_than', 'name' => '<', 'desc' => 'Value is less than (numeric comparison)'], + ['id' => 'greater_than_equal', 'name' => '>=', 'desc' => 'Value is greater than or equal to (numeric comparison)'], + ['id' => 'less_than_equal', 'name' => '<=', 'desc' => 'Value is less than or equal to (numeric comparison)'], + ]; + } } diff --git a/src/apm/writer/SqliteWriter.php b/src/apm/writer/SqliteWriter.php index a4aee87..529327e 100644 --- a/src/apm/writer/SqliteWriter.php +++ b/src/apm/writer/SqliteWriter.php @@ -362,19 +362,46 @@ protected function storeCustomEvents(string $requestId, array $customEvents): vo return; } - $stmt = $this->getStatement(' + // First statement for original table + $eventStmt = $this->getStatement(' INSERT INTO apm_custom_events ( request_id, event_type, event_data, timestamp ) VALUES (?, ?, ?, datetime(?, \'unixepoch\')) '); + // Second statement for new key-value table + $dataStmt = $this->getStatement(' + INSERT INTO apm_custom_event_data ( + custom_event_id, request_id, json_key, json_value + ) VALUES (?, ?, ?, ?) + '); + foreach ($customEvents as $event) { - $stmt->execute([ + // Insert into main events table first + $eventStmt->execute([ $requestId, $event['type'], json_encode($event['data'], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR), $event['timestamp'] ]); + + // Get the last inserted ID for the foreign key relationship + $eventId = $this->pdo->lastInsertId(); + + // Now insert each key-value pair into the event_data table + foreach ($event['data'] as $key => $value) { + // If value is an array, convert it to JSON string + if (is_array($value)) { + $value = json_encode($value, JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); + } + + $dataStmt->execute([ + $eventId, + $requestId, + $key, + $value + ]); + } } } From b3da61f9fb010ebd4c989408634799684f25eb76 Mon Sep 17 00:00:00 2001 From: n0nag0n Date: Tue, 1 Apr 2025 20:32:25 -0600 Subject: [PATCH 4/5] updates to allow for proper searching and dark mode adjustments --- dashboard/css/style.css | 117 ++++++-- dashboard/js/script.js | 33 ++- dashboard/views/dashboard.php | 6 +- .../migration/sqlite/0004-custom-events.sql | 15 +- src/apm/presenter/SqlitePresenter.php | 258 +++++++++++------- 5 files changed, 272 insertions(+), 157 deletions(-) diff --git a/dashboard/css/style.css b/dashboard/css/style.css index 63e7e9f..c48bc82 100644 --- a/dashboard/css/style.css +++ b/dashboard/css/style.css @@ -14,66 +14,127 @@ } [data-theme="dark"] { - --bg-color: #212529; - --text-color: #f8f9fa; - --card-bg: #343a40; - --card-shadow: 0 4px 6px rgba(0, 0, 0, 0.3); - --header-bg: #2c3034; - --badge-primary: #6ea8fe; - --badge-success: #20c997; - --badge-danger: #f27474; - --badge-info: #6edff6; - --badge-warning: #ffca2c; - --border-color: #495057; + --bg-color: #121212; /* Darker background for better contrast */ + --text-color: #e0e0e0; /* Lighter text for readability */ + --card-bg: #1e1e1e; /* Slightly lighter card background */ + --card-shadow: 0 4px 6px rgba(0, 0, 0, 0.5); /* Stronger shadow for depth */ + --header-bg: #1a1a1a; /* Consistent header background */ + --badge-primary: #4a90e2; /* Softer blue for badges */ + --badge-success: #28a745; /* Standard green for success */ + --badge-danger: #dc3545; /* Standard red for danger */ + --badge-info: #17a2b8; /* Standard cyan for info */ + --badge-warning: #ffc107; /* Standard yellow for warning */ + --border-color: #333333; /* Subtle border color */ } [data-theme="dark"] .card-header { - color: var(--text-color); + background-color: #2a2a2a; /* Slightly darker background for contrast */ + color: var(--text-color); /* Ensure text is readable */ + border-bottom: 1px solid var(--border-color); /* Add subtle border */ + font-weight: 600; /* Keep font weight consistent */ } [data-theme="dark"] .list-group-item { color: var(--text-color); + background-color: var(--card-bg); /* Match card background */ } [data-theme="dark"] .card-body { color: var(--text-color); + background-color: var(--card-bg); /* Match card background */ } [data-theme="dark"] .badge { - color: #fff; /* Ensure badge text is always white for readability */ + color: #ffffff; /* Ensure badge text is always white */ + background-color: var(--badge-primary); /* Use primary badge color */ } [data-theme="dark"] .table { - background-color: var(--card-bg); /* Match the card background */ + background-color: var(--card-bg); /* Match card background */ color: var(--text-color); } [data-theme="dark"] .table thead th, .table>:not(caption)>*>* { - background-color: var(--header-bg); /* Match the header background */ + background-color: var(--header-bg); /* Match header background */ color: var(--text-color); - border-bottom: 2px solid #495057; /* Slightly lighter border for contrast */ + border-bottom: 2px solid var(--border-color); /* Subtle border for contrast */ } -[data-theme="dark"] .table tbody tr, .table>:not(caption)>*>* { - background-color: var(--card-bg); - border-bottom: 1px solid #495057; /* Subtle border between rows */ +[data-theme="dark"] .table tbody tr { + background-color: var(--card-bg); /* Match card background */ + border-bottom: 1px solid var(--border-color); /* Subtle border between rows */ } [data-theme="dark"] .table-hover tbody tr:hover { - background-color: rgba(255, 255, 255, 0.1); /* Keep the hover effect */ + background-color: rgba(255, 255, 255, 0.1); /* Slight hover effect */ } [data-theme="dark"] .form-control { - background-color: #495057; + background-color: #2a2a2a; /* Slightly lighter input background */ color: var(--text-color); - border-color: #6c757d; + border-color: var(--border-color); } [data-theme="dark"] .form-control::placeholder { - color: #ced4da; /* Lighter gray for better visibility */ + color: #b0b0b0; /* Lighter gray for better visibility */ opacity: 1; /* Ensure full opacity for readability */ } +[data-theme="dark"] .json-container { + background-color: #1e1e1e; /* Match card background */ + color: var(--text-color); + border: 1px solid var(--border-color); /* Add subtle border */ +} + +[data-theme="dark"] .json-key { + color: #82aaff; /* Brighter blue for keys */ +} + +[data-theme="dark"] .json-string { + color: #c3e88d; /* Brighter green for strings */ +} + +[data-theme="dark"] .json-number { + color: #f78c6c; /* Brighter orange for numbers */ +} + +[data-theme="dark"] .json-boolean { + color: #ffcb6b; /* Brighter yellow for booleans */ +} + +[data-theme="dark"] .json-null { + color: #d4d4d4; /* Neutral gray for null */ +} + +[data-theme="dark"] .form-select { + background-color: #2a2a2a; /* Match input background */ + color: var(--text-color); + border-color: var(--border-color); +} + +[data-theme="dark"] .form-select:focus { + background-color: #2a2a2a; /* Keep consistent on focus */ + color: var(--text-color); + border-color: var(--badge-primary); /* Highlight border on focus */ + box-shadow: 0 0 0 0.2rem rgba(74, 144, 226, 0.25); /* Subtle focus shadow */ +} + +[data-theme="dark"] .card-header.bg-light { + background-color: var(--header-bg); /* Match header background */ + color: var(--text-color); /* Ensure text is readable */ + border-bottom: 1px solid var(--border-color); /* Add subtle border */ +} + +[data-theme="dark"] .btn-outline-primary { + color: var(--badge-primary); /* Match primary badge color */ + border-color: var(--badge-primary); +} + +[data-theme="dark"] .btn-outline-primary:hover { + background-color: var(--badge-primary); + color: #ffffff; /* Ensure text is readable */ +} + body { background-color: var(--bg-color); color: var(--text-color); @@ -283,23 +344,23 @@ body { } [data-theme="dark"] .json-key { - color: #bb86fc; + color: #82aaff; /* Brighter blue for keys */ } [data-theme="dark"] .json-string { - color: #81c784; + color: #c3e88d; /* Brighter green for strings */ } [data-theme="dark"] .json-number { - color: #64b5f6; + color: #f78c6c; /* Brighter orange for numbers */ } [data-theme="dark"] .json-boolean { - color: #ffb74d; + color: #ffcb6b; /* Brighter yellow for booleans */ } [data-theme="dark"] .json-null { - color: #bdbdbd; + color: #d4d4d4; /* Neutral gray for null */ } /* Adjust table styles */ diff --git a/dashboard/js/script.js b/dashboard/js/script.js index 359e140..317e11c 100644 --- a/dashboard/js/script.js +++ b/dashboard/js/script.js @@ -699,8 +699,8 @@ function setupFilterHandlers() { // Define operators directly instead of loading them const availableOperators = [ - { id: 'contains', name: 'Contains', desc: 'Value contains the text (case-insensitive)' }, { id: 'exact', name: 'Equals', desc: 'Value exactly matches the text' }, + { id: 'contains', name: 'Contains', desc: 'Value contains the text (case-insensitive)' }, { id: 'starts_with', name: 'Starts with', desc: 'Value starts with the text' }, { id: 'ends_with', name: 'Ends with', desc: 'Value ends with the text' }, { id: 'greater_than', name: '>', desc: 'Value is greater than (numeric comparison)' }, @@ -754,7 +754,7 @@ function setupFilterHandlers() { // Update all operator dropdowns (now uses hardcoded values) function updateOperatorDropdowns() { document.querySelectorAll('.event-operator-select').forEach(select => { - const currentValue = select.value || 'contains'; + const currentValue = select.value || 'exact'; // Clear existing options select.innerHTML = ''; @@ -774,41 +774,40 @@ function setupFilterHandlers() { } // Create a new event filter row - function addEventFilterRow(keyValue = '', operator = 'contains', valueValue = '') { + function addEventFilterRow(keyValue = '', operator = 'exact', valueValue = '') { // Hide the "no filters" message noEventFiltersMsg.style.display = 'none'; - + // Clone the template const clone = document.importNode(eventFilterTemplate.content, true); const row = clone.querySelector('.event-filter-row'); - + + // Append to container first to ensure DOM is updated + eventFiltersContainer.appendChild(row); + + // Populate dropdowns after appending to the DOM + updateEventKeyDropdowns(); + updateOperatorDropdowns(); + // Set initial values if provided const keySelect = row.querySelector('.event-key-select'); const operatorSelect = row.querySelector('.event-operator-select'); const valueInput = row.querySelector('.event-value-input'); - - // Populate dropdowns - updateEventKeyDropdowns(); - updateOperatorDropdowns(); - - // Set values + if (keyValue) keySelect.value = keyValue; if (operator) operatorSelect.value = operator; if (valueValue) valueInput.value = valueValue; - + // Add remove handler row.querySelector('.remove-event-filter').addEventListener('click', function() { row.remove(); - + // Show message if no filters remain if (eventFiltersContainer.querySelectorAll('.event-filter-row').length === 0) { noEventFiltersMsg.style.display = 'block'; } }); - - // Append to container - eventFiltersContainer.appendChild(row); - + return row; } diff --git a/dashboard/views/dashboard.php b/dashboard/views/dashboard.php index 527031b..42b513a 100644 --- a/dashboard/views/dashboard.php +++ b/dashboard/views/dashboard.php @@ -188,8 +188,8 @@
-
- Event Data Filters +
+ Custom Event Filters @@ -364,6 +364,6 @@ - + \ No newline at end of file diff --git a/src/apm/migration/sqlite/0004-custom-events.sql b/src/apm/migration/sqlite/0004-custom-events.sql index 5a66143..78c454f 100644 --- a/src/apm/migration/sqlite/0004-custom-events.sql +++ b/src/apm/migration/sqlite/0004-custom-events.sql @@ -1,8 +1,5 @@ - --- Add a deprecated comment to the apm_custom_events.event_data column -ALTER TABLE apm_custom_events -CHANGE COLUMN event_data event_data TEXT COMMENT 'Deprecated: Use apm_custom_event_data instead.'; - +-- Note: SQLite doesn't support column comments or CHANGE COLUMN syntax +-- Keeping event_data column as is, but it will be considered deprecated -- Create a new table for custom event key-value data CREATE TABLE IF NOT EXISTS apm_custom_event_data ( @@ -22,10 +19,10 @@ CREATE INDEX IF NOT EXISTS idx_apm_custom_event_data_key ON apm_custom_event_dat -- Migrate existing event_data JSON into the new apm_custom_event_data table INSERT INTO apm_custom_event_data (custom_event_id, request_id, json_key, json_value) SELECT - id AS custom_event_id, - request_id, - json_each.json_key AS json_key, - json_each.json_value AS json_value + apm_custom_events.id AS custom_event_id, + apm_custom_events.request_id, + json_each.key AS json_key, + json_each.value AS json_value FROM apm_custom_events, json_each(apm_custom_events.event_data); diff --git a/src/apm/presenter/SqlitePresenter.php b/src/apm/presenter/SqlitePresenter.php index 3fddec1..dceadf8 100644 --- a/src/apm/presenter/SqlitePresenter.php +++ b/src/apm/presenter/SqlitePresenter.php @@ -3,6 +3,7 @@ namespace flight\apm\presenter; use PDO; +use Throwable; class SqlitePresenter implements PresenterInterface { @@ -266,123 +267,180 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri // Get request IDs from custom events if custom event type filter is set $customEventRequestIds = []; if (!empty($customEventType)) { - // Try to use JSON functions if available - if ($hasJsonFunctions) { - // SQLite JSON functions - $customEventsQuery = "SELECT request_id FROM apm_custom_events - WHERE event_type LIKE ? - LIMIT ?"; - $stmt = $this->db->prepare($customEventsQuery); - $stmt->execute(["%$customEventType%", $maxRequests]); - } else { - // Fallback: Search only in event_type - $customEventsQuery = "SELECT request_id FROM apm_custom_events WHERE event_type LIKE ? LIMIT ?"; - $stmt = $this->db->prepare($customEventsQuery); - $stmt->execute(["%$customEventType%", $maxRequests]); - } + // Search only in event_type + $customEventsQuery = "SELECT request_id FROM apm_custom_events WHERE event_type LIKE ? LIMIT ?"; + $stmt = $this->db->prepare($customEventsQuery); + $stmt->execute(["%$customEventType%", $maxRequests]); $customEventRequestIds = $stmt->fetchAll(PDO::FETCH_COLUMN); + + // If no matching custom events, we should return empty results + // This ensures that when a custom event type filter is specified but no matches are found, + // the final result set will be empty + if (empty($customEventRequestIds) && !empty($customEventType)) { + return [ + 'requests' => [], + 'pagination' => [ + 'currentPage' => $page, + 'totalPages' => 0, + 'perPage' => $perPage, + 'totalRequests' => 0, + ], + 'responseCodeDistribution' => [] + ]; + } } // Get request IDs from custom event data if key/value filters are set $customEventDataRequestIds = []; if (!empty($eventKeys)) { - // Build complex query for multiple key/value pairs - $keyValueConditions = []; - $keyValueParams = []; - $keyValueGroups = []; + // Use a simpler approach that's more compatible with SQLite + $validFilters = array_filter($eventKeys, fn($f) => !empty($f['key']) || !empty($f['value'])); - // Group number to match pairs of key/value conditions - $groupNum = 0; - - foreach ($eventKeys as $filter) { - $key = $filter['key'] ?? ''; - $operator = $filter['operator'] ?? 'contains'; - $value = $filter['value'] ?? ''; - - if (empty($key) && empty($value)) { - continue; - } + if (!empty($validFilters)) { + // Process each filter separately to build individual queries + $matchingRequestIdSets = []; - $groupConditions = []; - - // Add key condition if provided - if (!empty($key)) { - $groupConditions[] = "json_key = ?"; - $keyValueParams[] = $key; + foreach ($validFilters as $filter) { + $key = $filter['key'] ?? ''; + $operator = $filter['operator'] ?? 'exact'; + $value = $filter['value'] ?? ''; + + if (empty($key) && empty($value)) { + continue; + } + + // Build a simpler query for this specific filter + $filterParams = []; + $filterConditions = []; + + if (!empty($key)) { + $filterConditions[] = "json_key = ?"; + $filterParams[] = $key; + } + + if (!empty($value)) { + switch ($operator) { + case 'exact': + $filterConditions[] = "json_value = ?"; + $filterParams[] = $value; + break; + case 'contains': + $filterConditions[] = "json_value LIKE ?"; + $filterParams[] = "%$value%"; + break; + case 'starts_with': + $filterConditions[] = "json_value LIKE ?"; + $filterParams[] = "$value%"; + break; + case 'ends_with': + $filterConditions[] = "json_value LIKE ?"; + $filterParams[] = "%$value"; + break; + // For numeric comparisons, use simple string comparison + // This is more reliable in SQLite without CAST + case 'greater_than': + $filterConditions[] = "json_value > ?"; + $filterParams[] = (string)$value; + break; + case 'less_than': + $filterConditions[] = "json_value < ?"; + $filterParams[] = (string)$value; + break; + case 'greater_than_equal': + $filterConditions[] = "json_value >= ?"; + $filterParams[] = (string)$value; + break; + case 'less_than_equal': + $filterConditions[] = "json_value <= ?"; + $filterParams[] = (string)$value; + break; + } + } + + // If we have conditions for this filter + if (!empty($filterConditions)) { + $filterQuery = "SELECT DISTINCT request_id FROM apm_custom_event_data WHERE " . + implode(" AND ", $filterConditions); + + try { + + $stmt = $this->db->prepare($filterQuery); + $stmt->execute($filterParams); + $matchingIds = $stmt->fetchAll(PDO::FETCH_COLUMN); + + if (!empty($matchingIds)) { + $matchingRequestIdSets[] = $matchingIds; + } + } catch (Throwable $e) { + error_log("Error in filter query: " . $e->getMessage()); + } + } } - // Add value condition if provided - if (!empty($value)) { - switch ($operator) { - case 'exact': - $groupConditions[] = "json_value = ?"; - $keyValueParams[] = $value; - break; - case 'contains': - $groupConditions[] = "json_value LIKE ?"; - $keyValueParams[] = "%$value%"; - break; - case 'starts_with': - $groupConditions[] = "json_value LIKE ?"; - $keyValueParams[] = "$value%"; - break; - case 'ends_with': - $groupConditions[] = "json_value LIKE ?"; - $keyValueParams[] = "%$value"; - break; - case 'greater_than': - $groupConditions[] = "CAST(json_value AS NUMERIC) > ?"; - $keyValueParams[] = $value; - break; - case 'less_than': - $groupConditions[] = "CAST(json_value AS NUMERIC) < ?"; - $keyValueParams[] = $value; - break; - case 'greater_than_equal': - $groupConditions[] = "CAST(json_value AS NUMERIC) >= ?"; - $keyValueParams[] = $value; - break; - case 'less_than_equal': - $groupConditions[] = "CAST(json_value AS NUMERIC) <= ?"; - $keyValueParams[] = $value; - break; + // If we have multiple filters, find intersections + if (count($matchingRequestIdSets) > 0) { + // Start with the first set + $customEventDataRequestIds = array_shift($matchingRequestIdSets); + + // Intersect with each additional set + foreach ($matchingRequestIdSets as $idSet) { + $customEventDataRequestIds = array_intersect($customEventDataRequestIds, $idSet); } } - - if (!empty($groupConditions)) { - $keyValueGroups[] = "(" . implode(" AND ", $groupConditions) . ")"; - $groupNum++; + } + } + + // Merge request IDs from different sources + // This logic needs to be updated to use intersections when appropriate + $uniqueRequestIds = []; + + // Determine which filters are active + $hasCustomEventFilters = !empty($customEventType) || !empty($eventKeys); + $hasMainFilters = !empty($url) || !empty($responseCode) || !empty($responseCodePrefix) || + $isBot !== '' || !empty($minTime) || !empty($ip) || !empty($host) || + !empty($sessionId) || !empty($userAgent) || !empty($requestId); + + // Logic for applying filters + if ($hasCustomEventFilters) { + // Start with empty set if we're filtering by custom events + $matchingIds = []; + + // Apply custom event type filter if present + if (!empty($customEventType) && !empty($customEventRequestIds)) { + $matchingIds = $customEventRequestIds; + } + // Apply custom event data filter if present + if (!empty($eventKeys) && !empty($customEventDataRequestIds)) { + if (!empty($matchingIds)) { + // Intersect with existing matches + $matchingIds = array_intersect($matchingIds, $customEventDataRequestIds); + } else { + // Use these as the starting set + $matchingIds = $customEventDataRequestIds; } } - if (!empty($keyValueGroups)) { - // For "AND" logic between pairs (all conditions must match) - $keyValueQuery = " - SELECT request_id - FROM apm_requests - WHERE request_id IN ( - SELECT request_id - FROM apm_custom_event_data - WHERE " . implode(" OR ", $keyValueGroups) . " - GROUP BY request_id - HAVING COUNT(DISTINCT json_key) >= ? - ) - LIMIT ? - "; - - // Add the count of distinct conditions to ensure all match - $keyValueParams[] = count(array_filter($eventKeys, fn($f) => !empty($f['key']) || !empty($f['value']))); - $keyValueParams[] = $maxRequests; - - $stmt = $this->db->prepare($keyValueQuery); - $stmt->execute($keyValueParams); - $customEventDataRequestIds = $stmt->fetchAll(PDO::FETCH_COLUMN); + // Apply main filters if present + if ($hasMainFilters) { + if (!empty($matchingIds)) { + // Intersect with main request IDs + $uniqueRequestIds = array_intersect($matchingIds, $mainRequestIds); + } else { + // This case shouldn't happen in practice + $uniqueRequestIds = []; + } + } else { + // No main filters, use custom event matches directly + $uniqueRequestIds = $matchingIds; } + } else { + // No custom event filters, use main request IDs directly + $uniqueRequestIds = $mainRequestIds; } - // Merge request IDs from different sources - $allRequestIds = array_merge($mainRequestIds, $customEventRequestIds, $customEventDataRequestIds); - $uniqueRequestIds = array_unique($allRequestIds); + // Make sure we have unique IDs + $uniqueRequestIds = array_unique($uniqueRequestIds); + // var_dump($uniqueRequestIds); // If we have no matching requests, return empty result if (empty($uniqueRequestIds)) { @@ -421,7 +479,7 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri // Get the actual request data $requestQuery = "SELECT request_id, timestamp, request_url, total_time, response_code, is_bot, ip, user_agent, host, session_id FROM apm_requests - WHERE request_id IN ($placeholders) ORDER BY id DESC"; + WHERE request_id IN ($placeholders) ORDER BY timestamp DESC"; $stmt = $this->db->prepare($requestQuery); $stmt->execute($paginatedRequestIds); $requests = $stmt->fetchAll(PDO::FETCH_ASSOC); From c34d7cdd92a5c179145a733bdfbebc850f807936 Mon Sep 17 00:00:00 2001 From: n0nag0n Date: Tue, 1 Apr 2025 20:56:06 -0600 Subject: [PATCH 5/5] Last few fixes for masking ips to work correctly --- src/apm/presenter/PresenterFactory.php | 2 +- src/apm/presenter/SqlitePresenter.php | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/apm/presenter/PresenterFactory.php b/src/apm/presenter/PresenterFactory.php index a52ed9c..bf17c43 100644 --- a/src/apm/presenter/PresenterFactory.php +++ b/src/apm/presenter/PresenterFactory.php @@ -22,7 +22,7 @@ public static function create(string $runwayConfigPath): PresenterInterface $storageType = $runwayConfig['apm']['storage_type']; switch($storageType) { case 'sqlite': - return new SqlitePresenter($runwayConfig['apm']['dest_db_dsn']); + return new SqlitePresenter($runwayConfig); default: throw new InvalidArgumentException("Unsupported storage type: $storageType"); } diff --git a/src/apm/presenter/SqlitePresenter.php b/src/apm/presenter/SqlitePresenter.php index dceadf8..f7c0b81 100644 --- a/src/apm/presenter/SqlitePresenter.php +++ b/src/apm/presenter/SqlitePresenter.php @@ -12,14 +12,20 @@ class SqlitePresenter implements PresenterInterface */ protected PDO $db; + /** + * Runway Config + */ + protected array $config; + /** * Constructor * - * @param string $dsn PDO connection dsn + * @param array $config Runway Config */ - public function __construct(string $dsn) + public function __construct(array $config) { - $this->db = new PDO($dsn, null, null, [ + $this->config = $config; + $this->db = new PDO($config['apm']['dest_db_dsn'], null, null, [ PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_EMULATE_PREPARES => false @@ -483,6 +489,8 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri $stmt = $this->db->prepare($requestQuery); $stmt->execute($paginatedRequestIds); $requests = $stmt->fetchAll(PDO::FETCH_ASSOC); + + $shouldMaskIp = $this->shouldMaskIpAddresses(); // Fetch details for each request foreach ($requests as &$request) { @@ -490,7 +498,7 @@ public function getRequestsData(string $threshold, int $page, int $perPage, stri $request = array_merge($request, $details); // Mask IP address if the option is enabled - if ($this->shouldMaskIpAddresses()) { + if ($shouldMaskIp === true) { $request['ip'] = $this->maskIpAddress($request['ip']); } }