From 43362a97366c7d03bf21b8e94d08659312bd986b Mon Sep 17 00:00:00 2001 From: Sony Surahmn Date: Tue, 16 Dec 2025 23:39:13 +0700 Subject: [PATCH] Add duration recalculation feature and improve device info handling in CrawlingService - Introduced a new cron job for daily duration recalculation of door logs. - Enhanced device info retrieval to include device name and other details. - Updated logging to provide clearer information on device processing and duration recalculation. - Implemented duplicate event checks before sending events to prevent redundancy. --- src/index.ts | 139 +++++++--- src/services/crawling.ts | 566 ++++++++++++++++++++++++++++----------- 2 files changed, 523 insertions(+), 182 deletions(-) diff --git a/src/index.ts b/src/index.ts index 18b669a..db1bfb1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ const log: Logger = new Logger({ log.settings.prettyLogTimeZone = "local"; let isCrawlingRunning = false; +let isRecalculatingDuration = false; /** * Start the crawling service @@ -42,42 +43,121 @@ async function startCrawlingService() { // Schedule cron job with timezone support // node-cron v3+ supports timezone option - const crawlingJob = cron.schedule( - cronInterval, + // const crawlingJob = cron.schedule( + // cronInterval, + // async () => { + // const scheduleTime = moment() + // .tz("Asia/Dili") + // .format("YYYY-MM-DD HH:mm:ss"); + // const utcTime = new Date().toISOString(); + + // if (isCrawlingRunning) { + // log.warn( + // `[Cron] ⚠️ Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})` + // ); + // return; + // } + + // log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`); + // log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`); + // isCrawlingRunning = true; + + // try { + // await CrawlingService.runCrawling(); + // const completeTime = moment() + // .tz("Asia/Dili") + // .format("YYYY-MM-DD HH:mm:ss"); + // log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`); + // } catch (error) { + // const errorTime = moment() + // .tz("Asia/Dili") + // .format("YYYY-MM-DD HH:mm:ss"); + // log.error( + // `[Cron] ❌ Crawling service failed at ${errorTime}:`, + // error + // ); + // } finally { + // isCrawlingRunning = false; + // log.info(`[Cron] ==========================================`); + // } + // }, + // { + // timezone: "Asia/Dili", + // } + // ); + + // // Verify cron job is scheduled + // if (crawlingJob) { + // const nextRun = moment() + // .tz("Asia/Dili") + // .add(1, "minute") + // .startOf("minute") + // .format("YYYY-MM-DD HH:mm:ss"); + // const currentTime = moment() + // .tz("Asia/Dili") + // .format("YYYY-MM-DD HH:mm:ss"); + + // log.info("═══════════════════════════════════════════════════════════"); + // log.info(`[Cron] ✅ Cron job scheduled successfully`); + // log.info( + // `[Cron] App: ${config.get("app.name")} v${config.get("app.version")}` + // ); + // log.info(`[Cron] Environment: ${config.get("app.env")}`); + // log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`); + // log.info(`[Cron] Timezone: Asia/Dili`); + // log.info(`[Cron] Current time: ${currentTime}`); + // log.info(`[Cron] Next run scheduled at: ${nextRun}`); + // log.info("═══════════════════════════════════════════════════════════"); + // } else { + // log.error("[Cron] ❌ Failed to schedule cron job"); + // throw new Error("Failed to schedule cron job"); + // } + + // Schedule daily cron job for duration recalculation (runs at 00:00 daily) + const durationRecalcJob = cron.schedule( + "* * * * *", // Daily at midnight async () => { const scheduleTime = moment() .tz("Asia/Dili") .format("YYYY-MM-DD HH:mm:ss"); const utcTime = new Date().toISOString(); - if (isCrawlingRunning) { + if (isRecalculatingDuration) { log.warn( - `[Cron] ⚠️ Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})` + `[Duration Recalc Cron] ⚠️ Duration recalculation is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})` ); return; } - log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`); - log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`); - isCrawlingRunning = true; + log.info( + `[Duration Recalc Cron] ⏰ ========== DURATION RECALCULATION CRON TRIGGERED ==========` + ); + log.info( + `[Duration Recalc Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})` + ); + isRecalculatingDuration = true; try { - await CrawlingService.runCrawling(); + await CrawlingService.recalculateDuration(); const completeTime = moment() .tz("Asia/Dili") .format("YYYY-MM-DD HH:mm:ss"); - log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`); + log.info( + `[Duration Recalc Cron] ✅ Duration recalculation completed at ${completeTime}` + ); } catch (error) { const errorTime = moment() .tz("Asia/Dili") .format("YYYY-MM-DD HH:mm:ss"); log.error( - `[Cron] ❌ Crawling service failed at ${errorTime}:`, + `[Duration Recalc Cron] ❌ Duration recalculation failed at ${errorTime}:`, error ); } finally { - isCrawlingRunning = false; - log.info(`[Cron] ==========================================`); + isRecalculatingDuration = false; + log.info( + `[Duration Recalc Cron] ==========================================` + ); } }, { @@ -85,31 +165,30 @@ async function startCrawlingService() { } ); - // Verify cron job is scheduled - if (crawlingJob) { - const nextRun = moment() + // Verify duration recalculation cron job is scheduled + if (durationRecalcJob) { + const nextRecalcRun = moment() .tz("Asia/Dili") - .add(1, "minute") - .startOf("minute") + .add(1, "day") + .startOf("day") .format("YYYY-MM-DD HH:mm:ss"); - const currentTime = moment() - .tz("Asia/Dili") - .format("YYYY-MM-DD HH:mm:ss"); - log.info("═══════════════════════════════════════════════════════════"); - log.info(`[Cron] ✅ Cron job scheduled successfully`); log.info( - `[Cron] App: ${config.get("app.name")} v${config.get("app.version")}` + `[Duration Recalc Cron] ✅ Daily duration recalculation cron job scheduled successfully` + ); + log.info( + `[Duration Recalc Cron] Schedule pattern: 0 0 * * * (daily at midnight)` + ); + log.info(`[Duration Recalc Cron] Timezone: Asia/Dili`); + log.info( + `[Duration Recalc Cron] Next run scheduled at: ${nextRecalcRun}` ); - log.info(`[Cron] Environment: ${config.get("app.env")}`); - log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`); - log.info(`[Cron] Timezone: Asia/Dili`); - log.info(`[Cron] Current time: ${currentTime}`); - log.info(`[Cron] Next run scheduled at: ${nextRun}`); log.info("═══════════════════════════════════════════════════════════"); } else { - log.error("[Cron] ❌ Failed to schedule cron job"); - throw new Error("Failed to schedule cron job"); + log.error( + "[Duration Recalc Cron] ❌ Failed to schedule duration recalculation cron job" + ); + // Don't throw error - this is not critical for the main service } // Run immediately on startup to verify everything works diff --git a/src/services/crawling.ts b/src/services/crawling.ts index 5e86d51..456dab1 100644 --- a/src/services/crawling.ts +++ b/src/services/crawling.ts @@ -49,11 +49,18 @@ interface HikvisionResponse { }; } +interface DeviceSystemInfo { + macAddress: string; + deviceName?: string; + deviceID?: string; + model?: string; +} + interface CrawlingState { [deviceIp: string]: { - searchID: string; - lastPosition: number; macAddress?: string; // Cache MAC address per device + deviceName?: string; // Cache device name from device info + deviceInfo?: DeviceSystemInfo; // Cache full device info }; } @@ -669,30 +676,30 @@ export class CrawlingService { } /** - * Get device info including MAC address + * Get device info including MAC address and device name from device system info */ static async getDeviceInfo( deviceIp: string, username: string, password: string - ): Promise { + ): Promise { try { - // Check cache first - but only if it's a valid MAC address format - if (crawlingState[deviceIp]?.macAddress) { - const cachedMac = crawlingState[deviceIp].macAddress!; - // Validate MAC address format (should be XX:XX:XX:XX:XX:XX) + // Check cache first - return cached device info if available + if (crawlingState[deviceIp]?.deviceInfo) { + const cachedInfo = crawlingState[deviceIp].deviceInfo!; const macRegex = /^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$/; - if (macRegex.test(cachedMac)) { - console.log( - `[CrawlingService] Using cached MAC address for ${deviceIp}: ${cachedMac}` + if (macRegex.test(cachedInfo.macAddress)) { + log.info( + `[Get Device Info] Using cached device info for ${deviceIp} - MAC: ${ + cachedInfo.macAddress + }, DeviceName: ${cachedInfo.deviceName || "N/A"}` ); - return cachedMac; + return cachedInfo; } else { - console.warn( - `[CrawlingService] Cached MAC address "${cachedMac}" for ${deviceIp} is invalid format. Fetching fresh.` + log.warn( + `[Get Device Info] Cached MAC address "${cachedInfo.macAddress}" for ${deviceIp} is invalid format. Fetching fresh.` ); - // Clear invalid cache - delete crawlingState[deviceIp].macAddress; + delete crawlingState[deviceIp].deviceInfo; } } @@ -710,109 +717,107 @@ export class CrawlingService { // Handle both XML and JSON responses let macAddress = ""; + let deviceName = ""; + let deviceID = ""; + let model = ""; - // Debug: Log the exact structure - console.log( - `[CrawlingService] Response type: ${typeof response}, has DeviceInfo: ${!!( + // Helper function to extract value from device info + const extractValue = (obj: any, key: string): string => { + if (!obj || !obj[key]) return ""; + if (typeof obj[key] === "string") return obj[key].trim(); + if (Array.isArray(obj[key])) return obj[key][0]?.trim() || ""; + if (typeof obj[key] === "object") { + return obj[key]._ || obj[key]["#text"] || ""; + } + return ""; + }; + + log.info( + `[Get Device Info] Response type: ${typeof response}, has DeviceInfo: ${!!( response && response.DeviceInfo )}` ); if (response && response.DeviceInfo) { - // XML response structure: { DeviceInfo: { macAddress: "..." } } - // xml2js with explicitArray: false should give us a string directly const deviceInfo = response.DeviceInfo; - // Try multiple ways to extract MAC address - if (typeof deviceInfo.macAddress === "string") { - macAddress = deviceInfo.macAddress; - } else if (Array.isArray(deviceInfo.macAddress)) { - macAddress = deviceInfo.macAddress[0] || ""; - } else if ( - deviceInfo.macAddress && - typeof deviceInfo.macAddress === "object" - ) { - // Sometimes xml2js wraps it in an object - macAddress = - deviceInfo.macAddress._ || deviceInfo.macAddress["#text"] || ""; - } else { - macAddress = ""; - } + // Extract MAC address + macAddress = extractValue(deviceInfo, "macAddress"); - // Trim whitespace - macAddress = macAddress.trim(); + // Extract device name (this is what we need!) + deviceName = extractValue(deviceInfo, "deviceName"); - console.log( - `[CrawlingService] Extracted MAC address from DeviceInfo: "${macAddress}" (type: ${typeof deviceInfo.macAddress}, isArray: ${Array.isArray( - deviceInfo.macAddress - )})` + // Extract device ID + deviceID = extractValue(deviceInfo, "deviceID"); + + // Extract model + model = extractValue(deviceInfo, "model"); + + log.info( + `[Get Device Info] Extracted from DeviceInfo - MAC: "${macAddress}", DeviceName: "${deviceName}", DeviceID: "${deviceID}", Model: "${model}"` ); // If still empty, try to extract from raw response string - if (!macAddress && typeof response === "string") { - const macMatch = response.match( - /]*>([^<]+)<\/macAddress>/i - ); - if (macMatch && macMatch[1]) { - macAddress = macMatch[1].trim(); - console.log( - `[CrawlingService] Extracted MAC address from raw XML string: "${macAddress}"` + if (typeof response === "string") { + if (!macAddress) { + const macMatch = response.match( + /]*>([^<]+)<\/macAddress>/i ); + if (macMatch && macMatch[1]) macAddress = macMatch[1].trim(); + } + if (!deviceName) { + const nameMatch = response.match( + /]*>([^<]+)<\/deviceName>/i + ); + if (nameMatch && nameMatch[1]) deviceName = nameMatch[1].trim(); } } - } else if (response && response.macAddress) { - // Direct property (unlikely but handle it) - macAddress = - typeof response.macAddress === "string" - ? response.macAddress.trim() - : ""; - console.log( - `[CrawlingService] Extracted MAC address from direct property: "${macAddress}"` - ); - } else { - console.warn( - `[CrawlingService] Could not find DeviceInfo or macAddress in response for ${deviceIp}. Response keys: ${ - response ? Object.keys(response).join(", ") : "null" - }` - ); - - // Last resort: try to extract from stringified response - const responseStr = JSON.stringify(response); - const macMatch = responseStr.match(/"macAddress"\s*:\s*"([^"]+)"/i); - if (macMatch && macMatch[1]) { - macAddress = macMatch[1].trim(); - console.log( - `[CrawlingService] Extracted MAC address from JSON string: "${macAddress}"` - ); - } + } else if (response) { + // Direct properties (JSON response) + macAddress = extractValue(response, "macAddress"); + deviceName = extractValue(response, "deviceName"); + deviceID = extractValue(response, "deviceID"); + model = extractValue(response, "model"); } + // Build device info object + const deviceSystemInfo: DeviceSystemInfo = { + macAddress: macAddress || "", + deviceName: deviceName || undefined, + deviceID: deviceID || undefined, + model: model || undefined, + }; + if (macAddress) { - // Cache it + // Cache full device info if (!crawlingState[deviceIp]) { - crawlingState[deviceIp] = { - searchID: uuidv4(), - lastPosition: 0, - }; + crawlingState[deviceIp] = {}; } + crawlingState[deviceIp].deviceInfo = deviceSystemInfo; crawlingState[deviceIp].macAddress = macAddress; - console.log( - `[CrawlingService] MAC address for ${deviceIp}: ${macAddress}` + if (deviceName) { + crawlingState[deviceIp].deviceName = deviceName; + } + + log.info( + `[Get Device Info] Cached device info for ${deviceIp} - MAC: ${macAddress}, DeviceName: ${ + deviceName || "N/A" + }` ); - return macAddress; + return deviceSystemInfo; } - console.warn( - `[CrawlingService] MAC address is empty for device ${deviceIp}. Response structure:`, + log.warn( + `[Get Device Info] MAC address is empty for device ${deviceIp}. Response structure:`, JSON.stringify(response, null, 2) ); - return ""; + return { macAddress: "" }; } catch (error) { - console.error( - `[CrawlingService] Error fetching device info for ${deviceIp}:`, + log.error( + `[Get Device Info] Error fetching device info for ${deviceIp}:`, error ); - return ""; + return { macAddress: "" }; } } @@ -852,29 +857,49 @@ export class CrawlingService { } } + /** + * Determine deviceName from ACS Controller Event major/minor event types + * Based on Hikvision ACS Event specifications + */ + /** + * Get deviceName from device system info (cached from device API) + */ + static getDeviceNameFromDeviceInfo(deviceIp: string): string { + try { + const cachedDeviceName = crawlingState[deviceIp]?.deviceName; + + if (cachedDeviceName) { + log.info( + `[Map Event] DeviceName from device info: ${cachedDeviceName} for IP: ${deviceIp}` + ); + return cachedDeviceName; + } + + // Fallback: Default to DeviceIn if not available from device + log.warn( + `[Map Event] DeviceName not available from device info for IP: ${deviceIp}. Defaulting to DeviceIn.` + ); + return "DeviceIn"; + } catch (error) { + log.error( + `[Map Event] Error getting deviceName from device info for IP: ${deviceIp}:`, + error + ); + return "DeviceIn"; + } + } + /** * Map Hikvision event to webhook format */ static async mapHikvisionToWebhook( event: HikvisionEvent, deviceIp: string, - macAddress: string + deviceInfo: DeviceSystemInfo ): Promise { - // Map attendanceStatus to deviceName - // "breakIn", "checkIn" -> "DeviceIn" - // "breakOut", "checkOut" -> "DeviceOut" - let deviceName = "DeviceIn"; - if ( - event.attendanceStatus === "breakOut" || - event.attendanceStatus === "checkOut" - ) { - deviceName = "DeviceOut"; - } else if ( - event.attendanceStatus === "breakIn" || - event.attendanceStatus === "checkIn" - ) { - deviceName = "DeviceIn"; - } + // Get deviceName from device system info (from device API) + const deviceName = this.getDeviceNameFromDeviceInfo(deviceIp); + const macAddress = deviceInfo.macAddress || ""; // Get employee accessdoorid const employeeAccessDoorId = await this.getEmployeeAccessDoorId( @@ -933,12 +958,9 @@ export class CrawlingService { const port = config.get("hikvision.defaultPort"); try { - // Initialize or get existing search state + // Initialize or get existing state (only for MAC address caching) if (!crawlingState[deviceIp]) { - crawlingState[deviceIp] = { - searchID: uuidv4(), - lastPosition: 0, - }; + crawlingState[deviceIp] = {}; } const state = crawlingState[deviceIp]; @@ -950,40 +972,46 @@ export class CrawlingService { `[CrawlingService] Getting MAC address for device ${deviceIp}...` ); - const macStartTime = Date.now(); - const macAddress = await this.getDeviceInfo(deviceIp, username, password); - const macDuration = Date.now() - macStartTime; + const deviceInfoStartTime = Date.now(); + const deviceInfo = await this.getDeviceInfo(deviceIp, username, password); + const deviceInfoDuration = Date.now() - deviceInfoStartTime; - if (!macAddress) { + if (!deviceInfo.macAddress) { log.warn( - `[Crawl Device] ⚠️ Could not get MAC address for device ${deviceIp} (took ${macDuration}ms). Events will be saved without MAC address.` + `[Crawl Device] ⚠️ Could not get device info for device ${deviceIp} (took ${deviceInfoDuration}ms). Events will be saved without MAC address.` ); console.warn( - `[CrawlingService] Warning: Could not get MAC address for device ${deviceIp}. Events will be saved without MAC address.` + `[CrawlingService] Warning: Could not get device info for device ${deviceIp}. Events will be saved without MAC address.` ); } else { log.info( - `[Crawl Device] ✓ MAC address retrieved: ${macAddress} for device ${deviceIp} (took ${macDuration}ms)` + `[Crawl Device] ✓ Device info retrieved for ${deviceIp} (took ${deviceInfoDuration}ms) - MAC: ${ + deviceInfo.macAddress + }, DeviceName: ${deviceInfo.deviceName || "N/A"}` ); console.log( - `[CrawlingService] Successfully retrieved MAC address ${macAddress} for device ${deviceIp}` + `[CrawlingService] Successfully retrieved device info for ${deviceIp} - MAC: ${ + deviceInfo.macAddress + }, DeviceName: ${deviceInfo.deviceName || "N/A"}` ); } - // Prepare request payload + // Prepare request payload for REAL-TIME events only (no batching) + // Always fetch from position 0 to get only the latest events + // Since we run every minute, we'll only get new events const requestPayload = { AcsEventCond: { - searchID: state.searchID, - searchResultPosition: state.lastPosition, - maxResults: 10, // Fetch multiple events at once + searchID: uuidv4(), // New search ID each time for real-time + searchResultPosition: 0, // Always start from position 0 (latest events) + maxResults: config.get("crawler.maxResults") || 10, major: 5, minor: 75, - timeReverseOrder: true, + timeReverseOrder: true, // Get newest events first }, }; log.info( - `[Crawl Device] Starting crawl for device ${deviceIp} from position ${state.lastPosition} (SearchID: ${state.searchID})` + `[Crawl Device] Starting REAL-TIME crawl for device ${deviceIp} - Fetching latest ${requestPayload.AcsEventCond.maxResults} events (no batching)` ); // Make API call with digest auth @@ -1023,12 +1051,16 @@ export class CrawlingService { log.info( `[Crawl Device] Processing ${ events.length - } event(s) from device ${deviceIp} (MAC: ${macAddress || "NONE"})` + } event(s) from device ${deviceIp} (MAC: ${ + deviceInfo.macAddress || "NONE" + }, DeviceName: ${deviceInfo.deviceName || "N/A"})` ); console.log( `[CrawlingService] Processing ${ events.length - } events for device ${deviceIp} with MAC: ${macAddress || "NONE"}` + } events for device ${deviceIp} with MAC: ${ + deviceInfo.macAddress || "NONE" + }, DeviceName: ${deviceInfo.deviceName || "N/A"}` ); let successCount = 0; @@ -1056,7 +1088,7 @@ export class CrawlingService { const webhookEvent = await this.mapHikvisionToWebhook( event, deviceIp, - macAddress + deviceInfo ); log.info( @@ -1099,29 +1131,14 @@ export class CrawlingService { } log.info( - `[Crawl Device] Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}` + `[Crawl Device] ✅ Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}` ); - // Update position for next crawl - state.lastPosition += events.length; - + // Real-time mode: No position tracking, just process new events + // Each crawl fetches only events from the last minute log.info( - `[Crawl Device] Updated crawl position for device ${deviceIp} - New position: ${state.lastPosition}, Total matches: ${response.AcsEvent.totalMatches}, Status: ${response.AcsEvent.responseStatusStrg}` + `[Crawl Device] Real-time crawl completed for device ${deviceIp} - Processed ${events.length} new event(s)` ); - - // If there are more matches, we'll continue in next crawl - if (response.AcsEvent.responseStatusStrg === "MORE") { - log.info( - `[Crawl Device] ⏭️ More events available for device ${deviceIp} - Total: ${response.AcsEvent.totalMatches}, Current position: ${state.lastPosition}` - ); - } else { - // Reset position if no more events - log.info( - `[Crawl Device] ✅ All events processed for device ${deviceIp} - Resetting position to 0` - ); - state.lastPosition = 0; - state.searchID = uuidv4(); - } } catch (error: any) { const errorMsg = error?.message || String(error); log.error( @@ -1134,14 +1151,90 @@ export class CrawlingService { }, Data: ${JSON.stringify(error.response.data)}` ); } - // Reset state on error - if (crawlingState[deviceIp]) { - log.info( - `[Crawl Device] Resetting crawl state for device ${deviceIp} due to error` + // No state to reset in real-time mode (only MAC address is cached) + log.info( + `[Crawl Device] Error occurred but continuing for next crawl cycle` + ); + } + } + + /** + * Check if event already exists in database (duplicate check) + * Checks based on: accessdoorid, datelogs, ipaddress, macaddress + */ + static async isDuplicateEvent(eventData: any): Promise { + try { + const accessdoorid = parseInt( + eventData.AccessControllerEvent?.employeeNoString || "0" + ); + const datelogs = moment.parseZone(eventData.dateTime).toDate(); + const ipaddress = eventData.ipAddress || ""; + const macaddress = eventData.macAddress || ""; + + if (!accessdoorid || accessdoorid === 0) { + log.warn( + `[Duplicate Check] ⚠️ Invalid accessdoorid: ${eventData.AccessControllerEvent?.employeeNoString}, skipping duplicate check` ); - crawlingState[deviceIp].lastPosition = 0; - crawlingState[deviceIp].searchID = uuidv4(); + return false; } + + if (!ipaddress || !macaddress) { + log.warn( + `[Duplicate Check] ⚠️ Missing IP or MAC address - IP: ${ipaddress}, MAC: ${macaddress}, skipping duplicate check` + ); + return false; + } + + // Check if a record with the same accessdoorid, datelogs (exact match or within 1 second), ipaddress, and macaddress exists + // Use exact timestamp match first, then 1 second tolerance as fallback + const datelogsStr = moment(datelogs).format("YYYY-MM-DD HH:mm:ss"); + + log.info( + `[Duplicate Check] Checking for duplicate - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}` + ); + + const existing = await OrmHelper.DB.query( + ` + SELECT _idx, datelogs, accessdoorid, ipaddress, macaddress + FROM tbl_attendancedoorlogs + WHERE accessdoorid = ? + AND ipaddress = ? + AND macaddress = ? + AND ( + DATE_FORMAT(datelogs, '%Y-%m-%d %H:%i:%s') = ? + OR ABS(TIMESTAMPDIFF(SECOND, datelogs, ?)) <= 1 + ) + AND isdeleted = 0 + LIMIT 1 + `, + [accessdoorid, ipaddress, macaddress, datelogsStr, datelogs] + ); + + const isDuplicate = existing && existing.length > 0; + + if (isDuplicate) { + const existingRecord = existing[0]; + log.info( + `[Duplicate Check] ✓ DUPLICATE FOUND - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}, Existing _idx: ${ + existingRecord._idx + }, Existing DateTime: ${moment(existingRecord.datelogs).format( + "YYYY-MM-DD HH:mm:ss" + )}` + ); + } else { + log.info( + `[Duplicate Check] ✓ NEW EVENT - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}` + ); + } + + return isDuplicate; + } catch (error: any) { + const errorMsg = error?.message || String(error); + log.error( + `[Duplicate Check] ✗ ERROR checking duplicate event - EmployeeID: ${eventData.AccessControllerEvent?.employeeNoString}, DateTime: ${eventData.dateTime}, Error: ${errorMsg}` + ); + // If there's an error checking, assume it's not a duplicate to avoid blocking valid events + return false; } } @@ -1161,6 +1254,15 @@ export class CrawlingService { `[Send Event] Preparing to send event - EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}, IP: ${ipAddress}, MAC: ${macAddress}` ); + // Check for duplicates before sending (FIRST CHECK) + const isDuplicate = await this.isDuplicateEvent(eventData); + if (isDuplicate) { + log.info( + `[Send Event] ⏭️ SKIPPED (Duplicate - First Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}` + ); + return; + } + const bridgeHost = config.get("bridge.host"); const bridgePort = config.get("bridge.port"); const bridgeEndpoint = config.get("bridge.endpoint"); @@ -1172,6 +1274,15 @@ export class CrawlingService { `[Send Event] Sending to bridge - URL: ${url}, EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}` ); + // Double-check for duplicates right before sending (SECOND CHECK - prevents race conditions) + const isDuplicateFinal = await this.isDuplicateEvent(eventData); + if (isDuplicateFinal) { + log.info( + `[Send Event] ⏭️ SKIPPED (Duplicate - Final Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}` + ); + return; + } + // Send as form data with event_log field (matching the webhook format) const startTime = Date.now(); await axios.post( @@ -1344,4 +1455,155 @@ export class CrawlingService { throw error; // Re-throw so the caller can handle it } } + + /** + * Recalculate duration for door logs from devices with iscrawling=1 and flag=1 + * For each DeviceIn record, find the previous DeviceOut for the same accessdoorid on the same day + */ + static async recalculateDuration(): Promise { + const startTime = Date.now(); + log.info("═══════════════════════════════════════════════════════════"); + log.info("[Recalculate Duration] ⏰ Starting duration recalculation..."); + + try { + // Get all devices with iscrawling=1 and flag=1 + const devices = await OrmHelper.DB.query(` + SELECT + _idx, + deviceid, + deviceip, + flag, + iscrawling + FROM tbl_deviceinfo + WHERE (flag = 1 AND iscrawling = 1) + AND isdeleted = 0 + AND deviceip IS NOT NULL + `); + + if (!devices || devices.length === 0) { + log.info( + "[Recalculate Duration] ℹ️ No devices found with iscrawling=1 and flag=1" + ); + return; + } + + log.info( + `[Recalculate Duration] Found ${devices.length} device(s) to process` + ); + + // Get all DeviceIn records from these devices using JOIN for better performance and security + // Fix collation mismatch by using COLLATE in JOIN condition + const deviceInLogs = await OrmHelper.DB.query( + ` + SELECT + d._idx, + d.accessdoorid, + d.datelogs, + d.devicename, + d.ipaddress + FROM tbl_attendancedoorlogs d + INNER JOIN tbl_deviceinfo di ON d.ipaddress COLLATE utf8mb4_unicode_ci = di.deviceip COLLATE utf8mb4_unicode_ci + WHERE di.flag = 1 + AND di.iscrawling = 1 + AND di.isdeleted = 0 + AND d.devicename = 'DeviceIn' + AND d.isdeleted = 0 + AND DATE(d.datelogs) >= DATE_SUB(CURDATE(), INTERVAL 1 DAY) + ORDER BY d.accessdoorid, d.datelogs ASC + ` + ); + + if (!deviceInLogs || deviceInLogs.length === 0) { + log.info( + "[Recalculate Duration] ℹ️ No DeviceIn records found for these devices" + ); + return; + } + + log.info( + `[Recalculate Duration] Found ${deviceInLogs.length} DeviceIn record(s) to process` + ); + + let totalUpdated = 0; + let totalSkipped = 0; + let totalErrors = 0; + + // Process each DeviceIn record + for (const deviceInLog of deviceInLogs) { + try { + const accessdoorid = deviceInLog.accessdoorid; + const deviceInDate = moment(deviceInLog.datelogs); + const logDate = deviceInDate.format("YYYY-MM-DD"); + + // Find the previous DeviceOut for the same accessdoorid on the same day + const deviceOutLog = await OrmHelper.DB.query(` + SELECT + _idx, + datelogs + FROM tbl_attendancedoorlogs + WHERE accessdoorid = ? + AND devicename = 'DeviceOut' + AND DATE(datelogs) = ? + AND datelogs < ? + AND isdeleted = 0 + ORDER BY datelogs DESC + LIMIT 1 + `, [accessdoorid, logDate, deviceInLog.datelogs]); + + if (!deviceOutLog || deviceOutLog.length === 0) { + totalSkipped++; + continue; + } + + const deviceOutDate = moment(deviceOutLog[0].datelogs); + const duration = moment.duration(deviceInDate.diff(deviceOutDate)); + const formattedDuration = moment + .utc(duration.asMilliseconds()) + .format("HH:mm:ss"); + + // Update the duration + await OrmHelper.DB.query(` + UPDATE tbl_attendancedoorlogs + SET duration = ?, + uby = 'svc-hcm-crawler', + udt = NOW() + WHERE _idx = ? + `, [formattedDuration, deviceInLog._idx]); + + totalUpdated++; + + if (totalUpdated % 100 === 0) { + log.info( + `[Recalculate Duration] Progress: ${totalUpdated} updated, ${totalSkipped} skipped` + ); + } + } catch (error: any) { + totalErrors++; + log.error( + `[Recalculate Duration] Error processing log _idx=${deviceInLog._idx}:`, + error.message + ); + } + } + + const totalDuration = Date.now() - startTime; + log.info("═══════════════════════════════════════════════════════════"); + log.info( + `[Recalculate Duration] ✅ Completed in ${totalDuration}ms` + ); + log.info( + `[Recalculate Duration] Summary: ${totalUpdated} updated, ${totalSkipped} skipped, ${totalErrors} errors` + ); + log.info("═══════════════════════════════════════════════════════════"); + } catch (error: any) { + const totalDuration = Date.now() - startTime; + const errorMsg = error?.message || String(error); + log.error("═══════════════════════════════════════════════════════════"); + log.error( + `[Recalculate Duration] ❌ Fatal error after ${totalDuration}ms: ${errorMsg}` + ); + log.error("═══════════════════════════════════════════════════════════"); + throw error; + } + } }