Add duration recalculation feature and improve device info handling in CrawlingService

- Introduced a new cron job for daily duration recalculation of door logs.
- Enhanced device info retrieval to include device name and other details.
- Updated logging to provide clearer information on device processing and duration recalculation.
- Implemented duplicate event checks before sending events to prevent redundancy.

src/index.ts

@@ -15,6 +15,7 @@ const log: Logger<ILogObj> = new Logger({
 log.settings.prettyLogTimeZone = "local";
 
 let isCrawlingRunning = false;
+let isRecalculatingDuration = false;
 
 /**
  * Start the crawling service
@@ -42,42 +43,121 @@ async function startCrawlingService() {
 
     // Schedule cron job with timezone support
     // node-cron v3+ supports timezone option
-    const crawlingJob = cron.schedule(
+    // const crawlingJob = cron.schedule(
-      cronInterval,
+    //   cronInterval,
+    //   async () => {
+    //     const scheduleTime = moment()
+    //       .tz("Asia/Dili")
+    //       .format("YYYY-MM-DD HH:mm:ss");
+    //     const utcTime = new Date().toISOString();
+
+    //     if (isCrawlingRunning) {
+    //       log.warn(
+    //         `[Cron] ⚠️  Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
+    //       );
+    //       return;
+    //     }
+
+    //     log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`);
+    //     log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`);
+    //     isCrawlingRunning = true;
+
+    //     try {
+    //       await CrawlingService.runCrawling();
+    //       const completeTime = moment()
+    //         .tz("Asia/Dili")
+    //         .format("YYYY-MM-DD HH:mm:ss");
+    //       log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`);
+    //     } catch (error) {
+    //       const errorTime = moment()
+    //         .tz("Asia/Dili")
+    //         .format("YYYY-MM-DD HH:mm:ss");
+    //       log.error(
+    //         `[Cron] ❌ Crawling service failed at ${errorTime}:`,
+    //         error
+    //       );
+    //     } finally {
+    //       isCrawlingRunning = false;
+    //       log.info(`[Cron] ==========================================`);
+    //     }
+    //   },
+    //   {
+    //     timezone: "Asia/Dili",
+    //   }
+    // );
+
+    // // Verify cron job is scheduled
+    // if (crawlingJob) {
+    //   const nextRun = moment()
+    //     .tz("Asia/Dili")
+    //     .add(1, "minute")
+    //     .startOf("minute")
+    //     .format("YYYY-MM-DD HH:mm:ss");
+    //   const currentTime = moment()
+    //     .tz("Asia/Dili")
+    //     .format("YYYY-MM-DD HH:mm:ss");
+
+    //   log.info("═══════════════════════════════════════════════════════════");
+    //   log.info(`[Cron] ✅ Cron job scheduled successfully`);
+    //   log.info(
+    //     `[Cron] App: ${config.get("app.name")} v${config.get("app.version")}`
+    //   );
+    //   log.info(`[Cron] Environment: ${config.get("app.env")}`);
+    //   log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`);
+    //   log.info(`[Cron] Timezone: Asia/Dili`);
+    //   log.info(`[Cron] Current time: ${currentTime}`);
+    //   log.info(`[Cron] Next run scheduled at: ${nextRun}`);
+    //   log.info("═══════════════════════════════════════════════════════════");
+    // } else {
+    //   log.error("[Cron] ❌ Failed to schedule cron job");
+    //   throw new Error("Failed to schedule cron job");
+    // }
+
+    // Schedule daily cron job for duration recalculation (runs at 00:00 daily)
+    const durationRecalcJob = cron.schedule(
+      "* * * * *", // Daily at midnight
       async () => {
         const scheduleTime = moment()
           .tz("Asia/Dili")
           .format("YYYY-MM-DD HH:mm:ss");
         const utcTime = new Date().toISOString();
 
-        if (isCrawlingRunning) {
+        if (isRecalculatingDuration) {
           log.warn(
-            `[Cron] ⚠️  Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
+            `[Duration Recalc Cron] ⚠️  Duration recalculation is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
           );
           return;
         }
 
-        log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`);
+        log.info(
-        log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`);
+          `[Duration Recalc Cron] ⏰ ========== DURATION RECALCULATION CRON TRIGGERED ==========`
-        isCrawlingRunning = true;
+        );
+        log.info(
+          `[Duration Recalc Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`
+        );
+        isRecalculatingDuration = true;
 
         try {
-          await CrawlingService.runCrawling();
+          await CrawlingService.recalculateDuration();
           const completeTime = moment()
             .tz("Asia/Dili")
             .format("YYYY-MM-DD HH:mm:ss");
-          log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`);
+          log.info(
+            `[Duration Recalc Cron] ✅ Duration recalculation completed at ${completeTime}`
+          );
         } catch (error) {
           const errorTime = moment()
             .tz("Asia/Dili")
             .format("YYYY-MM-DD HH:mm:ss");
           log.error(
-            `[Cron] ❌ Crawling service failed at ${errorTime}:`,
+            `[Duration Recalc Cron] ❌ Duration recalculation failed at ${errorTime}:`,
             error
           );
         } finally {
-          isCrawlingRunning = false;
+          isRecalculatingDuration = false;
-          log.info(`[Cron] ==========================================`);
+          log.info(
+            `[Duration Recalc Cron] ==========================================`
+          );
         }
       },
       {
@@ -85,31 +165,30 @@ async function startCrawlingService() {
       }
     );
 
-    // Verify cron job is scheduled
+    // Verify duration recalculation cron job is scheduled
-    if (crawlingJob) {
+    if (durationRecalcJob) {
-      const nextRun = moment()
+      const nextRecalcRun = moment()
         .tz("Asia/Dili")
-        .add(1, "minute")
+        .add(1, "day")
-        .startOf("minute")
+        .startOf("day")
         .format("YYYY-MM-DD HH:mm:ss");
-      const currentTime = moment()
-        .tz("Asia/Dili")
-        .format("YYYY-MM-DD HH:mm:ss");
-
       log.info("═══════════════════════════════════════════════════════════");
-      log.info(`[Cron] ✅ Cron job scheduled successfully`);
       log.info(
-        `[Cron] App: ${config.get("app.name")} v${config.get("app.version")}`
+        `[Duration Recalc Cron] ✅ Daily duration recalculation cron job scheduled successfully`
+      );
+      log.info(
+        `[Duration Recalc Cron] Schedule pattern: 0 0 * * * (daily at midnight)`
+      );
+      log.info(`[Duration Recalc Cron] Timezone: Asia/Dili`);
+      log.info(
+        `[Duration Recalc Cron] Next run scheduled at: ${nextRecalcRun}`
       );
-      log.info(`[Cron] Environment: ${config.get("app.env")}`);
-      log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`);
-      log.info(`[Cron] Timezone: Asia/Dili`);
-      log.info(`[Cron] Current time: ${currentTime}`);
-      log.info(`[Cron] Next run scheduled at: ${nextRun}`);
       log.info("═══════════════════════════════════════════════════════════");
     } else {
-      log.error("[Cron] ❌ Failed to schedule cron job");
+      log.error(
-      throw new Error("Failed to schedule cron job");
+        "[Duration Recalc Cron] ❌ Failed to schedule duration recalculation cron job"
+      );
+      // Don't throw error - this is not critical for the main service
     }
 
     // Run immediately on startup to verify everything works

src/services/crawling.ts

@@ -49,11 +49,18 @@ interface HikvisionResponse {
   };
 }
 
+interface DeviceSystemInfo {
+  macAddress: string;
+  deviceName?: string;
+  deviceID?: string;
+  model?: string;
+}
+
 interface CrawlingState {
   [deviceIp: string]: {
-    searchID: string;
-    lastPosition: number;
     macAddress?: string; // Cache MAC address per device
+    deviceName?: string; // Cache device name from device info
+    deviceInfo?: DeviceSystemInfo; // Cache full device info
   };
 }
 
@@ -669,30 +676,30 @@ export class CrawlingService {
   }
 
   /**
-   * Get device info including MAC address
+   * Get device info including MAC address and device name from device system info
    */
   static async getDeviceInfo(
     deviceIp: string,
     username: string,
     password: string
-  ): Promise<string> {
+  ): Promise<DeviceSystemInfo> {
     try {
-      // Check cache first - but only if it's a valid MAC address format
+      // Check cache first - return cached device info if available
-      if (crawlingState[deviceIp]?.macAddress) {
+      if (crawlingState[deviceIp]?.deviceInfo) {
-        const cachedMac = crawlingState[deviceIp].macAddress!;
+        const cachedInfo = crawlingState[deviceIp].deviceInfo!;
-        // Validate MAC address format (should be XX:XX:XX:XX:XX:XX)
         const macRegex = /^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$/;
-        if (macRegex.test(cachedMac)) {
+        if (macRegex.test(cachedInfo.macAddress)) {
-          console.log(
+          log.info(
-            `[CrawlingService] Using cached MAC address for ${deviceIp}: ${cachedMac}`
+            `[Get Device Info] Using cached device info for ${deviceIp} - MAC: ${
+              cachedInfo.macAddress
+            }, DeviceName: ${cachedInfo.deviceName || "N/A"}`
           );
-          return cachedMac;
+          return cachedInfo;
         } else {
-          console.warn(
+          log.warn(
-            `[CrawlingService] Cached MAC address "${cachedMac}" for ${deviceIp} is invalid format. Fetching fresh.`
+            `[Get Device Info] Cached MAC address "${cachedInfo.macAddress}" for ${deviceIp} is invalid format. Fetching fresh.`
           );
-          // Clear invalid cache
+          delete crawlingState[deviceIp].deviceInfo;
-          delete crawlingState[deviceIp].macAddress;
         }
       }
 
@@ -710,109 +717,107 @@ export class CrawlingService {
 
       // Handle both XML and JSON responses
       let macAddress = "";
+      let deviceName = "";
+      let deviceID = "";
+      let model = "";
 
-      // Debug: Log the exact structure
+      // Helper function to extract value from device info
-      console.log(
+      const extractValue = (obj: any, key: string): string => {
-        `[CrawlingService] Response type: ${typeof response}, has DeviceInfo: ${!!(
+        if (!obj || !obj[key]) return "";
+        if (typeof obj[key] === "string") return obj[key].trim();
+        if (Array.isArray(obj[key])) return obj[key][0]?.trim() || "";
+        if (typeof obj[key] === "object") {
+          return obj[key]._ || obj[key]["#text"] || "";
+        }
+        return "";
+      };
+
+      log.info(
+        `[Get Device Info] Response type: ${typeof response}, has DeviceInfo: ${!!(
           response && response.DeviceInfo
         )}`
       );
 
       if (response && response.DeviceInfo) {
-        // XML response structure: { DeviceInfo: { macAddress: "..." } }
-        // xml2js with explicitArray: false should give us a string directly
         const deviceInfo = response.DeviceInfo;
 
-        // Try multiple ways to extract MAC address
+        // Extract MAC address
-        if (typeof deviceInfo.macAddress === "string") {
+        macAddress = extractValue(deviceInfo, "macAddress");
-          macAddress = deviceInfo.macAddress;
-        } else if (Array.isArray(deviceInfo.macAddress)) {
-          macAddress = deviceInfo.macAddress[0] || "";
-        } else if (
-          deviceInfo.macAddress &&
-          typeof deviceInfo.macAddress === "object"
-        ) {
-          // Sometimes xml2js wraps it in an object
-          macAddress =
-            deviceInfo.macAddress._ || deviceInfo.macAddress["#text"] || "";
-        } else {
-          macAddress = "";
-        }
 
-        // Trim whitespace
+        // Extract device name (this is what we need!)
-        macAddress = macAddress.trim();
+        deviceName = extractValue(deviceInfo, "deviceName");
 
-        console.log(
+        // Extract device ID
-          `[CrawlingService] Extracted MAC address from DeviceInfo: "${macAddress}" (type: ${typeof deviceInfo.macAddress}, isArray: ${Array.isArray(
+        deviceID = extractValue(deviceInfo, "deviceID");
-            deviceInfo.macAddress
+
-          )})`
+        // Extract model
+        model = extractValue(deviceInfo, "model");
+
+        log.info(
+          `[Get Device Info] Extracted from DeviceInfo - MAC: "${macAddress}", DeviceName: "${deviceName}", DeviceID: "${deviceID}", Model: "${model}"`
         );
 
         // If still empty, try to extract from raw response string
-        if (!macAddress && typeof response === "string") {
+        if (typeof response === "string") {
+          if (!macAddress) {
             const macMatch = response.match(
               /<macAddress[^>]*>([^<]+)<\/macAddress>/i
             );
-          if (macMatch && macMatch[1]) {
+            if (macMatch && macMatch[1]) macAddress = macMatch[1].trim();
-            macAddress = macMatch[1].trim();
+          }
-            console.log(
+          if (!deviceName) {
-              `[CrawlingService] Extracted MAC address from raw XML string: "${macAddress}"`
+            const nameMatch = response.match(
+              /<deviceName[^>]*>([^<]+)<\/deviceName>/i
             );
+            if (nameMatch && nameMatch[1]) deviceName = nameMatch[1].trim();
           }
         }
-      } else if (response && response.macAddress) {
+      } else if (response) {
-        // Direct property (unlikely but handle it)
+        // Direct properties (JSON response)
-        macAddress =
+        macAddress = extractValue(response, "macAddress");
-          typeof response.macAddress === "string"
+        deviceName = extractValue(response, "deviceName");
-            ? response.macAddress.trim()
+        deviceID = extractValue(response, "deviceID");
-            : "";
+        model = extractValue(response, "model");
-        console.log(
+      }
-          `[CrawlingService] Extracted MAC address from direct property: "${macAddress}"`
-        );
-      } else {
-        console.warn(
-          `[CrawlingService] Could not find DeviceInfo or macAddress in response for ${deviceIp}. Response keys: ${
-            response ? Object.keys(response).join(", ") : "null"
-          }`
-        );
 
-        // Last resort: try to extract from stringified response
+      // Build device info object
-        const responseStr = JSON.stringify(response);
+      const deviceSystemInfo: DeviceSystemInfo = {
-        const macMatch = responseStr.match(/"macAddress"\s*:\s*"([^"]+)"/i);
+        macAddress: macAddress || "",
-        if (macMatch && macMatch[1]) {
+        deviceName: deviceName || undefined,
-          macAddress = macMatch[1].trim();
+        deviceID: deviceID || undefined,
-          console.log(
+        model: model || undefined,
-            `[CrawlingService] Extracted MAC address from JSON string: "${macAddress}"`
+      };
-          );
-        }
-      }
 
       if (macAddress) {
-        // Cache it
+        // Cache full device info
         if (!crawlingState[deviceIp]) {
-          crawlingState[deviceIp] = {
+          crawlingState[deviceIp] = {};
-            searchID: uuidv4(),
-            lastPosition: 0,
-          };
         }
+        crawlingState[deviceIp].deviceInfo = deviceSystemInfo;
         crawlingState[deviceIp].macAddress = macAddress;
-        console.log(
+        if (deviceName) {
-          `[CrawlingService] MAC address for ${deviceIp}: ${macAddress}`
+          crawlingState[deviceIp].deviceName = deviceName;
-        );
-        return macAddress;
         }
 
-      console.warn(
+        log.info(
-        `[CrawlingService] MAC address is empty for device ${deviceIp}. Response structure:`,
+          `[Get Device Info] Cached device info for ${deviceIp} - MAC: ${macAddress}, DeviceName: ${
+            deviceName || "N/A"
+          }`
+        );
+        return deviceSystemInfo;
+      }
+
+      log.warn(
+        `[Get Device Info] MAC address is empty for device ${deviceIp}. Response structure:`,
         JSON.stringify(response, null, 2)
       );
-      return "";
+      return { macAddress: "" };
     } catch (error) {
-      console.error(
+      log.error(
-        `[CrawlingService] Error fetching device info for ${deviceIp}:`,
+        `[Get Device Info] Error fetching device info for ${deviceIp}:`,
         error
       );
-      return "";
+      return { macAddress: "" };
     }
   }
 
@@ -852,29 +857,49 @@ export class CrawlingService {
     }
   }
 
+  /**
+   * Determine deviceName from ACS Controller Event major/minor event types
+   * Based on Hikvision ACS Event specifications
+   */
+  /**
+   * Get deviceName from device system info (cached from device API)
+   */
+  static getDeviceNameFromDeviceInfo(deviceIp: string): string {
+    try {
+      const cachedDeviceName = crawlingState[deviceIp]?.deviceName;
+
+      if (cachedDeviceName) {
+        log.info(
+          `[Map Event] DeviceName from device info: ${cachedDeviceName} for IP: ${deviceIp}`
+        );
+        return cachedDeviceName;
+      }
+
+      // Fallback: Default to DeviceIn if not available from device
+      log.warn(
+        `[Map Event] DeviceName not available from device info for IP: ${deviceIp}. Defaulting to DeviceIn.`
+      );
+      return "DeviceIn";
+    } catch (error) {
+      log.error(
+        `[Map Event] Error getting deviceName from device info for IP: ${deviceIp}:`,
+        error
+      );
+      return "DeviceIn";
+    }
+  }
+
   /**
    * Map Hikvision event to webhook format
    */
   static async mapHikvisionToWebhook(
     event: HikvisionEvent,
     deviceIp: string,
-    macAddress: string
+    deviceInfo: DeviceSystemInfo
   ): Promise<any> {
-    // Map attendanceStatus to deviceName
+    // Get deviceName from device system info (from device API)
-    // "breakIn", "checkIn" -> "DeviceIn"
+    const deviceName = this.getDeviceNameFromDeviceInfo(deviceIp);
-    // "breakOut", "checkOut" -> "DeviceOut"
+    const macAddress = deviceInfo.macAddress || "";
-    let deviceName = "DeviceIn";
-    if (
-      event.attendanceStatus === "breakOut" ||
-      event.attendanceStatus === "checkOut"
-    ) {
-      deviceName = "DeviceOut";
-    } else if (
-      event.attendanceStatus === "breakIn" ||
-      event.attendanceStatus === "checkIn"
-    ) {
-      deviceName = "DeviceIn";
-    }
 
     // Get employee accessdoorid
     const employeeAccessDoorId = await this.getEmployeeAccessDoorId(
@@ -933,12 +958,9 @@ export class CrawlingService {
     const port = config.get("hikvision.defaultPort");
 
     try {
-      // Initialize or get existing search state
+      // Initialize or get existing state (only for MAC address caching)
       if (!crawlingState[deviceIp]) {
-        crawlingState[deviceIp] = {
+        crawlingState[deviceIp] = {};
-          searchID: uuidv4(),
-          lastPosition: 0,
-        };
       }
 
       const state = crawlingState[deviceIp];
@@ -950,40 +972,46 @@ export class CrawlingService {
         `[CrawlingService] Getting MAC address for device ${deviceIp}...`
       );
 
-      const macStartTime = Date.now();
+      const deviceInfoStartTime = Date.now();
-      const macAddress = await this.getDeviceInfo(deviceIp, username, password);
+      const deviceInfo = await this.getDeviceInfo(deviceIp, username, password);
-      const macDuration = Date.now() - macStartTime;
+      const deviceInfoDuration = Date.now() - deviceInfoStartTime;
 
-      if (!macAddress) {
+      if (!deviceInfo.macAddress) {
         log.warn(
-          `[Crawl Device] ⚠️  Could not get MAC address for device ${deviceIp} (took ${macDuration}ms). Events will be saved without MAC address.`
+          `[Crawl Device] ⚠️  Could not get device info for device ${deviceIp} (took ${deviceInfoDuration}ms). Events will be saved without MAC address.`
         );
         console.warn(
-          `[CrawlingService] Warning: Could not get MAC address for device ${deviceIp}. Events will be saved without MAC address.`
+          `[CrawlingService] Warning: Could not get device info for device ${deviceIp}. Events will be saved without MAC address.`
         );
       } else {
         log.info(
-          `[Crawl Device] ✓ MAC address retrieved: ${macAddress} for device ${deviceIp} (took ${macDuration}ms)`
+          `[Crawl Device] ✓ Device info retrieved for ${deviceIp} (took ${deviceInfoDuration}ms) - MAC: ${
+            deviceInfo.macAddress
+          }, DeviceName: ${deviceInfo.deviceName || "N/A"}`
         );
         console.log(
-          `[CrawlingService] Successfully retrieved MAC address ${macAddress} for device ${deviceIp}`
+          `[CrawlingService] Successfully retrieved device info for ${deviceIp} - MAC: ${
+            deviceInfo.macAddress
+          }, DeviceName: ${deviceInfo.deviceName || "N/A"}`
         );
       }
 
-      // Prepare request payload
+      // Prepare request payload for REAL-TIME events only (no batching)
+      // Always fetch from position 0 to get only the latest events
+      // Since we run every minute, we'll only get new events
       const requestPayload = {
         AcsEventCond: {
-          searchID: state.searchID,
+          searchID: uuidv4(), // New search ID each time for real-time
-          searchResultPosition: state.lastPosition,
+          searchResultPosition: 0, // Always start from position 0 (latest events)
-          maxResults: 10, // Fetch multiple events at once
+          maxResults: config.get("crawler.maxResults") || 10,
           major: 5,
           minor: 75,
-          timeReverseOrder: true,
+          timeReverseOrder: true, // Get newest events first
         },
       };
 
       log.info(
-        `[Crawl Device] Starting crawl for device ${deviceIp} from position ${state.lastPosition} (SearchID: ${state.searchID})`
+        `[Crawl Device] Starting REAL-TIME crawl for device ${deviceIp} - Fetching latest ${requestPayload.AcsEventCond.maxResults} events (no batching)`
       );
 
       // Make API call with digest auth
@@ -1023,12 +1051,16 @@ export class CrawlingService {
       log.info(
         `[Crawl Device] Processing ${
           events.length
-        } event(s) from device ${deviceIp} (MAC: ${macAddress || "NONE"})`
+        } event(s) from device ${deviceIp} (MAC: ${
+          deviceInfo.macAddress || "NONE"
+        }, DeviceName: ${deviceInfo.deviceName || "N/A"})`
       );
       console.log(
         `[CrawlingService] Processing ${
           events.length
-        } events for device ${deviceIp} with MAC: ${macAddress || "NONE"}`
+        } events for device ${deviceIp} with MAC: ${
+          deviceInfo.macAddress || "NONE"
+        }, DeviceName: ${deviceInfo.deviceName || "N/A"}`
       );
 
       let successCount = 0;
@@ -1056,7 +1088,7 @@ export class CrawlingService {
           const webhookEvent = await this.mapHikvisionToWebhook(
             event,
             deviceIp,
-            macAddress
+            deviceInfo
           );
 
           log.info(
@@ -1099,29 +1131,14 @@ export class CrawlingService {
       }
 
       log.info(
-        `[Crawl Device] Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}`
+        `[Crawl Device] ✅ Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}`
       );
 
-      // Update position for next crawl
+      // Real-time mode: No position tracking, just process new events
-      state.lastPosition += events.length;
+      // Each crawl fetches only events from the last minute
-
       log.info(
-        `[Crawl Device] Updated crawl position for device ${deviceIp} - New position: ${state.lastPosition}, Total matches: ${response.AcsEvent.totalMatches}, Status: ${response.AcsEvent.responseStatusStrg}`
+        `[Crawl Device] Real-time crawl completed for device ${deviceIp} - Processed ${events.length} new event(s)`
       );
-
-      // If there are more matches, we'll continue in next crawl
-      if (response.AcsEvent.responseStatusStrg === "MORE") {
-        log.info(
-          `[Crawl Device] ⏭️  More events available for device ${deviceIp} - Total: ${response.AcsEvent.totalMatches}, Current position: ${state.lastPosition}`
-        );
-      } else {
-        // Reset position if no more events
-        log.info(
-          `[Crawl Device] ✅ All events processed for device ${deviceIp} - Resetting position to 0`
-        );
-        state.lastPosition = 0;
-        state.searchID = uuidv4();
-      }
     } catch (error: any) {
       const errorMsg = error?.message || String(error);
       log.error(
@@ -1134,15 +1151,91 @@ export class CrawlingService {
           }, Data: ${JSON.stringify(error.response.data)}`
         );
       }
-      // Reset state on error
+      // No state to reset in real-time mode (only MAC address is cached)
-      if (crawlingState[deviceIp]) {
       log.info(
-          `[Crawl Device] Resetting crawl state for device ${deviceIp} due to error`
+        `[Crawl Device] Error occurred but continuing for next crawl cycle`
       );
-        crawlingState[deviceIp].lastPosition = 0;
-        crawlingState[deviceIp].searchID = uuidv4();
     }
   }
+
+  /**
+   * Check if event already exists in database (duplicate check)
+   * Checks based on: accessdoorid, datelogs, ipaddress, macaddress
+   */
+  static async isDuplicateEvent(eventData: any): Promise<boolean> {
+    try {
+      const accessdoorid = parseInt(
+        eventData.AccessControllerEvent?.employeeNoString || "0"
+      );
+      const datelogs = moment.parseZone(eventData.dateTime).toDate();
+      const ipaddress = eventData.ipAddress || "";
+      const macaddress = eventData.macAddress || "";
+
+      if (!accessdoorid || accessdoorid === 0) {
+        log.warn(
+          `[Duplicate Check] ⚠️  Invalid accessdoorid: ${eventData.AccessControllerEvent?.employeeNoString}, skipping duplicate check`
+        );
+        return false;
+      }
+
+      if (!ipaddress || !macaddress) {
+        log.warn(
+          `[Duplicate Check] ⚠️  Missing IP or MAC address - IP: ${ipaddress}, MAC: ${macaddress}, skipping duplicate check`
+        );
+        return false;
+      }
+
+      // Check if a record with the same accessdoorid, datelogs (exact match or within 1 second), ipaddress, and macaddress exists
+      // Use exact timestamp match first, then 1 second tolerance as fallback
+      const datelogsStr = moment(datelogs).format("YYYY-MM-DD HH:mm:ss");
+
+      log.info(
+        `[Duplicate Check] Checking for duplicate - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}`
+      );
+
+      const existing = await OrmHelper.DB.query(
+        `
+        SELECT _idx, datelogs, accessdoorid, ipaddress, macaddress
+        FROM tbl_attendancedoorlogs 
+        WHERE accessdoorid = ?
+          AND ipaddress = ?
+          AND macaddress = ?
+          AND (
+            DATE_FORMAT(datelogs, '%Y-%m-%d %H:%i:%s') = ?
+            OR ABS(TIMESTAMPDIFF(SECOND, datelogs, ?)) <= 1
+          )
+          AND isdeleted = 0
+        LIMIT 1
+      `,
+        [accessdoorid, ipaddress, macaddress, datelogsStr, datelogs]
+      );
+
+      const isDuplicate = existing && existing.length > 0;
+
+      if (isDuplicate) {
+        const existingRecord = existing[0];
+        log.info(
+          `[Duplicate Check] ✓ DUPLICATE FOUND - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}, Existing _idx: ${
+            existingRecord._idx
+          }, Existing DateTime: ${moment(existingRecord.datelogs).format(
+            "YYYY-MM-DD HH:mm:ss"
+          )}`
+        );
+      } else {
+        log.info(
+          `[Duplicate Check] ✓ NEW EVENT - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}`
+        );
+      }
+
+      return isDuplicate;
+    } catch (error: any) {
+      const errorMsg = error?.message || String(error);
+      log.error(
+        `[Duplicate Check] ✗ ERROR checking duplicate event - EmployeeID: ${eventData.AccessControllerEvent?.employeeNoString}, DateTime: ${eventData.dateTime}, Error: ${errorMsg}`
+      );
+      // If there's an error checking, assume it's not a duplicate to avoid blocking valid events
+      return false;
+    }
   }
 
   /**
@@ -1161,6 +1254,15 @@ export class CrawlingService {
         `[Send Event] Preparing to send event - EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}, IP: ${ipAddress}, MAC: ${macAddress}`
       );
 
+      // Check for duplicates before sending (FIRST CHECK)
+      const isDuplicate = await this.isDuplicateEvent(eventData);
+      if (isDuplicate) {
+        log.info(
+          `[Send Event] ⏭️  SKIPPED (Duplicate - First Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}`
+        );
+        return;
+      }
+
       const bridgeHost = config.get("bridge.host");
       const bridgePort = config.get("bridge.port");
       const bridgeEndpoint = config.get("bridge.endpoint");
@@ -1172,6 +1274,15 @@ export class CrawlingService {
         `[Send Event] Sending to bridge - URL: ${url}, EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}`
       );
 
+      // Double-check for duplicates right before sending (SECOND CHECK - prevents race conditions)
+      const isDuplicateFinal = await this.isDuplicateEvent(eventData);
+      if (isDuplicateFinal) {
+        log.info(
+          `[Send Event] ⏭️  SKIPPED (Duplicate - Final Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}`
+        );
+        return;
+      }
+
       // Send as form data with event_log field (matching the webhook format)
       const startTime = Date.now();
       await axios.post(
@@ -1344,4 +1455,155 @@ export class CrawlingService {
       throw error; // Re-throw so the caller can handle it
     }
   }
+
+  /**
+   * Recalculate duration for door logs from devices with iscrawling=1 and flag=1
+   * For each DeviceIn record, find the previous DeviceOut for the same accessdoorid on the same day
+   */
+  static async recalculateDuration(): Promise<void> {
+    const startTime = Date.now();
+    log.info("═══════════════════════════════════════════════════════════");
+    log.info("[Recalculate Duration] ⏰ Starting duration recalculation...");
+
+    try {
+      // Get all devices with iscrawling=1 and flag=1
+      const devices = await OrmHelper.DB.query(`
+        SELECT 
+          _idx,
+          deviceid,
+          deviceip,
+          flag,
+          iscrawling
+        FROM tbl_deviceinfo
+        WHERE (flag = 1 AND iscrawling = 1)
+          AND isdeleted = 0
+          AND deviceip IS NOT NULL
+      `);
+
+      if (!devices || devices.length === 0) {
+        log.info(
+          "[Recalculate Duration] ℹ️  No devices found with iscrawling=1 and flag=1"
+        );
+        return;
+      }
+
+      log.info(
+        `[Recalculate Duration] Found ${devices.length} device(s) to process`
+      );
+
+      // Get all DeviceIn records from these devices using JOIN for better performance and security
+      // Fix collation mismatch by using COLLATE in JOIN condition
+      const deviceInLogs = await OrmHelper.DB.query(
+        `
+        SELECT 
+          d._idx,
+          d.accessdoorid,
+          d.datelogs,
+          d.devicename,
+          d.ipaddress
+        FROM tbl_attendancedoorlogs d
+        INNER JOIN tbl_deviceinfo di ON d.ipaddress COLLATE utf8mb4_unicode_ci = di.deviceip COLLATE utf8mb4_unicode_ci
+        WHERE di.flag = 1 
+          AND di.iscrawling = 1
+          AND di.isdeleted = 0
+          AND d.devicename = 'DeviceIn'
+          AND d.isdeleted = 0
+          AND DATE(d.datelogs) >= DATE_SUB(CURDATE(), INTERVAL 1 DAY)
+        ORDER BY d.accessdoorid, d.datelogs ASC
+      `
+      );
+
+      if (!deviceInLogs || deviceInLogs.length === 0) {
+        log.info(
+          "[Recalculate Duration] ℹ️  No DeviceIn records found for these devices"
+        );
+        return;
+      }
+
+      log.info(
+        `[Recalculate Duration] Found ${deviceInLogs.length} DeviceIn record(s) to process`
+      );
+
+      let totalUpdated = 0;
+      let totalSkipped = 0;
+      let totalErrors = 0;
+
+      // Process each DeviceIn record
+      for (const deviceInLog of deviceInLogs) {
+        try {
+          const accessdoorid = deviceInLog.accessdoorid;
+          const deviceInDate = moment(deviceInLog.datelogs);
+          const logDate = deviceInDate.format("YYYY-MM-DD");
+
+          // Find the previous DeviceOut for the same accessdoorid on the same day
+          const deviceOutLog = await OrmHelper.DB.query(`
+            SELECT 
+              _idx,
+              datelogs
+            FROM tbl_attendancedoorlogs
+            WHERE accessdoorid = ?
+              AND devicename = 'DeviceOut'
+              AND DATE(datelogs) = ?
+              AND datelogs < ?
+              AND isdeleted = 0
+            ORDER BY datelogs DESC
+            LIMIT 1
+          `, [accessdoorid, logDate, deviceInLog.datelogs]);
+
+          if (!deviceOutLog || deviceOutLog.length === 0) {
+            totalSkipped++;
+            continue;
+          }
+
+          const deviceOutDate = moment(deviceOutLog[0].datelogs);
+          const duration = moment.duration(deviceInDate.diff(deviceOutDate));
+          const formattedDuration = moment
+            .utc(duration.asMilliseconds())
+            .format("HH:mm:ss");
+
+          // Update the duration
+          await OrmHelper.DB.query(`
+            UPDATE tbl_attendancedoorlogs
+            SET duration = ?,
+                uby = 'svc-hcm-crawler',
+                udt = NOW()
+            WHERE _idx = ?
+          `, [formattedDuration, deviceInLog._idx]);
+
+          totalUpdated++;
+
+          if (totalUpdated % 100 === 0) {
+            log.info(
+              `[Recalculate Duration] Progress: ${totalUpdated} updated, ${totalSkipped} skipped`
+            );
+          }
+        } catch (error: any) {
+          totalErrors++;
+          log.error(
+            `[Recalculate Duration] Error processing log _idx=${deviceInLog._idx}:`,
+            error.message
+          );
+        }
+      }
+
+      const totalDuration = Date.now() - startTime;
+      log.info("═══════════════════════════════════════════════════════════");
+      log.info(
+        `[Recalculate Duration] ✅ Completed in ${totalDuration}ms`
+      );
+      log.info(
+        `[Recalculate Duration] Summary: ${totalUpdated} updated, ${totalSkipped} skipped, ${totalErrors} errors`
+      );
+      log.info("═══════════════════════════════════════════════════════════");
+    } catch (error: any) {
+      const totalDuration = Date.now() - startTime;
+      const errorMsg = error?.message || String(error);
+      log.error("═══════════════════════════════════════════════════════════");
+      log.error(
+        `[Recalculate Duration] ❌ Fatal error after ${totalDuration}ms: ${errorMsg}`
+      );
+      log.error("═══════════════════════════════════════════════════════════");
+      throw error;
+    }
+  }
 }