Add duration recalculation feature and improve device info handling in CrawlingService
- Introduced a new cron job for daily duration recalculation of door logs. - Enhanced device info retrieval to include device name and other details. - Updated logging to provide clearer information on device processing and duration recalculation. - Implemented duplicate event checks before sending events to prevent redundancy.
This commit is contained in:
Sony Surahmn
139
src/index.ts
139
src/index.ts
@ -15,6 +15,7 @@ const log: Logger<ILogObj> = new Logger({
|
||||
log.settings.prettyLogTimeZone = "local";
|
||||
|
||||
let isCrawlingRunning = false;
|
||||
let isRecalculatingDuration = false;
|
||||
|
||||
/**
|
||||
* Start the crawling service
|
||||
@ -42,42 +43,121 @@ async function startCrawlingService() {
|
||||
|
||||
// Schedule cron job with timezone support
|
||||
// node-cron v3+ supports timezone option
|
||||
const crawlingJob = cron.schedule(
|
||||
cronInterval,
|
||||
// const crawlingJob = cron.schedule(
|
||||
// cronInterval,
|
||||
// async () => {
|
||||
// const scheduleTime = moment()
|
||||
// .tz("Asia/Dili")
|
||||
// .format("YYYY-MM-DD HH:mm:ss");
|
||||
// const utcTime = new Date().toISOString();
|
||||
|
||||
// if (isCrawlingRunning) {
|
||||
// log.warn(
|
||||
// `[Cron] ⚠️ Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
|
||||
// );
|
||||
// return;
|
||||
// }
|
||||
|
||||
// log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`);
|
||||
// log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`);
|
||||
// isCrawlingRunning = true;
|
||||
|
||||
// try {
|
||||
// await CrawlingService.runCrawling();
|
||||
// const completeTime = moment()
|
||||
// .tz("Asia/Dili")
|
||||
// .format("YYYY-MM-DD HH:mm:ss");
|
||||
// log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`);
|
||||
// } catch (error) {
|
||||
// const errorTime = moment()
|
||||
// .tz("Asia/Dili")
|
||||
// .format("YYYY-MM-DD HH:mm:ss");
|
||||
// log.error(
|
||||
// `[Cron] ❌ Crawling service failed at ${errorTime}:`,
|
||||
// error
|
||||
// );
|
||||
// } finally {
|
||||
// isCrawlingRunning = false;
|
||||
// log.info(`[Cron] ==========================================`);
|
||||
// }
|
||||
// },
|
||||
// {
|
||||
// timezone: "Asia/Dili",
|
||||
// }
|
||||
// );
|
||||
|
||||
// // Verify cron job is scheduled
|
||||
// if (crawlingJob) {
|
||||
// const nextRun = moment()
|
||||
// .tz("Asia/Dili")
|
||||
// .add(1, "minute")
|
||||
// .startOf("minute")
|
||||
// .format("YYYY-MM-DD HH:mm:ss");
|
||||
// const currentTime = moment()
|
||||
// .tz("Asia/Dili")
|
||||
// .format("YYYY-MM-DD HH:mm:ss");
|
||||
|
||||
// log.info("═══════════════════════════════════════════════════════════");
|
||||
// log.info(`[Cron] ✅ Cron job scheduled successfully`);
|
||||
// log.info(
|
||||
// `[Cron] App: ${config.get("app.name")} v${config.get("app.version")}`
|
||||
// );
|
||||
// log.info(`[Cron] Environment: ${config.get("app.env")}`);
|
||||
// log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`);
|
||||
// log.info(`[Cron] Timezone: Asia/Dili`);
|
||||
// log.info(`[Cron] Current time: ${currentTime}`);
|
||||
// log.info(`[Cron] Next run scheduled at: ${nextRun}`);
|
||||
// log.info("═══════════════════════════════════════════════════════════");
|
||||
// } else {
|
||||
// log.error("[Cron] ❌ Failed to schedule cron job");
|
||||
// throw new Error("Failed to schedule cron job");
|
||||
// }
|
||||
|
||||
// Schedule daily cron job for duration recalculation (runs at 00:00 daily)
|
||||
const durationRecalcJob = cron.schedule(
|
||||
"* * * * *", // Daily at midnight
|
||||
async () => {
|
||||
const scheduleTime = moment()
|
||||
.tz("Asia/Dili")
|
||||
.format("YYYY-MM-DD HH:mm:ss");
|
||||
const utcTime = new Date().toISOString();
|
||||
|
||||
if (isCrawlingRunning) {
|
||||
if (isRecalculatingDuration) {
|
||||
log.warn(
|
||||
`[Cron] ⚠️ Crawling service is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
|
||||
`[Duration Recalc Cron] ⚠️ Duration recalculation is still running. Skipping schedule at ${scheduleTime} (UTC: ${utcTime})`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(`[Cron] ⏰ ========== CRON TRIGGERED ==========`);
|
||||
log.info(`[Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`);
|
||||
isCrawlingRunning = true;
|
||||
log.info(
|
||||
`[Duration Recalc Cron] ⏰ ========== DURATION RECALCULATION CRON TRIGGERED ==========`
|
||||
);
|
||||
log.info(
|
||||
`[Duration Recalc Cron] ⏰ Schedule time: ${scheduleTime} (UTC: ${utcTime})`
|
||||
);
|
||||
isRecalculatingDuration = true;
|
||||
|
||||
try {
|
||||
await CrawlingService.runCrawling();
|
||||
await CrawlingService.recalculateDuration();
|
||||
const completeTime = moment()
|
||||
.tz("Asia/Dili")
|
||||
.format("YYYY-MM-DD HH:mm:ss");
|
||||
log.info(`[Cron] ✅ Crawling service completed at ${completeTime}`);
|
||||
log.info(
|
||||
`[Duration Recalc Cron] ✅ Duration recalculation completed at ${completeTime}`
|
||||
);
|
||||
} catch (error) {
|
||||
const errorTime = moment()
|
||||
.tz("Asia/Dili")
|
||||
.format("YYYY-MM-DD HH:mm:ss");
|
||||
log.error(
|
||||
`[Cron] ❌ Crawling service failed at ${errorTime}:`,
|
||||
`[Duration Recalc Cron] ❌ Duration recalculation failed at ${errorTime}:`,
|
||||
error
|
||||
);
|
||||
} finally {
|
||||
isCrawlingRunning = false;
|
||||
log.info(`[Cron] ==========================================`);
|
||||
isRecalculatingDuration = false;
|
||||
log.info(
|
||||
`[Duration Recalc Cron] ==========================================`
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
@ -85,31 +165,30 @@ async function startCrawlingService() {
|
||||
}
|
||||
);
|
||||
|
||||
// Verify cron job is scheduled
|
||||
if (crawlingJob) {
|
||||
const nextRun = moment()
|
||||
// Verify duration recalculation cron job is scheduled
|
||||
if (durationRecalcJob) {
|
||||
const nextRecalcRun = moment()
|
||||
.tz("Asia/Dili")
|
||||
.add(1, "minute")
|
||||
.startOf("minute")
|
||||
.add(1, "day")
|
||||
.startOf("day")
|
||||
.format("YYYY-MM-DD HH:mm:ss");
|
||||
const currentTime = moment()
|
||||
.tz("Asia/Dili")
|
||||
.format("YYYY-MM-DD HH:mm:ss");
|
||||
|
||||
log.info("═══════════════════════════════════════════════════════════");
|
||||
log.info(`[Cron] ✅ Cron job scheduled successfully`);
|
||||
log.info(
|
||||
`[Cron] App: ${config.get("app.name")} v${config.get("app.version")}`
|
||||
`[Duration Recalc Cron] ✅ Daily duration recalculation cron job scheduled successfully`
|
||||
);
|
||||
log.info(
|
||||
`[Duration Recalc Cron] Schedule pattern: 0 0 * * * (daily at midnight)`
|
||||
);
|
||||
log.info(`[Duration Recalc Cron] Timezone: Asia/Dili`);
|
||||
log.info(
|
||||
`[Duration Recalc Cron] Next run scheduled at: ${nextRecalcRun}`
|
||||
);
|
||||
log.info(`[Cron] Environment: ${config.get("app.env")}`);
|
||||
log.info(`[Cron] Schedule pattern: ${cronInterval} (every minute)`);
|
||||
log.info(`[Cron] Timezone: Asia/Dili`);
|
||||
log.info(`[Cron] Current time: ${currentTime}`);
|
||||
log.info(`[Cron] Next run scheduled at: ${nextRun}`);
|
||||
log.info("═══════════════════════════════════════════════════════════");
|
||||
} else {
|
||||
log.error("[Cron] ❌ Failed to schedule cron job");
|
||||
throw new Error("Failed to schedule cron job");
|
||||
log.error(
|
||||
"[Duration Recalc Cron] ❌ Failed to schedule duration recalculation cron job"
|
||||
);
|
||||
// Don't throw error - this is not critical for the main service
|
||||
}
|
||||
|
||||
// Run immediately on startup to verify everything works
|
||||
|
||||
@ -49,11 +49,18 @@ interface HikvisionResponse {
|
||||
};
|
||||
}
|
||||
|
||||
interface DeviceSystemInfo {
|
||||
macAddress: string;
|
||||
deviceName?: string;
|
||||
deviceID?: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
interface CrawlingState {
|
||||
[deviceIp: string]: {
|
||||
searchID: string;
|
||||
lastPosition: number;
|
||||
macAddress?: string; // Cache MAC address per device
|
||||
deviceName?: string; // Cache device name from device info
|
||||
deviceInfo?: DeviceSystemInfo; // Cache full device info
|
||||
};
|
||||
}
|
||||
|
||||
@ -669,30 +676,30 @@ export class CrawlingService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get device info including MAC address
|
||||
* Get device info including MAC address and device name from device system info
|
||||
*/
|
||||
static async getDeviceInfo(
|
||||
deviceIp: string,
|
||||
username: string,
|
||||
password: string
|
||||
): Promise<string> {
|
||||
): Promise<DeviceSystemInfo> {
|
||||
try {
|
||||
// Check cache first - but only if it's a valid MAC address format
|
||||
if (crawlingState[deviceIp]?.macAddress) {
|
||||
const cachedMac = crawlingState[deviceIp].macAddress!;
|
||||
// Validate MAC address format (should be XX:XX:XX:XX:XX:XX)
|
||||
// Check cache first - return cached device info if available
|
||||
if (crawlingState[deviceIp]?.deviceInfo) {
|
||||
const cachedInfo = crawlingState[deviceIp].deviceInfo!;
|
||||
const macRegex = /^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$/;
|
||||
if (macRegex.test(cachedMac)) {
|
||||
console.log(
|
||||
`[CrawlingService] Using cached MAC address for ${deviceIp}: ${cachedMac}`
|
||||
if (macRegex.test(cachedInfo.macAddress)) {
|
||||
log.info(
|
||||
`[Get Device Info] Using cached device info for ${deviceIp} - MAC: ${
|
||||
cachedInfo.macAddress
|
||||
}, DeviceName: ${cachedInfo.deviceName || "N/A"}`
|
||||
);
|
||||
return cachedMac;
|
||||
return cachedInfo;
|
||||
} else {
|
||||
console.warn(
|
||||
`[CrawlingService] Cached MAC address "${cachedMac}" for ${deviceIp} is invalid format. Fetching fresh.`
|
||||
log.warn(
|
||||
`[Get Device Info] Cached MAC address "${cachedInfo.macAddress}" for ${deviceIp} is invalid format. Fetching fresh.`
|
||||
);
|
||||
// Clear invalid cache
|
||||
delete crawlingState[deviceIp].macAddress;
|
||||
delete crawlingState[deviceIp].deviceInfo;
|
||||
}
|
||||
}
|
||||
|
||||
@ -710,109 +717,107 @@ export class CrawlingService {
|
||||
|
||||
// Handle both XML and JSON responses
|
||||
let macAddress = "";
|
||||
let deviceName = "";
|
||||
let deviceID = "";
|
||||
let model = "";
|
||||
|
||||
// Debug: Log the exact structure
|
||||
console.log(
|
||||
`[CrawlingService] Response type: ${typeof response}, has DeviceInfo: ${!!(
|
||||
// Helper function to extract value from device info
|
||||
const extractValue = (obj: any, key: string): string => {
|
||||
if (!obj || !obj[key]) return "";
|
||||
if (typeof obj[key] === "string") return obj[key].trim();
|
||||
if (Array.isArray(obj[key])) return obj[key][0]?.trim() || "";
|
||||
if (typeof obj[key] === "object") {
|
||||
return obj[key]._ || obj[key]["#text"] || "";
|
||||
}
|
||||
return "";
|
||||
};
|
||||
|
||||
log.info(
|
||||
`[Get Device Info] Response type: ${typeof response}, has DeviceInfo: ${!!(
|
||||
response && response.DeviceInfo
|
||||
)}`
|
||||
);
|
||||
|
||||
if (response && response.DeviceInfo) {
|
||||
// XML response structure: { DeviceInfo: { macAddress: "..." } }
|
||||
// xml2js with explicitArray: false should give us a string directly
|
||||
const deviceInfo = response.DeviceInfo;
|
||||
|
||||
// Try multiple ways to extract MAC address
|
||||
if (typeof deviceInfo.macAddress === "string") {
|
||||
macAddress = deviceInfo.macAddress;
|
||||
} else if (Array.isArray(deviceInfo.macAddress)) {
|
||||
macAddress = deviceInfo.macAddress[0] || "";
|
||||
} else if (
|
||||
deviceInfo.macAddress &&
|
||||
typeof deviceInfo.macAddress === "object"
|
||||
) {
|
||||
// Sometimes xml2js wraps it in an object
|
||||
macAddress =
|
||||
deviceInfo.macAddress._ || deviceInfo.macAddress["#text"] || "";
|
||||
} else {
|
||||
macAddress = "";
|
||||
}
|
||||
// Extract MAC address
|
||||
macAddress = extractValue(deviceInfo, "macAddress");
|
||||
|
||||
// Trim whitespace
|
||||
macAddress = macAddress.trim();
|
||||
// Extract device name (this is what we need!)
|
||||
deviceName = extractValue(deviceInfo, "deviceName");
|
||||
|
||||
console.log(
|
||||
`[CrawlingService] Extracted MAC address from DeviceInfo: "${macAddress}" (type: ${typeof deviceInfo.macAddress}, isArray: ${Array.isArray(
|
||||
deviceInfo.macAddress
|
||||
)})`
|
||||
// Extract device ID
|
||||
deviceID = extractValue(deviceInfo, "deviceID");
|
||||
|
||||
// Extract model
|
||||
model = extractValue(deviceInfo, "model");
|
||||
|
||||
log.info(
|
||||
`[Get Device Info] Extracted from DeviceInfo - MAC: "${macAddress}", DeviceName: "${deviceName}", DeviceID: "${deviceID}", Model: "${model}"`
|
||||
);
|
||||
|
||||
// If still empty, try to extract from raw response string
|
||||
if (!macAddress && typeof response === "string") {
|
||||
const macMatch = response.match(
|
||||
/<macAddress[^>]*>([^<]+)<\/macAddress>/i
|
||||
);
|
||||
if (macMatch && macMatch[1]) {
|
||||
macAddress = macMatch[1].trim();
|
||||
console.log(
|
||||
`[CrawlingService] Extracted MAC address from raw XML string: "${macAddress}"`
|
||||
if (typeof response === "string") {
|
||||
if (!macAddress) {
|
||||
const macMatch = response.match(
|
||||
/<macAddress[^>]*>([^<]+)<\/macAddress>/i
|
||||
);
|
||||
if (macMatch && macMatch[1]) macAddress = macMatch[1].trim();
|
||||
}
|
||||
if (!deviceName) {
|
||||
const nameMatch = response.match(
|
||||
/<deviceName[^>]*>([^<]+)<\/deviceName>/i
|
||||
);
|
||||
if (nameMatch && nameMatch[1]) deviceName = nameMatch[1].trim();
|
||||
}
|
||||
}
|
||||
} else if (response && response.macAddress) {
|
||||
// Direct property (unlikely but handle it)
|
||||
macAddress =
|
||||
typeof response.macAddress === "string"
|
||||
? response.macAddress.trim()
|
||||
: "";
|
||||
console.log(
|
||||
`[CrawlingService] Extracted MAC address from direct property: "${macAddress}"`
|
||||
);
|
||||
} else {
|
||||
console.warn(
|
||||
`[CrawlingService] Could not find DeviceInfo or macAddress in response for ${deviceIp}. Response keys: ${
|
||||
response ? Object.keys(response).join(", ") : "null"
|
||||
}`
|
||||
);
|
||||
|
||||
// Last resort: try to extract from stringified response
|
||||
const responseStr = JSON.stringify(response);
|
||||
const macMatch = responseStr.match(/"macAddress"\s*:\s*"([^"]+)"/i);
|
||||
if (macMatch && macMatch[1]) {
|
||||
macAddress = macMatch[1].trim();
|
||||
console.log(
|
||||
`[CrawlingService] Extracted MAC address from JSON string: "${macAddress}"`
|
||||
);
|
||||
}
|
||||
} else if (response) {
|
||||
// Direct properties (JSON response)
|
||||
macAddress = extractValue(response, "macAddress");
|
||||
deviceName = extractValue(response, "deviceName");
|
||||
deviceID = extractValue(response, "deviceID");
|
||||
model = extractValue(response, "model");
|
||||
}
|
||||
|
||||
// Build device info object
|
||||
const deviceSystemInfo: DeviceSystemInfo = {
|
||||
macAddress: macAddress || "",
|
||||
deviceName: deviceName || undefined,
|
||||
deviceID: deviceID || undefined,
|
||||
model: model || undefined,
|
||||
};
|
||||
|
||||
if (macAddress) {
|
||||
// Cache it
|
||||
// Cache full device info
|
||||
if (!crawlingState[deviceIp]) {
|
||||
crawlingState[deviceIp] = {
|
||||
searchID: uuidv4(),
|
||||
lastPosition: 0,
|
||||
};
|
||||
crawlingState[deviceIp] = {};
|
||||
}
|
||||
crawlingState[deviceIp].deviceInfo = deviceSystemInfo;
|
||||
crawlingState[deviceIp].macAddress = macAddress;
|
||||
console.log(
|
||||
`[CrawlingService] MAC address for ${deviceIp}: ${macAddress}`
|
||||
if (deviceName) {
|
||||
crawlingState[deviceIp].deviceName = deviceName;
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[Get Device Info] Cached device info for ${deviceIp} - MAC: ${macAddress}, DeviceName: ${
|
||||
deviceName || "N/A"
|
||||
}`
|
||||
);
|
||||
return macAddress;
|
||||
return deviceSystemInfo;
|
||||
}
|
||||
|
||||
console.warn(
|
||||
`[CrawlingService] MAC address is empty for device ${deviceIp}. Response structure:`,
|
||||
log.warn(
|
||||
`[Get Device Info] MAC address is empty for device ${deviceIp}. Response structure:`,
|
||||
JSON.stringify(response, null, 2)
|
||||
);
|
||||
return "";
|
||||
return { macAddress: "" };
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[CrawlingService] Error fetching device info for ${deviceIp}:`,
|
||||
log.error(
|
||||
`[Get Device Info] Error fetching device info for ${deviceIp}:`,
|
||||
error
|
||||
);
|
||||
return "";
|
||||
return { macAddress: "" };
|
||||
}
|
||||
}
|
||||
|
||||
@ -852,29 +857,49 @@ export class CrawlingService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine deviceName from ACS Controller Event major/minor event types
|
||||
* Based on Hikvision ACS Event specifications
|
||||
*/
|
||||
/**
|
||||
* Get deviceName from device system info (cached from device API)
|
||||
*/
|
||||
static getDeviceNameFromDeviceInfo(deviceIp: string): string {
|
||||
try {
|
||||
const cachedDeviceName = crawlingState[deviceIp]?.deviceName;
|
||||
|
||||
if (cachedDeviceName) {
|
||||
log.info(
|
||||
`[Map Event] DeviceName from device info: ${cachedDeviceName} for IP: ${deviceIp}`
|
||||
);
|
||||
return cachedDeviceName;
|
||||
}
|
||||
|
||||
// Fallback: Default to DeviceIn if not available from device
|
||||
log.warn(
|
||||
`[Map Event] DeviceName not available from device info for IP: ${deviceIp}. Defaulting to DeviceIn.`
|
||||
);
|
||||
return "DeviceIn";
|
||||
} catch (error) {
|
||||
log.error(
|
||||
`[Map Event] Error getting deviceName from device info for IP: ${deviceIp}:`,
|
||||
error
|
||||
);
|
||||
return "DeviceIn";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Hikvision event to webhook format
|
||||
*/
|
||||
static async mapHikvisionToWebhook(
|
||||
event: HikvisionEvent,
|
||||
deviceIp: string,
|
||||
macAddress: string
|
||||
deviceInfo: DeviceSystemInfo
|
||||
): Promise<any> {
|
||||
// Map attendanceStatus to deviceName
|
||||
// "breakIn", "checkIn" -> "DeviceIn"
|
||||
// "breakOut", "checkOut" -> "DeviceOut"
|
||||
let deviceName = "DeviceIn";
|
||||
if (
|
||||
event.attendanceStatus === "breakOut" ||
|
||||
event.attendanceStatus === "checkOut"
|
||||
) {
|
||||
deviceName = "DeviceOut";
|
||||
} else if (
|
||||
event.attendanceStatus === "breakIn" ||
|
||||
event.attendanceStatus === "checkIn"
|
||||
) {
|
||||
deviceName = "DeviceIn";
|
||||
}
|
||||
// Get deviceName from device system info (from device API)
|
||||
const deviceName = this.getDeviceNameFromDeviceInfo(deviceIp);
|
||||
const macAddress = deviceInfo.macAddress || "";
|
||||
|
||||
// Get employee accessdoorid
|
||||
const employeeAccessDoorId = await this.getEmployeeAccessDoorId(
|
||||
@ -933,12 +958,9 @@ export class CrawlingService {
|
||||
const port = config.get("hikvision.defaultPort");
|
||||
|
||||
try {
|
||||
// Initialize or get existing search state
|
||||
// Initialize or get existing state (only for MAC address caching)
|
||||
if (!crawlingState[deviceIp]) {
|
||||
crawlingState[deviceIp] = {
|
||||
searchID: uuidv4(),
|
||||
lastPosition: 0,
|
||||
};
|
||||
crawlingState[deviceIp] = {};
|
||||
}
|
||||
|
||||
const state = crawlingState[deviceIp];
|
||||
@ -950,40 +972,46 @@ export class CrawlingService {
|
||||
`[CrawlingService] Getting MAC address for device ${deviceIp}...`
|
||||
);
|
||||
|
||||
const macStartTime = Date.now();
|
||||
const macAddress = await this.getDeviceInfo(deviceIp, username, password);
|
||||
const macDuration = Date.now() - macStartTime;
|
||||
const deviceInfoStartTime = Date.now();
|
||||
const deviceInfo = await this.getDeviceInfo(deviceIp, username, password);
|
||||
const deviceInfoDuration = Date.now() - deviceInfoStartTime;
|
||||
|
||||
if (!macAddress) {
|
||||
if (!deviceInfo.macAddress) {
|
||||
log.warn(
|
||||
`[Crawl Device] ⚠️ Could not get MAC address for device ${deviceIp} (took ${macDuration}ms). Events will be saved without MAC address.`
|
||||
`[Crawl Device] ⚠️ Could not get device info for device ${deviceIp} (took ${deviceInfoDuration}ms). Events will be saved without MAC address.`
|
||||
);
|
||||
console.warn(
|
||||
`[CrawlingService] Warning: Could not get MAC address for device ${deviceIp}. Events will be saved without MAC address.`
|
||||
`[CrawlingService] Warning: Could not get device info for device ${deviceIp}. Events will be saved without MAC address.`
|
||||
);
|
||||
} else {
|
||||
log.info(
|
||||
`[Crawl Device] ✓ MAC address retrieved: ${macAddress} for device ${deviceIp} (took ${macDuration}ms)`
|
||||
`[Crawl Device] ✓ Device info retrieved for ${deviceIp} (took ${deviceInfoDuration}ms) - MAC: ${
|
||||
deviceInfo.macAddress
|
||||
}, DeviceName: ${deviceInfo.deviceName || "N/A"}`
|
||||
);
|
||||
console.log(
|
||||
`[CrawlingService] Successfully retrieved MAC address ${macAddress} for device ${deviceIp}`
|
||||
`[CrawlingService] Successfully retrieved device info for ${deviceIp} - MAC: ${
|
||||
deviceInfo.macAddress
|
||||
}, DeviceName: ${deviceInfo.deviceName || "N/A"}`
|
||||
);
|
||||
}
|
||||
|
||||
// Prepare request payload
|
||||
// Prepare request payload for REAL-TIME events only (no batching)
|
||||
// Always fetch from position 0 to get only the latest events
|
||||
// Since we run every minute, we'll only get new events
|
||||
const requestPayload = {
|
||||
AcsEventCond: {
|
||||
searchID: state.searchID,
|
||||
searchResultPosition: state.lastPosition,
|
||||
maxResults: 10, // Fetch multiple events at once
|
||||
searchID: uuidv4(), // New search ID each time for real-time
|
||||
searchResultPosition: 0, // Always start from position 0 (latest events)
|
||||
maxResults: config.get("crawler.maxResults") || 10,
|
||||
major: 5,
|
||||
minor: 75,
|
||||
timeReverseOrder: true,
|
||||
timeReverseOrder: true, // Get newest events first
|
||||
},
|
||||
};
|
||||
|
||||
log.info(
|
||||
`[Crawl Device] Starting crawl for device ${deviceIp} from position ${state.lastPosition} (SearchID: ${state.searchID})`
|
||||
`[Crawl Device] Starting REAL-TIME crawl for device ${deviceIp} - Fetching latest ${requestPayload.AcsEventCond.maxResults} events (no batching)`
|
||||
);
|
||||
|
||||
// Make API call with digest auth
|
||||
@ -1023,12 +1051,16 @@ export class CrawlingService {
|
||||
log.info(
|
||||
`[Crawl Device] Processing ${
|
||||
events.length
|
||||
} event(s) from device ${deviceIp} (MAC: ${macAddress || "NONE"})`
|
||||
} event(s) from device ${deviceIp} (MAC: ${
|
||||
deviceInfo.macAddress || "NONE"
|
||||
}, DeviceName: ${deviceInfo.deviceName || "N/A"})`
|
||||
);
|
||||
console.log(
|
||||
`[CrawlingService] Processing ${
|
||||
events.length
|
||||
} events for device ${deviceIp} with MAC: ${macAddress || "NONE"}`
|
||||
} events for device ${deviceIp} with MAC: ${
|
||||
deviceInfo.macAddress || "NONE"
|
||||
}, DeviceName: ${deviceInfo.deviceName || "N/A"}`
|
||||
);
|
||||
|
||||
let successCount = 0;
|
||||
@ -1056,7 +1088,7 @@ export class CrawlingService {
|
||||
const webhookEvent = await this.mapHikvisionToWebhook(
|
||||
event,
|
||||
deviceIp,
|
||||
macAddress
|
||||
deviceInfo
|
||||
);
|
||||
|
||||
log.info(
|
||||
@ -1099,29 +1131,14 @@ export class CrawlingService {
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[Crawl Device] Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}`
|
||||
`[Crawl Device] ✅ Completed processing ${events.length} event(s) from device ${deviceIp} - Success: ${successCount}, Errors: ${errorCount}`
|
||||
);
|
||||
|
||||
// Update position for next crawl
|
||||
state.lastPosition += events.length;
|
||||
|
||||
// Real-time mode: No position tracking, just process new events
|
||||
// Each crawl fetches only events from the last minute
|
||||
log.info(
|
||||
`[Crawl Device] Updated crawl position for device ${deviceIp} - New position: ${state.lastPosition}, Total matches: ${response.AcsEvent.totalMatches}, Status: ${response.AcsEvent.responseStatusStrg}`
|
||||
`[Crawl Device] Real-time crawl completed for device ${deviceIp} - Processed ${events.length} new event(s)`
|
||||
);
|
||||
|
||||
// If there are more matches, we'll continue in next crawl
|
||||
if (response.AcsEvent.responseStatusStrg === "MORE") {
|
||||
log.info(
|
||||
`[Crawl Device] ⏭️ More events available for device ${deviceIp} - Total: ${response.AcsEvent.totalMatches}, Current position: ${state.lastPosition}`
|
||||
);
|
||||
} else {
|
||||
// Reset position if no more events
|
||||
log.info(
|
||||
`[Crawl Device] ✅ All events processed for device ${deviceIp} - Resetting position to 0`
|
||||
);
|
||||
state.lastPosition = 0;
|
||||
state.searchID = uuidv4();
|
||||
}
|
||||
} catch (error: any) {
|
||||
const errorMsg = error?.message || String(error);
|
||||
log.error(
|
||||
@ -1134,14 +1151,90 @@ export class CrawlingService {
|
||||
}, Data: ${JSON.stringify(error.response.data)}`
|
||||
);
|
||||
}
|
||||
// Reset state on error
|
||||
if (crawlingState[deviceIp]) {
|
||||
log.info(
|
||||
`[Crawl Device] Resetting crawl state for device ${deviceIp} due to error`
|
||||
// No state to reset in real-time mode (only MAC address is cached)
|
||||
log.info(
|
||||
`[Crawl Device] Error occurred but continuing for next crawl cycle`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if event already exists in database (duplicate check)
|
||||
* Checks based on: accessdoorid, datelogs, ipaddress, macaddress
|
||||
*/
|
||||
static async isDuplicateEvent(eventData: any): Promise<boolean> {
|
||||
try {
|
||||
const accessdoorid = parseInt(
|
||||
eventData.AccessControllerEvent?.employeeNoString || "0"
|
||||
);
|
||||
const datelogs = moment.parseZone(eventData.dateTime).toDate();
|
||||
const ipaddress = eventData.ipAddress || "";
|
||||
const macaddress = eventData.macAddress || "";
|
||||
|
||||
if (!accessdoorid || accessdoorid === 0) {
|
||||
log.warn(
|
||||
`[Duplicate Check] ⚠️ Invalid accessdoorid: ${eventData.AccessControllerEvent?.employeeNoString}, skipping duplicate check`
|
||||
);
|
||||
crawlingState[deviceIp].lastPosition = 0;
|
||||
crawlingState[deviceIp].searchID = uuidv4();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ipaddress || !macaddress) {
|
||||
log.warn(
|
||||
`[Duplicate Check] ⚠️ Missing IP or MAC address - IP: ${ipaddress}, MAC: ${macaddress}, skipping duplicate check`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if a record with the same accessdoorid, datelogs (exact match or within 1 second), ipaddress, and macaddress exists
|
||||
// Use exact timestamp match first, then 1 second tolerance as fallback
|
||||
const datelogsStr = moment(datelogs).format("YYYY-MM-DD HH:mm:ss");
|
||||
|
||||
log.info(
|
||||
`[Duplicate Check] Checking for duplicate - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}`
|
||||
);
|
||||
|
||||
const existing = await OrmHelper.DB.query(
|
||||
`
|
||||
SELECT _idx, datelogs, accessdoorid, ipaddress, macaddress
|
||||
FROM tbl_attendancedoorlogs
|
||||
WHERE accessdoorid = ?
|
||||
AND ipaddress = ?
|
||||
AND macaddress = ?
|
||||
AND (
|
||||
DATE_FORMAT(datelogs, '%Y-%m-%d %H:%i:%s') = ?
|
||||
OR ABS(TIMESTAMPDIFF(SECOND, datelogs, ?)) <= 1
|
||||
)
|
||||
AND isdeleted = 0
|
||||
LIMIT 1
|
||||
`,
|
||||
[accessdoorid, ipaddress, macaddress, datelogsStr, datelogs]
|
||||
);
|
||||
|
||||
const isDuplicate = existing && existing.length > 0;
|
||||
|
||||
if (isDuplicate) {
|
||||
const existingRecord = existing[0];
|
||||
log.info(
|
||||
`[Duplicate Check] ✓ DUPLICATE FOUND - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}, Existing _idx: ${
|
||||
existingRecord._idx
|
||||
}, Existing DateTime: ${moment(existingRecord.datelogs).format(
|
||||
"YYYY-MM-DD HH:mm:ss"
|
||||
)}`
|
||||
);
|
||||
} else {
|
||||
log.info(
|
||||
`[Duplicate Check] ✓ NEW EVENT - EmployeeID: ${accessdoorid}, DateTime: ${datelogsStr}, IP: ${ipaddress}, MAC: ${macaddress}`
|
||||
);
|
||||
}
|
||||
|
||||
return isDuplicate;
|
||||
} catch (error: any) {
|
||||
const errorMsg = error?.message || String(error);
|
||||
log.error(
|
||||
`[Duplicate Check] ✗ ERROR checking duplicate event - EmployeeID: ${eventData.AccessControllerEvent?.employeeNoString}, DateTime: ${eventData.dateTime}, Error: ${errorMsg}`
|
||||
);
|
||||
// If there's an error checking, assume it's not a duplicate to avoid blocking valid events
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1161,6 +1254,15 @@ export class CrawlingService {
|
||||
`[Send Event] Preparing to send event - EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}, IP: ${ipAddress}, MAC: ${macAddress}`
|
||||
);
|
||||
|
||||
// Check for duplicates before sending (FIRST CHECK)
|
||||
const isDuplicate = await this.isDuplicateEvent(eventData);
|
||||
if (isDuplicate) {
|
||||
log.info(
|
||||
`[Send Event] ⏭️ SKIPPED (Duplicate - First Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const bridgeHost = config.get("bridge.host");
|
||||
const bridgePort = config.get("bridge.port");
|
||||
const bridgeEndpoint = config.get("bridge.endpoint");
|
||||
@ -1172,6 +1274,15 @@ export class CrawlingService {
|
||||
`[Send Event] Sending to bridge - URL: ${url}, EmployeeID: ${employeeId}, Device: ${deviceName}, DateTime: ${dateTime}`
|
||||
);
|
||||
|
||||
// Double-check for duplicates right before sending (SECOND CHECK - prevents race conditions)
|
||||
const isDuplicateFinal = await this.isDuplicateEvent(eventData);
|
||||
if (isDuplicateFinal) {
|
||||
log.info(
|
||||
`[Send Event] ⏭️ SKIPPED (Duplicate - Final Check) - EmployeeID: ${employeeId}, DateTime: ${dateTime}, Device: ${deviceName}, IP: ${ipAddress}, MAC: ${macAddress}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Send as form data with event_log field (matching the webhook format)
|
||||
const startTime = Date.now();
|
||||
await axios.post(
|
||||
@ -1344,4 +1455,155 @@ export class CrawlingService {
|
||||
throw error; // Re-throw so the caller can handle it
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recalculate duration for door logs from devices with iscrawling=1 and flag=1
|
||||
* For each DeviceIn record, find the previous DeviceOut for the same accessdoorid on the same day
|
||||
*/
|
||||
static async recalculateDuration(): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
log.info("═══════════════════════════════════════════════════════════");
|
||||
log.info("[Recalculate Duration] ⏰ Starting duration recalculation...");
|
||||
|
||||
try {
|
||||
// Get all devices with iscrawling=1 and flag=1
|
||||
const devices = await OrmHelper.DB.query(`
|
||||
SELECT
|
||||
_idx,
|
||||
deviceid,
|
||||
deviceip,
|
||||
flag,
|
||||
iscrawling
|
||||
FROM tbl_deviceinfo
|
||||
WHERE (flag = 1 AND iscrawling = 1)
|
||||
AND isdeleted = 0
|
||||
AND deviceip IS NOT NULL
|
||||
`);
|
||||
|
||||
if (!devices || devices.length === 0) {
|
||||
log.info(
|
||||
"[Recalculate Duration] ℹ️ No devices found with iscrawling=1 and flag=1"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[Recalculate Duration] Found ${devices.length} device(s) to process`
|
||||
);
|
||||
|
||||
// Get all DeviceIn records from these devices using JOIN for better performance and security
|
||||
// Fix collation mismatch by using COLLATE in JOIN condition
|
||||
const deviceInLogs = await OrmHelper.DB.query(
|
||||
`
|
||||
SELECT
|
||||
d._idx,
|
||||
d.accessdoorid,
|
||||
d.datelogs,
|
||||
d.devicename,
|
||||
d.ipaddress
|
||||
FROM tbl_attendancedoorlogs d
|
||||
INNER JOIN tbl_deviceinfo di ON d.ipaddress COLLATE utf8mb4_unicode_ci = di.deviceip COLLATE utf8mb4_unicode_ci
|
||||
WHERE di.flag = 1
|
||||
AND di.iscrawling = 1
|
||||
AND di.isdeleted = 0
|
||||
AND d.devicename = 'DeviceIn'
|
||||
AND d.isdeleted = 0
|
||||
AND DATE(d.datelogs) >= DATE_SUB(CURDATE(), INTERVAL 1 DAY)
|
||||
ORDER BY d.accessdoorid, d.datelogs ASC
|
||||
`
|
||||
);
|
||||
|
||||
if (!deviceInLogs || deviceInLogs.length === 0) {
|
||||
log.info(
|
||||
"[Recalculate Duration] ℹ️ No DeviceIn records found for these devices"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[Recalculate Duration] Found ${deviceInLogs.length} DeviceIn record(s) to process`
|
||||
);
|
||||
|
||||
let totalUpdated = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
// Process each DeviceIn record
|
||||
for (const deviceInLog of deviceInLogs) {
|
||||
try {
|
||||
const accessdoorid = deviceInLog.accessdoorid;
|
||||
const deviceInDate = moment(deviceInLog.datelogs);
|
||||
const logDate = deviceInDate.format("YYYY-MM-DD");
|
||||
|
||||
// Find the previous DeviceOut for the same accessdoorid on the same day
|
||||
const deviceOutLog = await OrmHelper.DB.query(`
|
||||
SELECT
|
||||
_idx,
|
||||
datelogs
|
||||
FROM tbl_attendancedoorlogs
|
||||
WHERE accessdoorid = ?
|
||||
AND devicename = 'DeviceOut'
|
||||
AND DATE(datelogs) = ?
|
||||
AND datelogs < ?
|
||||
AND isdeleted = 0
|
||||
ORDER BY datelogs DESC
|
||||
LIMIT 1
|
||||
`, [accessdoorid, logDate, deviceInLog.datelogs]);
|
||||
|
||||
if (!deviceOutLog || deviceOutLog.length === 0) {
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const deviceOutDate = moment(deviceOutLog[0].datelogs);
|
||||
const duration = moment.duration(deviceInDate.diff(deviceOutDate));
|
||||
const formattedDuration = moment
|
||||
.utc(duration.asMilliseconds())
|
||||
.format("HH:mm:ss");
|
||||
|
||||
// Update the duration
|
||||
await OrmHelper.DB.query(`
|
||||
UPDATE tbl_attendancedoorlogs
|
||||
SET duration = ?,
|
||||
uby = 'svc-hcm-crawler',
|
||||
udt = NOW()
|
||||
WHERE _idx = ?
|
||||
`, [formattedDuration, deviceInLog._idx]);
|
||||
|
||||
totalUpdated++;
|
||||
|
||||
if (totalUpdated % 100 === 0) {
|
||||
log.info(
|
||||
`[Recalculate Duration] Progress: ${totalUpdated} updated, ${totalSkipped} skipped`
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
totalErrors++;
|
||||
log.error(
|
||||
`[Recalculate Duration] Error processing log _idx=${deviceInLog._idx}:`,
|
||||
error.message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const totalDuration = Date.now() - startTime;
|
||||
log.info("═══════════════════════════════════════════════════════════");
|
||||
log.info(
|
||||
`[Recalculate Duration] ✅ Completed in ${totalDuration}ms`
|
||||
);
|
||||
log.info(
|
||||
`[Recalculate Duration] Summary: ${totalUpdated} updated, ${totalSkipped} skipped, ${totalErrors} errors`
|
||||
);
|
||||
log.info("═══════════════════════════════════════════════════════════");
|
||||
} catch (error: any) {
|
||||
const totalDuration = Date.now() - startTime;
|
||||
const errorMsg = error?.message || String(error);
|
||||
log.error("═══════════════════════════════════════════════════════════");
|
||||
log.error(
|
||||
`[Recalculate Duration] ❌ Fatal error after ${totalDuration}ms: ${errorMsg}`
|
||||
);
|
||||
log.error("═══════════════════════════════════════════════════════════");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user