From 38ca3af0a128f6e1723cb72cab5b9a7f7b1a5369 Mon Sep 17 00:00:00 2001 From: sergeyyarkov Date: Tue, 3 Mar 2026 16:28:25 +0300 Subject: [PATCH] upd --- index.d.ts | 1 + index.js | 8 +++ package-lock.json | 52 ++++++++++++++----- package.json | 1 + router/reports/parseExcel.handler.js | 74 +++++++++++++++++++--------- router/reports/syncParsed.handler.js | 6 ++- 6 files changed, 107 insertions(+), 35 deletions(-) diff --git a/index.d.ts b/index.d.ts index 0d86061..093d793 100644 --- a/index.d.ts +++ b/index.d.ts @@ -10,6 +10,7 @@ declare global { root_cause: string; applicantName: string; executorNames: string; + isMarked?: boolean; }; type HTTPRequest = { diff --git a/index.js b/index.js index 1271599..218da70 100644 --- a/index.js +++ b/index.js @@ -13,6 +13,14 @@ function application() { try { const url = new URL(`http://localhost:${SERVER_PORT}${req.url}`); + res.setHeaders( + new Headers({ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST", + "Access-Control-Allow-Headers": "Content-Type" + }) + ); + /** @type {HTTPContext} */ const ctx = { data: null, diff --git a/package-lock.json b/package-lock.json index c9f0ace..9f6e38c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "formidable": "^3.5.4", "mssql": "^12.2.0", + "p-limit": "^7.3.0", "serialize-error": "^13.0.1", "xlsx": "^0.18.5", "yup": "^1.7.1" @@ -1671,6 +1672,37 @@ } }, "node_modules/p-limit": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.2.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", @@ -1686,15 +1718,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, "engines": { "node": ">=10" }, @@ -2146,13 +2175,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 6a0870c..d2c4efe 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "formidable": "^3.5.4", "mssql": "^12.2.0", + "p-limit": "^7.3.0", "serialize-error": "^13.0.1", "xlsx": "^0.18.5", "yup": "^1.7.1" diff --git a/router/reports/parseExcel.handler.js b/router/reports/parseExcel.handler.js index e4ae2bf..e65fcaf 100644 --- a/router/reports/parseExcel.handler.js +++ b/router/reports/parseExcel.handler.js @@ -1,8 +1,11 @@ import { BadRequestError } from "#root/errors/index.js"; import db from "#root/db.js"; import xlsx from "xlsx"; +import pLimit from "p-limit"; +import { serializeError } from "serialize-error"; const sheetName = "Отчеты"; +const limit = pLimit(10); /** @type {xlsx.ParsingOptions} */ const xlsxParseOptions = { @@ -16,8 +19,8 @@ const xlsxParseOptions = { /** * URL: /api/reports_excel_file_parse * Method: POST - * Description: Читает Excel файл отчетов и возвращает его в формате JSON. Принимает на вход файл - * отчетов в формате Excel. + * Description: Принимает на вход файл отчетов в формате Excel, парсит его, проверяет связи с БД, + * отдает отчет в формате JSON частями. * * @type {HTTPRouteHandler} */ @@ -29,12 +32,13 @@ export default async (ctx) => { const [, files] = ctx.data; const excelFilePath = files?.report?.at(0)?.filepath; - /** @type {{ markedReports: ReportModel[], reports: ReportModel[] }} */ - const ret = { reports: [], markedReports: [] }; + /** @type {ReportModel[]} */ + const reports = []; - let foundEqupments = 0; + let parseResultCount = { equipments: 0, applicants: 0, executors: 0 }; + let isReqClosed = false; - ctx.res.res.setHeaders( + res.res.setHeaders( new Headers({ Connection: "keep-alive", "Cache-Control": "no-cache", @@ -42,6 +46,8 @@ export default async (ctx) => { }) ); + res.res.on("close", () => (isReqClosed = true)); + if (excelFilePath) { const workbook = xlsx.readFile(excelFilePath, xlsxParseOptions); const sheet = workbook.Sheets[sheetName]; @@ -57,53 +63,77 @@ export default async (ctx) => { /** Несинхронизированные помеченные отчеты */ if ("__EMPTY_3" in data[i] && data[i]["__EMPTY_3"] === "r") { - ret.markedReports.push({ + reports.push({ date: data[i][cols[0]], equipment: data[i][cols[1]], reason_call: data[i][cols[3]].split("\r\r\n")[1] || "", job_description: data[i][cols[4]].split("\r\r\n")[1] || "", root_cause: data[i]["__EMPTY_2"] || "", applicantName: data[i][cols[3]].split("\r\r\n")[0] || "", - executorNames: data[i][cols[4]].split("\r\r\n")[0] || "" + executorNames: data[i][cols[4]].split("\r\r\n")[0] || "", + isMarked: true }); continue; } /** Остальные отчеты */ - ret.reports.push({ + reports.push({ date: data[i][cols[0]], equipment: data[i][cols[1]], - reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]), - job_description: parseReasonCallAndJobDesc(data[i][cols[4]]), + reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(), + job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(), root_cause: data[i]["__EMPTY_2"] || "", - applicantName: parseNames(data[i][cols[3]]), - executorNames: parseNames(data[i][cols[4]]) + applicantName: parseNames(data[i][cols[3]]).trim(), + executorNames: parseNames(data[i][cols[4]]).trim() }); } /** @param {ReportModel} report */ const checkReportAssigments = async (report) => { + if (isReqClosed) { + return Promise.reject("Client closed the connection."); + } + const equipment = await db .request() - .query(`select id, name from dbo.asu_system_api_subsystemlist where name like N'%${report.equipment}%';`); + .query(`select id from dbo.asu_system_api_subsystemlist where name like N'%${report.equipment}%';`); - if (equipment.recordset.length !== 0) foundEqupments++; + const applicant = await db + .request() + .query( + `select id from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';` + ); - res.sendSSEJson(report, "reportParseProgress"); + if (equipment.recordset.length !== 0) parseResultCount.equipments++; + if (applicant.recordset.length !== 0) parseResultCount.applicants++; + + res.sendSSEJson( + { + report: { + ...report, + equipment: equipment.recordset.at(0)?.id || null, + applicant: applicant.recordset.at(0)?.id || null + } + }, + "progress" + ); return equipment; }; - console.log("Total reports:", ret.reports.concat(ret.markedReports).length); - await Promise.allSettled(ret.reports.concat(ret.markedReports).map((r) => checkReportAssigments(r))); - ctx.res.sendSSEJson({ foundEqupments }, "reportParseDone"); - console.log("Parse done"); + const requests = reports.map((r) => limit(() => checkReportAssigments(r))); + + await Promise.all(requests) + .then(() => res.sendSSEJson(parseResultCount, "done")) + .catch((error) => { + console.error(error); + res.sendSSEJson({ error: serializeError(error) }, "error"); + }) + .finally(() => res.res.end()); } else { throw new BadRequestError(); } - - res.res.end(); }; function parseReasonCallAndJobDesc(str = "") { diff --git a/router/reports/syncParsed.handler.js b/router/reports/syncParsed.handler.js index 188f296..a83bb49 100644 --- a/router/reports/syncParsed.handler.js +++ b/router/reports/syncParsed.handler.js @@ -1,3 +1,4 @@ +import { BadRequestError } from "#root/errors/index.js"; import yup from "yup"; var reportItemSchema = yup.object({ @@ -14,6 +15,9 @@ var reportItemSchema = yup.object({ export default async ({ res, data }) => { const inputSchema = yup.object({ reports: yup.array(reportItemSchema).required() }).required(); - await inputSchema.validate(data); + await inputSchema.validate(data).catch((err) => { + throw new BadRequestError(err.message); + }); + res.sendJson({ data }); };