From 433a412985a4d8956b0725202db462c54ba1e764 Mon Sep 17 00:00:00 2001 From: sergeyyarkov Date: Wed, 11 Mar 2026 12:53:03 +0300 Subject: [PATCH] refactor --- handlers/reports/parseExcel.handler.js | 197 ++---------------------- handlers/reports/statusParse.handler.js | 2 + http-server/route.js | 7 +- index.js | 2 +- jsconfig.json | 4 +- models.d.ts | 1 + package.json | 4 +- services/index.js | 1 + services/reports.service.js | 195 +++++++++++++++++++++++ 9 files changed, 226 insertions(+), 187 deletions(-) create mode 100644 services/index.js create mode 100644 services/reports.service.js diff --git a/handlers/reports/parseExcel.handler.js b/handlers/reports/parseExcel.handler.js index 778adfe..70f956a 100644 --- a/handlers/reports/parseExcel.handler.js +++ b/handlers/reports/parseExcel.handler.js @@ -1,22 +1,10 @@ import { BadRequestError, ServiceUnavailableError } from "#root/http-server/index.js"; -import db from "#root/db.js"; -import xlsx from "xlsx"; -import pLimit from "p-limit"; import { clients } from "#root/index.js"; +import { reportsService } from "#services/index.js"; import { serializeError } from "serialize-error"; -const sheetName = "Отчеты"; const MAX_CONCURRENT_PARSE_CLIENT = Number.parseInt(process.env.REPORT_CONCURRENT_PARSE_CLIENTS || "3", 10); -/** @type {xlsx.ParsingOptions} */ -const xlsxParseOptions = { - sheets: sheetName, - cellHTML: false, - cellFormula: false, - cellDates: true, - dateNF: "yyyy-mm-dd" -}; - /** * URL: /api/reports_excel_file_parse * Method: POST @@ -28,12 +16,8 @@ const xlsxParseOptions = { export default async (ctx) => { const { res, searchParams } = ctx; const [, files] = ctx.data; - const limit = pLimit(10); const clientId = searchParams.get("clientId"); - const excelFilePath = files?.report?.at(0)?.filepath; - const reports = []; - let isSseClientClosed = false; - let parseResultCount = { equipments: 0, applicants: 0, executors: 0, reportsFullfilled: 0 }; + const excelFilepath = files?.report?.at(0)?.filepath; if (!ctx.data) { throw new BadRequestError("Cannot proceed uploaded file"); @@ -61,82 +45,24 @@ export default async (ctx) => { throw new ServiceUnavailableError("Your request on parsing is processing"); } - sseClientCtx.res.res.on("close", () => (isSseClientClosed = true)); + /** @param {ReportModel} report */ + const onParseProgress = async (report) => sseClientCtx.res.sendSSEJson(report, "progress"); - if (excelFilePath) { + const cancelAnalyzeCond = () => Boolean(sseClientCtx.local?.isClosed); + + if (excelFilepath) { try { - const workbook = xlsx.readFile(excelFilePath, xlsxParseOptions); - const sheet = workbook.Sheets[sheetName]; - - if (!sheet) throw new Error("Invalid reports workbook.", { cause: { statusCode: 400 } }); - - const data = xlsx.utils.sheet_to_json(sheet, { raw: false, blankrows: true, header: "A" }); - - res.sendJson({ message: "OK" }); - sseClientCtx.local.isParsing = true; - - for (let i = 0; i < data.length; i++) { - const cols = Object.keys(data[i]); - - if (i === 0) continue; - - /** Несинхронизированные помеченные отчеты */ - if ("G" in data[i] && data[i]["G"] === "r") { - reports.push({ - date: data[i][cols[0]], - equipment: data[i][cols[1]], - reason_call: data[i][cols[3]].split("\r\r\n")[1] || "", - job_description: data[i][cols[4]].split("\r\r\n")[1] || "", - root_cause: data[i]["F"] || "", - applicantName: data[i][cols[3]].split("\r\r\n")[0] || "", - executorNames: data[i][cols[4]].split("\r\r\n")[0] || "", - rowNum: data[i].__rowNum__ + 1, - isMarked: true - }); - - continue; - } - - /** Остальные отчеты */ - reports.push({ - date: data[i][cols[0]], - equipment: data[i][cols[1]], - reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(), - job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(), - root_cause: data[i]["F"] || "", - applicantName: parseNames(data[i][cols[3]]).trim(), - executorNames: parseNames(data[i][cols[4]]).trim(), - rowNum: data[i].__rowNum__ + 1 - }); - } - } catch (error) { - sseClientCtx.local.isParsing = false; - throw error; - } - - try { - const requests = reports.map((report) => - limit(() => { - return checkReportAssigments( - report, - (checkedReport) => { - const isEquipmentLinked = checkedReport.equipment instanceof Object; - const isApplicantLinked = checkedReport.applicant instanceof Object; - const isExecutorsLinked = checkedReport.executors && checkedReport.executors.length !== 0; - - if (isEquipmentLinked) parseResultCount.equipments++; - if (isApplicantLinked) parseResultCount.applicants++; - if (isExecutorsLinked) parseResultCount.executors += checkedReport.executors?.length || 0; - if (isEquipmentLinked && isApplicantLinked && isExecutorsLinked) parseResultCount.reportsFullfilled++; - - sseClientCtx.res.sendSSEJson(checkedReport, "progress"); - }, - isSseClientClosed - ); - }) + const reports = reportsService.parseExcelFile(excelFilepath); + const { result, reports: reportsPromises } = reportsService.analyzeParseResult( + reports, + cancelAnalyzeCond, + onParseProgress ); - await Promise.all(requests); - sseClientCtx.res.sendSSEJson(parseResultCount, "done"); + + sseClientCtx.local.isParsing = true; + res.sendJson({ message: "OK" }); + await Promise.all(reportsPromises); + sseClientCtx.res.sendSSEJson(result, "done"); } catch (error) { console.error(error); sseClientCtx.res.sendSSEJson({ error: serializeError(error) }, "parseError"); @@ -147,92 +73,3 @@ export default async (ctx) => { throw new BadRequestError(); } }; - -/** - * @param {ReportModel} report - * @param {(report: ReportModel) => void} callback - * @param {boolean} rejectCond - */ -async function checkReportAssigments(report, callback, rejectCond) { - if (rejectCond) return Promise.reject(`Report check is rejected`); - - const executorSurnames = extractSurnamesFromExecutorField(report.executorNames); - - const equipment = await db.request() - .query(`select t1.id, t1.name, t1.location_id, t3.name as base_location_name, t2.name as location_name, t2.base_location_id - from dbo.asu_system_api_subsystemlist as t1 - left join dbo.asu_system_api_location as t2 on t2.id = t1.location_id - left join dbo.asu_system_api_baselocation as t3 on t3.id = t2.base_location_id - where t1.name like N'%${report.equipment}%'; - `); - - const applicant = await db - .request() - .query( - `select id, name from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';` - ); - - /** @type {ExecutorModel[]} */ - const executors = []; - - for (const eSurname of executorSurnames) { - if (eSurname) { - const executor = await db - .request() - .query( - `select id, surname + ' ' + name + ' ' + patronymic as fullname from user_user where surname like N'%${eSurname}%';` - ); - if (executor.recordset.length !== 0) { - executors.push({ id: executor.recordset[0].id, fullname: executor.recordset[0].fullname }); - } - } - } - - const hasEquipmentLink = equipment.recordset.length !== 0; - const hasApplicantLink = applicant.recordset.length !== 0; - - const location = - hasEquipmentLink && equipment.recordset[0].location_id - ? { - id: equipment.recordset[0].location_id, - name: equipment.recordset[0].location_name, - base_location_name: equipment.recordset[0].base_location_name, - base_location: equipment.recordset[0].base_location_id - } - : null; - - callback({ - ...report, - executors, - equipment: hasEquipmentLink ? { id: equipment.recordset[0].id, name: equipment.recordset[0].name, location } : null, - applicant: hasApplicantLink ? { id: applicant.recordset[0].id, name: applicant.recordset[0].name } : null - }); -} - -function parseReasonCallAndJobDesc(str = "") { - const splitted = splitLongSpacedSentence(str); - if (splitted.length > 1) return splitted.slice(1).join(" "); - return str.split("\r\n").slice(1).join(" "); -} - -function parseNames(str = "") { - const splitted = splitLongSpacedSentence(str); - if (splitted.length > 1) return splitted[0]; - return str.split("\r\n")[0]; -} - -/** @param {string} str */ -function extractSurnamesFromExecutorField(str) { - return str - .replaceAll(",", "") - .split(/(?:.\..\.)|(?:.\..\s)|(?:.\.\s.\.)|(?:.\..)/g) - .map((s) => s.trim()); -} - -/** - * @param {string} str - * @param {number} min - */ -function splitLongSpacedSentence(str, min = 5) { - return str.split(new RegExp(`\\s{${min},}`)); -} diff --git a/handlers/reports/statusParse.handler.js b/handlers/reports/statusParse.handler.js index c9d5081..e941d22 100644 --- a/handlers/reports/statusParse.handler.js +++ b/handlers/reports/statusParse.handler.js @@ -13,6 +13,7 @@ export default (ctx) => { const clientId = crypto.randomUUID(); clients.set(clientId, ctx); + ctx.local.isClosed = false; res.res.setHeaders( new Headers({ @@ -25,6 +26,7 @@ export default (ctx) => { res.sendSSEJson({ message: "Connected!", clientId }, "data"); res.res.on("close", () => { + ctx.local.isClosed = true; clients.delete(clientId); res.res.end(); }); diff --git a/http-server/route.js b/http-server/route.js index 7c43d44..626d6ca 100644 --- a/http-server/route.js +++ b/http-server/route.js @@ -1,7 +1,7 @@ import { serializeError } from "serialize-error"; import formidable from "formidable"; import { BadRequestError, HttpError, MethodNotAllowedError, NotFoundError } from "./errors.js"; -import * as utils from "./utils/index.js"; +import * as strUtils from "./utils/string.js"; /** * @returns {import("./types/http-server.js").HttpRouter} @@ -82,11 +82,10 @@ export function createRouter() { prefixNameGroup = prevPrefix; }, define(method, pathname, handler, options = {}) { - if (!utils.matchRoutePathname(pathname)) throw new Error(`Incorrect pathname "${pathname}"`); - if (prefixNameGroup) pathname = utils.formatRoutePathname(prefixNameGroup + pathname); + if (!strUtils.matchRoutePathname(pathname)) throw new Error(`Incorrect pathname "${pathname}"`); + if (prefixNameGroup) pathname = strUtils.formatRoutePathname(prefixNameGroup + pathname); /** @type {import("./types/http-server.js").HttpRoute} */ routes.push({ method, pattern: new URLPattern({ pathname }), handler, options }); - console.log("defined:", pathname); } }; } diff --git a/index.js b/index.js index 729e233..c65c70a 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ import path from "node:path"; import fsp from "node:fs/promises"; import { createServer } from "./http-server/index.js"; -import { reportsHandlers } from "./handlers/index.js"; +import { reportsHandlers } from "#handlers/index.js"; import packageJson from "./package.json" with { type: "json" }; import db from "./db.js"; diff --git a/jsconfig.json b/jsconfig.json index d5bd13c..82701e6 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -8,7 +8,9 @@ "module": "nodenext", "resolveJsonModule": true, "paths": { - "#root/*": ["./*"] + "#root/*": ["./*"], + "#handlers/*": ["./handlers/*"], + "#services/*": ["./services/*"] } }, "exclude": ["./http-server/types/*.d.ts"] diff --git a/models.d.ts b/models.d.ts index ee8fec3..5804862 100644 --- a/models.d.ts +++ b/models.d.ts @@ -16,6 +16,7 @@ type ReportModel = { applicantName: string; executorNames: string; isMarked?: boolean; + rowNum: number; }; type ExecutorModel = { id: string; fullname: string }; diff --git a/package.json b/package.json index 3550fbf..693cf65 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,9 @@ "yup": "^1.7.1" }, "imports": { - "#root/*": "./*" + "#root/*": "./*", + "#handlers/*": "./handlers/*", + "#services/*": "./services/*" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/services/index.js b/services/index.js new file mode 100644 index 0000000..3d61005 --- /dev/null +++ b/services/index.js @@ -0,0 +1 @@ +export * from "./reports.service.js"; diff --git a/services/reports.service.js b/services/reports.service.js new file mode 100644 index 0000000..1736d8f --- /dev/null +++ b/services/reports.service.js @@ -0,0 +1,195 @@ +import xlsx from "xlsx"; +import pLimit from "p-limit"; +import db from "#root/db.js"; + +const sheetName = "Отчеты"; + +/** @type {xlsx.ParsingOptions} */ +const xlsxParseOptions = { + sheets: sheetName, + cellHTML: false, + cellFormula: false, + cellDates: true, + dateNF: "yyyy-mm-dd" +}; + +export const reportsService = { + /** + * Читает файл Excel и возвращает данные о репорте в массиве + * + * @param {string} filepath + * @returns {ReportModel[]} + */ + parseExcelFile(filepath) { + const workbook = xlsx.readFile(filepath, xlsxParseOptions); + const sheet = workbook.Sheets[sheetName]; + + /** @type {ReportModel[]} */ + const reports = []; + + if (!sheet) throw new Error("Invalid reports workbook."); + + const data = xlsx.utils.sheet_to_json(sheet, { raw: false, blankrows: true, header: "A" }); + + for (let i = 0; i < data.length; i++) { + const cols = Object.keys(data[i]); + + if (i === 0) continue; + + /** Несинхронизированные помеченные отчеты */ + if ("G" in data[i] && data[i]["G"] === "r") { + reports.push({ + date: data[i][cols[0]], + equipment: data[i][cols[1]], + reason_call: data[i][cols[3]].split("\r\r\n")[1] || "", + job_description: data[i][cols[4]].split("\r\r\n")[1] || "", + root_cause: data[i]["F"] || "", + applicantName: data[i][cols[3]].split("\r\r\n")[0] || "", + executorNames: data[i][cols[4]].split("\r\r\n")[0] || "", + rowNum: data[i].__rowNum__ + 1, + isMarked: true + }); + + continue; + } + + /** Остальные отчеты */ + reports.push({ + date: data[i][cols[0]], + equipment: data[i][cols[1]], + reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(), + job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(), + root_cause: data[i]["F"] || "", + applicantName: parseNames(data[i][cols[3]]).trim(), + executorNames: parseNames(data[i][cols[4]]).trim(), + rowNum: data[i].__rowNum__ + 1 + }); + } + + return reports; + }, + + /** + * Проверяет отчет на наличие связей с БД АСУ и возвращает + * этот отчет с устанновленными связями или без них + * + * @param {ReportModel} report + * @param {() => boolean} cancelCondFn + * @returns {Promise} + */ + async checkReportLinks(report, cancelCondFn) { + const executorSurnames = extractSurnamesFromExecutorField(report.executorNames); + if (cancelCondFn()) return Promise.reject(); + + const equipment = await db.request() + .query(`select t1.id, t1.name, t1.location_id, t3.name as base_location_name, t2.name as location_name, t2.base_location_id + from dbo.asu_system_api_subsystemlist as t1 + left join dbo.asu_system_api_location as t2 on t2.id = t1.location_id + left join dbo.asu_system_api_baselocation as t3 on t3.id = t2.base_location_id + where t1.name like N'%${report.equipment}%'; + `); + + const applicant = await db + .request() + .query( + `select id, name from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';` + ); + + /** @type {ExecutorModel[]} */ + const executors = []; + + for (const eSurname of executorSurnames) { + if (eSurname) { + const executor = await db + .request() + .query( + `select id, surname + ' ' + name + ' ' + patronymic as fullname from user_user where surname like N'%${eSurname}%';` + ); + if (executor.recordset.length !== 0) { + executors.push({ id: executor.recordset[0].id, fullname: executor.recordset[0].fullname }); + } + } + } + + const hasEquipmentLink = equipment.recordset.length !== 0; + const hasApplicantLink = applicant.recordset.length !== 0; + + const location = + hasEquipmentLink && equipment.recordset[0].location_id + ? { + id: equipment.recordset[0].location_id, + name: equipment.recordset[0].location_name, + base_location_name: equipment.recordset[0].base_location_name, + base_location: equipment.recordset[0].base_location_id + } + : null; + + return { + ...report, + executors, + equipment: hasEquipmentLink + ? { id: equipment.recordset[0].id, name: equipment.recordset[0].name, location } + : null, + applicant: hasApplicantLink ? { id: applicant.recordset[0].id, name: applicant.recordset[0].name } : null + }; + }, + + /** + * Проверяет связи отчетов с БД и возвращает результат + * + * @param {ReportModel[]} reports + * @param {() => boolean} cancelCondFn + * @param {(report: ReportModel) => Promise | void} [progressCb] + */ + analyzeParseResult(reports, cancelCondFn, progressCb) { + const limit = pLimit(10); + const result = { reportsFullfilled: 0 }; + + return { + result, + reports: reports.map((report) => + limit(async () => { + const checkedReport = await this.checkReportLinks(report, cancelCondFn); + const isEquipmentLinked = checkedReport.equipment instanceof Object; + const isApplicantLinked = checkedReport.applicant instanceof Object; + const isExecutorsLinked = checkedReport.executors && checkedReport.executors.length !== 0; + + if (isEquipmentLinked && isApplicantLinked && isExecutorsLinked) { + result.reportsFullfilled++; + } + + progressCb && (await progressCb(checkedReport)); + return checkedReport; + }) + ) + }; + } +}; + +function parseReasonCallAndJobDesc(str = "") { + const splitted = splitLongSpacedSentence(str); + if (splitted.length > 1) return splitted.slice(1).join(" "); + return str.split("\r\n").slice(1).join(" "); +} + +function parseNames(str = "") { + const splitted = splitLongSpacedSentence(str); + if (splitted.length > 1) return splitted[0]; + return str.split("\r\n")[0]; +} + +/** @param {string} str */ +function extractSurnamesFromExecutorField(str) { + return str + .replaceAll(",", "") + .split(/(?:.\..\.)|(?:.\..\s)|(?:.\.\s.\.)|(?:.\..)/g) + .map((s) => s.trim()); +} + +/** + * @param {string} str + * @param {number} min + */ +function splitLongSpacedSentence(str, min = 5) { + return str.split(new RegExp(`\\s{${min},}`)); +}