upd

index.d.ts

@@ -10,6 +10,7 @@ declare global {
     root_cause: string;
     applicantName: string;
     executorNames: string;
+    isMarked?: boolean;
   };
 
   type HTTPRequest = {

index.js

@@ -13,6 +13,14 @@ function application() {
     try {
       const url = new URL(`http://localhost:${SERVER_PORT}${req.url}`);
 
+      res.setHeaders(
+        new Headers({
+          "Access-Control-Allow-Origin": "*",
+          "Access-Control-Allow-Methods": "GET, POST",
+          "Access-Control-Allow-Headers": "Content-Type"
+        })
+      );
+
       /** @type {HTTPContext} */
       const ctx = {
         data: null,

package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "formidable": "^3.5.4",
         "mssql": "^12.2.0",
+        "p-limit": "^7.3.0",
         "serialize-error": "^13.0.1",
         "xlsx": "^0.18.5",
         "yup": "^1.7.1"
@@ -1671,6 +1672,37 @@
       }
     },
     "node_modules/p-limit": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz",
+      "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==",
+      "license": "MIT",
+      "dependencies": {
+        "yocto-queue": "^1.2.1"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-locate": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+      "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "p-limit": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-locate/node_modules/p-limit": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
       "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
@@ -1686,15 +1718,12 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/p-locate": {
+    "node_modules/p-locate/node_modules/yocto-queue": {
-      "version": "5.0.0",
+      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
-      "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
       "dev": true,
       "license": "MIT",
-      "dependencies": {
-        "p-limit": "^3.0.2"
-      },
       "engines": {
         "node": ">=10"
       },
@@ -2146,13 +2175,12 @@
       }
     },
     "node_modules/yocto-queue": {
-      "version": "0.1.0",
+      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz",
-      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==",
-      "dev": true,
       "license": "MIT",
       "engines": {
-        "node": ">=10"
+        "node": ">=12.20"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"

package.json

@@ -16,6 +16,7 @@
   "dependencies": {
     "formidable": "^3.5.4",
     "mssql": "^12.2.0",
+    "p-limit": "^7.3.0",
     "serialize-error": "^13.0.1",
     "xlsx": "^0.18.5",
     "yup": "^1.7.1"

router/reports/parseExcel.handler.js

@@ -1,8 +1,11 @@
 import { BadRequestError } from "#root/errors/index.js";
 import db from "#root/db.js";
 import xlsx from "xlsx";
+import pLimit from "p-limit";
+import { serializeError } from "serialize-error";
 
 const sheetName = "Отчеты";
+const limit = pLimit(10);
 
 /** @type {xlsx.ParsingOptions} */
 const xlsxParseOptions = {
@@ -16,8 +19,8 @@ const xlsxParseOptions = {
 /**
  * URL: /api/reports_excel_file_parse
  * Method: POST
- * Description: Читает Excel файл отчетов и возвращает его в формате JSON. Принимает на вход файл
+ * Description: Принимает на вход файл отчетов в формате Excel, парсит его, проверяет связи с БД,
- *              отчетов в формате Excel.
+ *              отдает отчет в формате JSON частями.
  *
  * @type {HTTPRouteHandler}
  */
@@ -29,12 +32,13 @@ export default async (ctx) => {
   const [, files] = ctx.data;
   const excelFilePath = files?.report?.at(0)?.filepath;
 
-  /** @type {{ markedReports: ReportModel[], reports: ReportModel[] }} */
+  /** @type {ReportModel[]} */
-  const ret = { reports: [], markedReports: [] };
+  const reports = [];
 
-  let foundEqupments = 0;
+  let parseResultCount = { equipments: 0, applicants: 0, executors: 0 };
+  let isReqClosed = false;
 
-  ctx.res.res.setHeaders(
+  res.res.setHeaders(
     new Headers({
       Connection: "keep-alive",
       "Cache-Control": "no-cache",
@@ -42,6 +46,8 @@ export default async (ctx) => {
     })
   );
 
+  res.res.on("close", () => (isReqClosed = true));
+
   if (excelFilePath) {
     const workbook = xlsx.readFile(excelFilePath, xlsxParseOptions);
     const sheet = workbook.Sheets[sheetName];
@@ -57,53 +63,77 @@ export default async (ctx) => {
 
       /** Несинхронизированные помеченные отчеты */
       if ("__EMPTY_3" in data[i] && data[i]["__EMPTY_3"] === "r") {
-        ret.markedReports.push({
+        reports.push({
           date: data[i][cols[0]],
           equipment: data[i][cols[1]],
           reason_call: data[i][cols[3]].split("\r\r\n")[1] || "",
           job_description: data[i][cols[4]].split("\r\r\n")[1] || "",
           root_cause: data[i]["__EMPTY_2"] || "",
           applicantName: data[i][cols[3]].split("\r\r\n")[0] || "",
-          executorNames: data[i][cols[4]].split("\r\r\n")[0] || ""
+          executorNames: data[i][cols[4]].split("\r\r\n")[0] || "",
+          isMarked: true
         });
 
         continue;
       }
 
       /** Остальные отчеты */
-      ret.reports.push({
+      reports.push({
         date: data[i][cols[0]],
         equipment: data[i][cols[1]],
-        reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]),
+        reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(),
-        job_description: parseReasonCallAndJobDesc(data[i][cols[4]]),
+        job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(),
         root_cause: data[i]["__EMPTY_2"] || "",
-        applicantName: parseNames(data[i][cols[3]]),
+        applicantName: parseNames(data[i][cols[3]]).trim(),
-        executorNames: parseNames(data[i][cols[4]])
+        executorNames: parseNames(data[i][cols[4]]).trim()
       });
     }
 
     /** @param {ReportModel} report */
     const checkReportAssigments = async (report) => {
+      if (isReqClosed) {
+        return Promise.reject("Client closed the connection.");
+      }
+
       const equipment = await db
         .request()
-        .query(`select id, name from dbo.asu_system_api_subsystemlist where name like N'%${report.equipment}%';`);
+        .query(`select id from dbo.asu_system_api_subsystemlist where name like N'%${report.equipment}%';`);
 
-      if (equipment.recordset.length !== 0) foundEqupments++;
+      const applicant = await db
+        .request()
+        .query(
+          `select id from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';`
+        );
 
-      res.sendSSEJson(report, "reportParseProgress");
+      if (equipment.recordset.length !== 0) parseResultCount.equipments++;
+      if (applicant.recordset.length !== 0) parseResultCount.applicants++;
+
+      res.sendSSEJson(
+        {
+          report: {
+            ...report,
+            equipment: equipment.recordset.at(0)?.id || null,
+            applicant: applicant.recordset.at(0)?.id || null
+          }
+        },
+        "progress"
+      );
 
       return equipment;
     };
 
-    console.log("Total reports:", ret.reports.concat(ret.markedReports).length);
+    const requests = reports.map((r) => limit(() => checkReportAssigments(r)));
-    await Promise.allSettled(ret.reports.concat(ret.markedReports).map((r) => checkReportAssigments(r)));
+
-    ctx.res.sendSSEJson({ foundEqupments }, "reportParseDone");
+    await Promise.all(requests)
-    console.log("Parse done");
+      .then(() => res.sendSSEJson(parseResultCount, "done"))
+      .catch((error) => {
+        console.error(error);
+        res.sendSSEJson({ error: serializeError(error) }, "error");
+      })
+      .finally(() => res.res.end());
   } else {
     throw new BadRequestError();
   }
-
-  res.res.end();
 };
 
 function parseReasonCallAndJobDesc(str = "") {

router/reports/syncParsed.handler.js

@@ -1,3 +1,4 @@
+import { BadRequestError } from "#root/errors/index.js";
 import yup from "yup";
 
 var reportItemSchema = yup.object({
@@ -14,6 +15,9 @@ var reportItemSchema = yup.object({
 export default async ({ res, data }) => {
   const inputSchema = yup.object({ reports: yup.array(reportItemSchema).required() }).required();
 
-  await inputSchema.validate(data);
+  await inputSchema.validate(data).catch((err) => {
+    throw new BadRequestError(err.message);
+  });
+
   res.sendJson({ data });
 };