mirror of
https://github.com/sergeyyarkov/asu_tools_service.git
synced 2026-10-11 10:29:22 +03:00
refactor
This commit is contained in:
parent
ad361727b1
commit
433a412985
@ -1,22 +1,10 @@
|
||||
import { BadRequestError, ServiceUnavailableError } from "#root/http-server/index.js";
|
||||
import db from "#root/db.js";
|
||||
import xlsx from "xlsx";
|
||||
import pLimit from "p-limit";
|
||||
import { clients } from "#root/index.js";
|
||||
import { reportsService } from "#services/index.js";
|
||||
import { serializeError } from "serialize-error";
|
||||
|
||||
const sheetName = "Отчеты";
|
||||
const MAX_CONCURRENT_PARSE_CLIENT = Number.parseInt(process.env.REPORT_CONCURRENT_PARSE_CLIENTS || "3", 10);
|
||||
|
||||
/** @type {xlsx.ParsingOptions} */
|
||||
const xlsxParseOptions = {
|
||||
sheets: sheetName,
|
||||
cellHTML: false,
|
||||
cellFormula: false,
|
||||
cellDates: true,
|
||||
dateNF: "yyyy-mm-dd"
|
||||
};
|
||||
|
||||
/**
|
||||
* URL: /api/reports_excel_file_parse
|
||||
* Method: POST
|
||||
@ -28,12 +16,8 @@ const xlsxParseOptions = {
|
||||
export default async (ctx) => {
|
||||
const { res, searchParams } = ctx;
|
||||
const [, files] = ctx.data;
|
||||
const limit = pLimit(10);
|
||||
const clientId = searchParams.get("clientId");
|
||||
const excelFilePath = files?.report?.at(0)?.filepath;
|
||||
const reports = [];
|
||||
let isSseClientClosed = false;
|
||||
let parseResultCount = { equipments: 0, applicants: 0, executors: 0, reportsFullfilled: 0 };
|
||||
const excelFilepath = files?.report?.at(0)?.filepath;
|
||||
|
||||
if (!ctx.data) {
|
||||
throw new BadRequestError("Cannot proceed uploaded file");
|
||||
@ -61,82 +45,24 @@ export default async (ctx) => {
|
||||
throw new ServiceUnavailableError("Your request on parsing is processing");
|
||||
}
|
||||
|
||||
sseClientCtx.res.res.on("close", () => (isSseClientClosed = true));
|
||||
/** @param {ReportModel} report */
|
||||
const onParseProgress = async (report) => sseClientCtx.res.sendSSEJson(report, "progress");
|
||||
|
||||
if (excelFilePath) {
|
||||
const cancelAnalyzeCond = () => Boolean(sseClientCtx.local?.isClosed);
|
||||
|
||||
if (excelFilepath) {
|
||||
try {
|
||||
const workbook = xlsx.readFile(excelFilePath, xlsxParseOptions);
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
const reports = reportsService.parseExcelFile(excelFilepath);
|
||||
const { result, reports: reportsPromises } = reportsService.analyzeParseResult(
|
||||
reports,
|
||||
cancelAnalyzeCond,
|
||||
onParseProgress
|
||||
);
|
||||
|
||||
if (!sheet) throw new Error("Invalid reports workbook.", { cause: { statusCode: 400 } });
|
||||
|
||||
const data = xlsx.utils.sheet_to_json(sheet, { raw: false, blankrows: true, header: "A" });
|
||||
|
||||
res.sendJson({ message: "OK" });
|
||||
sseClientCtx.local.isParsing = true;
|
||||
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const cols = Object.keys(data[i]);
|
||||
|
||||
if (i === 0) continue;
|
||||
|
||||
/** Несинхронизированные помеченные отчеты */
|
||||
if ("G" in data[i] && data[i]["G"] === "r") {
|
||||
reports.push({
|
||||
date: data[i][cols[0]],
|
||||
equipment: data[i][cols[1]],
|
||||
reason_call: data[i][cols[3]].split("\r\r\n")[1] || "",
|
||||
job_description: data[i][cols[4]].split("\r\r\n")[1] || "",
|
||||
root_cause: data[i]["F"] || "",
|
||||
applicantName: data[i][cols[3]].split("\r\r\n")[0] || "",
|
||||
executorNames: data[i][cols[4]].split("\r\r\n")[0] || "",
|
||||
rowNum: data[i].__rowNum__ + 1,
|
||||
isMarked: true
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
/** Остальные отчеты */
|
||||
reports.push({
|
||||
date: data[i][cols[0]],
|
||||
equipment: data[i][cols[1]],
|
||||
reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(),
|
||||
job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(),
|
||||
root_cause: data[i]["F"] || "",
|
||||
applicantName: parseNames(data[i][cols[3]]).trim(),
|
||||
executorNames: parseNames(data[i][cols[4]]).trim(),
|
||||
rowNum: data[i].__rowNum__ + 1
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
sseClientCtx.local.isParsing = false;
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
const requests = reports.map((report) =>
|
||||
limit(() => {
|
||||
return checkReportAssigments(
|
||||
report,
|
||||
(checkedReport) => {
|
||||
const isEquipmentLinked = checkedReport.equipment instanceof Object;
|
||||
const isApplicantLinked = checkedReport.applicant instanceof Object;
|
||||
const isExecutorsLinked = checkedReport.executors && checkedReport.executors.length !== 0;
|
||||
|
||||
if (isEquipmentLinked) parseResultCount.equipments++;
|
||||
if (isApplicantLinked) parseResultCount.applicants++;
|
||||
if (isExecutorsLinked) parseResultCount.executors += checkedReport.executors?.length || 0;
|
||||
if (isEquipmentLinked && isApplicantLinked && isExecutorsLinked) parseResultCount.reportsFullfilled++;
|
||||
|
||||
sseClientCtx.res.sendSSEJson(checkedReport, "progress");
|
||||
},
|
||||
isSseClientClosed
|
||||
);
|
||||
})
|
||||
);
|
||||
await Promise.all(requests);
|
||||
sseClientCtx.res.sendSSEJson(parseResultCount, "done");
|
||||
res.sendJson({ message: "OK" });
|
||||
await Promise.all(reportsPromises);
|
||||
sseClientCtx.res.sendSSEJson(result, "done");
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
sseClientCtx.res.sendSSEJson({ error: serializeError(error) }, "parseError");
|
||||
@ -147,92 +73,3 @@ export default async (ctx) => {
|
||||
throw new BadRequestError();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {ReportModel} report
|
||||
* @param {(report: ReportModel) => void} callback
|
||||
* @param {boolean} rejectCond
|
||||
*/
|
||||
async function checkReportAssigments(report, callback, rejectCond) {
|
||||
if (rejectCond) return Promise.reject(`Report check is rejected`);
|
||||
|
||||
const executorSurnames = extractSurnamesFromExecutorField(report.executorNames);
|
||||
|
||||
const equipment = await db.request()
|
||||
.query(`select t1.id, t1.name, t1.location_id, t3.name as base_location_name, t2.name as location_name, t2.base_location_id
|
||||
from dbo.asu_system_api_subsystemlist as t1
|
||||
left join dbo.asu_system_api_location as t2 on t2.id = t1.location_id
|
||||
left join dbo.asu_system_api_baselocation as t3 on t3.id = t2.base_location_id
|
||||
where t1.name like N'%${report.equipment}%';
|
||||
`);
|
||||
|
||||
const applicant = await db
|
||||
.request()
|
||||
.query(
|
||||
`select id, name from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';`
|
||||
);
|
||||
|
||||
/** @type {ExecutorModel[]} */
|
||||
const executors = [];
|
||||
|
||||
for (const eSurname of executorSurnames) {
|
||||
if (eSurname) {
|
||||
const executor = await db
|
||||
.request()
|
||||
.query(
|
||||
`select id, surname + ' ' + name + ' ' + patronymic as fullname from user_user where surname like N'%${eSurname}%';`
|
||||
);
|
||||
if (executor.recordset.length !== 0) {
|
||||
executors.push({ id: executor.recordset[0].id, fullname: executor.recordset[0].fullname });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const hasEquipmentLink = equipment.recordset.length !== 0;
|
||||
const hasApplicantLink = applicant.recordset.length !== 0;
|
||||
|
||||
const location =
|
||||
hasEquipmentLink && equipment.recordset[0].location_id
|
||||
? {
|
||||
id: equipment.recordset[0].location_id,
|
||||
name: equipment.recordset[0].location_name,
|
||||
base_location_name: equipment.recordset[0].base_location_name,
|
||||
base_location: equipment.recordset[0].base_location_id
|
||||
}
|
||||
: null;
|
||||
|
||||
callback({
|
||||
...report,
|
||||
executors,
|
||||
equipment: hasEquipmentLink ? { id: equipment.recordset[0].id, name: equipment.recordset[0].name, location } : null,
|
||||
applicant: hasApplicantLink ? { id: applicant.recordset[0].id, name: applicant.recordset[0].name } : null
|
||||
});
|
||||
}
|
||||
|
||||
function parseReasonCallAndJobDesc(str = "") {
|
||||
const splitted = splitLongSpacedSentence(str);
|
||||
if (splitted.length > 1) return splitted.slice(1).join(" ");
|
||||
return str.split("\r\n").slice(1).join(" ");
|
||||
}
|
||||
|
||||
function parseNames(str = "") {
|
||||
const splitted = splitLongSpacedSentence(str);
|
||||
if (splitted.length > 1) return splitted[0];
|
||||
return str.split("\r\n")[0];
|
||||
}
|
||||
|
||||
/** @param {string} str */
|
||||
function extractSurnamesFromExecutorField(str) {
|
||||
return str
|
||||
.replaceAll(",", "")
|
||||
.split(/(?:.\..\.)|(?:.\..\s)|(?:.\.\s.\.)|(?:.\..)/g)
|
||||
.map((s) => s.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} str
|
||||
* @param {number} min
|
||||
*/
|
||||
function splitLongSpacedSentence(str, min = 5) {
|
||||
return str.split(new RegExp(`\\s{${min},}`));
|
||||
}
|
||||
|
||||
@ -13,6 +13,7 @@ export default (ctx) => {
|
||||
const clientId = crypto.randomUUID();
|
||||
|
||||
clients.set(clientId, ctx);
|
||||
ctx.local.isClosed = false;
|
||||
|
||||
res.res.setHeaders(
|
||||
new Headers({
|
||||
@ -25,6 +26,7 @@ export default (ctx) => {
|
||||
res.sendSSEJson({ message: "Connected!", clientId }, "data");
|
||||
|
||||
res.res.on("close", () => {
|
||||
ctx.local.isClosed = true;
|
||||
clients.delete(clientId);
|
||||
res.res.end();
|
||||
});
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { serializeError } from "serialize-error";
|
||||
import formidable from "formidable";
|
||||
import { BadRequestError, HttpError, MethodNotAllowedError, NotFoundError } from "./errors.js";
|
||||
import * as utils from "./utils/index.js";
|
||||
import * as strUtils from "./utils/string.js";
|
||||
|
||||
/**
|
||||
* @returns {import("./types/http-server.js").HttpRouter}
|
||||
@ -82,11 +82,10 @@ export function createRouter() {
|
||||
prefixNameGroup = prevPrefix;
|
||||
},
|
||||
define(method, pathname, handler, options = {}) {
|
||||
if (!utils.matchRoutePathname(pathname)) throw new Error(`Incorrect pathname "${pathname}"`);
|
||||
if (prefixNameGroup) pathname = utils.formatRoutePathname(prefixNameGroup + pathname);
|
||||
if (!strUtils.matchRoutePathname(pathname)) throw new Error(`Incorrect pathname "${pathname}"`);
|
||||
if (prefixNameGroup) pathname = strUtils.formatRoutePathname(prefixNameGroup + pathname);
|
||||
/** @type {import("./types/http-server.js").HttpRoute} */
|
||||
routes.push({ method, pattern: new URLPattern({ pathname }), handler, options });
|
||||
console.log("defined:", pathname);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
2
index.js
2
index.js
@ -1,7 +1,7 @@
|
||||
import path from "node:path";
|
||||
import fsp from "node:fs/promises";
|
||||
import { createServer } from "./http-server/index.js";
|
||||
import { reportsHandlers } from "./handlers/index.js";
|
||||
import { reportsHandlers } from "#handlers/index.js";
|
||||
import packageJson from "./package.json" with { type: "json" };
|
||||
import db from "./db.js";
|
||||
|
||||
|
||||
@ -8,7 +8,9 @@
|
||||
"module": "nodenext",
|
||||
"resolveJsonModule": true,
|
||||
"paths": {
|
||||
"#root/*": ["./*"]
|
||||
"#root/*": ["./*"],
|
||||
"#handlers/*": ["./handlers/*"],
|
||||
"#services/*": ["./services/*"]
|
||||
}
|
||||
},
|
||||
"exclude": ["./http-server/types/*.d.ts"]
|
||||
|
||||
1
models.d.ts
vendored
1
models.d.ts
vendored
@ -16,6 +16,7 @@ type ReportModel = {
|
||||
applicantName: string;
|
||||
executorNames: string;
|
||||
isMarked?: boolean;
|
||||
rowNum: number;
|
||||
};
|
||||
|
||||
type ExecutorModel = { id: string; fullname: string };
|
||||
|
||||
@ -22,7 +22,9 @@
|
||||
"yup": "^1.7.1"
|
||||
},
|
||||
"imports": {
|
||||
"#root/*": "./*"
|
||||
"#root/*": "./*",
|
||||
"#handlers/*": "./handlers/*",
|
||||
"#services/*": "./services/*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^10.0.1",
|
||||
|
||||
1
services/index.js
Normal file
1
services/index.js
Normal file
@ -0,0 +1 @@
|
||||
export * from "./reports.service.js";
|
||||
195
services/reports.service.js
Normal file
195
services/reports.service.js
Normal file
@ -0,0 +1,195 @@
|
||||
import xlsx from "xlsx";
|
||||
import pLimit from "p-limit";
|
||||
import db from "#root/db.js";
|
||||
|
||||
const sheetName = "Отчеты";
|
||||
|
||||
/** @type {xlsx.ParsingOptions} */
|
||||
const xlsxParseOptions = {
|
||||
sheets: sheetName,
|
||||
cellHTML: false,
|
||||
cellFormula: false,
|
||||
cellDates: true,
|
||||
dateNF: "yyyy-mm-dd"
|
||||
};
|
||||
|
||||
export const reportsService = {
|
||||
/**
|
||||
* Читает файл Excel и возвращает данные о репорте в массиве
|
||||
*
|
||||
* @param {string} filepath
|
||||
* @returns {ReportModel[]}
|
||||
*/
|
||||
parseExcelFile(filepath) {
|
||||
const workbook = xlsx.readFile(filepath, xlsxParseOptions);
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
|
||||
/** @type {ReportModel[]} */
|
||||
const reports = [];
|
||||
|
||||
if (!sheet) throw new Error("Invalid reports workbook.");
|
||||
|
||||
const data = xlsx.utils.sheet_to_json(sheet, { raw: false, blankrows: true, header: "A" });
|
||||
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const cols = Object.keys(data[i]);
|
||||
|
||||
if (i === 0) continue;
|
||||
|
||||
/** Несинхронизированные помеченные отчеты */
|
||||
if ("G" in data[i] && data[i]["G"] === "r") {
|
||||
reports.push({
|
||||
date: data[i][cols[0]],
|
||||
equipment: data[i][cols[1]],
|
||||
reason_call: data[i][cols[3]].split("\r\r\n")[1] || "",
|
||||
job_description: data[i][cols[4]].split("\r\r\n")[1] || "",
|
||||
root_cause: data[i]["F"] || "",
|
||||
applicantName: data[i][cols[3]].split("\r\r\n")[0] || "",
|
||||
executorNames: data[i][cols[4]].split("\r\r\n")[0] || "",
|
||||
rowNum: data[i].__rowNum__ + 1,
|
||||
isMarked: true
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
/** Остальные отчеты */
|
||||
reports.push({
|
||||
date: data[i][cols[0]],
|
||||
equipment: data[i][cols[1]],
|
||||
reason_call: parseReasonCallAndJobDesc(data[i][cols[3]]).trim(),
|
||||
job_description: parseReasonCallAndJobDesc(data[i][cols[4]]).trim(),
|
||||
root_cause: data[i]["F"] || "",
|
||||
applicantName: parseNames(data[i][cols[3]]).trim(),
|
||||
executorNames: parseNames(data[i][cols[4]]).trim(),
|
||||
rowNum: data[i].__rowNum__ + 1
|
||||
});
|
||||
}
|
||||
|
||||
return reports;
|
||||
},
|
||||
|
||||
/**
|
||||
* Проверяет отчет на наличие связей с БД АСУ и возвращает
|
||||
* этот отчет с устанновленными связями или без них
|
||||
*
|
||||
* @param {ReportModel} report
|
||||
* @param {() => boolean} cancelCondFn
|
||||
* @returns {Promise<ReportModel>}
|
||||
*/
|
||||
async checkReportLinks(report, cancelCondFn) {
|
||||
const executorSurnames = extractSurnamesFromExecutorField(report.executorNames);
|
||||
if (cancelCondFn()) return Promise.reject();
|
||||
|
||||
const equipment = await db.request()
|
||||
.query(`select t1.id, t1.name, t1.location_id, t3.name as base_location_name, t2.name as location_name, t2.base_location_id
|
||||
from dbo.asu_system_api_subsystemlist as t1
|
||||
left join dbo.asu_system_api_location as t2 on t2.id = t1.location_id
|
||||
left join dbo.asu_system_api_baselocation as t3 on t3.id = t2.base_location_id
|
||||
where t1.name like N'%${report.equipment}%';
|
||||
`);
|
||||
|
||||
const applicant = await db
|
||||
.request()
|
||||
.query(
|
||||
`select id, name from dbo.gpp_report_api_applicant where name like N'${report.applicantName.split(" ")[0]}%';`
|
||||
);
|
||||
|
||||
/** @type {ExecutorModel[]} */
|
||||
const executors = [];
|
||||
|
||||
for (const eSurname of executorSurnames) {
|
||||
if (eSurname) {
|
||||
const executor = await db
|
||||
.request()
|
||||
.query(
|
||||
`select id, surname + ' ' + name + ' ' + patronymic as fullname from user_user where surname like N'%${eSurname}%';`
|
||||
);
|
||||
if (executor.recordset.length !== 0) {
|
||||
executors.push({ id: executor.recordset[0].id, fullname: executor.recordset[0].fullname });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const hasEquipmentLink = equipment.recordset.length !== 0;
|
||||
const hasApplicantLink = applicant.recordset.length !== 0;
|
||||
|
||||
const location =
|
||||
hasEquipmentLink && equipment.recordset[0].location_id
|
||||
? {
|
||||
id: equipment.recordset[0].location_id,
|
||||
name: equipment.recordset[0].location_name,
|
||||
base_location_name: equipment.recordset[0].base_location_name,
|
||||
base_location: equipment.recordset[0].base_location_id
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
...report,
|
||||
executors,
|
||||
equipment: hasEquipmentLink
|
||||
? { id: equipment.recordset[0].id, name: equipment.recordset[0].name, location }
|
||||
: null,
|
||||
applicant: hasApplicantLink ? { id: applicant.recordset[0].id, name: applicant.recordset[0].name } : null
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Проверяет связи отчетов с БД и возвращает результат
|
||||
*
|
||||
* @param {ReportModel[]} reports
|
||||
* @param {() => boolean} cancelCondFn
|
||||
* @param {(report: ReportModel) => Promise<void> | void} [progressCb]
|
||||
*/
|
||||
analyzeParseResult(reports, cancelCondFn, progressCb) {
|
||||
const limit = pLimit(10);
|
||||
const result = { reportsFullfilled: 0 };
|
||||
|
||||
return {
|
||||
result,
|
||||
reports: reports.map((report) =>
|
||||
limit(async () => {
|
||||
const checkedReport = await this.checkReportLinks(report, cancelCondFn);
|
||||
const isEquipmentLinked = checkedReport.equipment instanceof Object;
|
||||
const isApplicantLinked = checkedReport.applicant instanceof Object;
|
||||
const isExecutorsLinked = checkedReport.executors && checkedReport.executors.length !== 0;
|
||||
|
||||
if (isEquipmentLinked && isApplicantLinked && isExecutorsLinked) {
|
||||
result.reportsFullfilled++;
|
||||
}
|
||||
|
||||
progressCb && (await progressCb(checkedReport));
|
||||
return checkedReport;
|
||||
})
|
||||
)
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
function parseReasonCallAndJobDesc(str = "") {
|
||||
const splitted = splitLongSpacedSentence(str);
|
||||
if (splitted.length > 1) return splitted.slice(1).join(" ");
|
||||
return str.split("\r\n").slice(1).join(" ");
|
||||
}
|
||||
|
||||
function parseNames(str = "") {
|
||||
const splitted = splitLongSpacedSentence(str);
|
||||
if (splitted.length > 1) return splitted[0];
|
||||
return str.split("\r\n")[0];
|
||||
}
|
||||
|
||||
/** @param {string} str */
|
||||
function extractSurnamesFromExecutorField(str) {
|
||||
return str
|
||||
.replaceAll(",", "")
|
||||
.split(/(?:.\..\.)|(?:.\..\s)|(?:.\.\s.\.)|(?:.\..)/g)
|
||||
.map((s) => s.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} str
|
||||
* @param {number} min
|
||||
*/
|
||||
function splitLongSpacedSentence(str, min = 5) {
|
||||
return str.split(new RegExp(`\\s{${min},}`));
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user