From 88ec289e5e90acb1e1a8298c99731884e3529f0d Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 06:28:55 +0000 Subject: [PATCH] perf: Optimize /api/upload-excel to resolve N+1 queries using pg-format bulk inserts Co-authored-by: deitaur <113350206+deitaur@users.noreply.github.com> --- __tests__/api.test.js | 24 +++++-- app.js | 143 ++++++++++++++++++++++++++++++++---------- package-lock.json | 10 +++ package.json | 1 + test_perf.js | 85 +++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 38 deletions(-) create mode 100644 test_perf.js diff --git a/__tests__/api.test.js b/__tests__/api.test.js index b81c2ad..da86eb3 100644 --- a/__tests__/api.test.js +++ b/__tests__/api.test.js @@ -1,14 +1,28 @@ const request = require('supertest'); +const http = require('http'); + +// Override IncomingMessage.prototype.isAuthenticated BEFORE app is required +http.IncomingMessage.prototype.isAuthenticated = function() { + return true; +}; +Object.defineProperty(http.IncomingMessage.prototype, 'user', { + get: function() { + return { id: 1 }; + } +}); + const { app, pool } = require('../app'); let server; -// Перед выполнением всех тестов, запускаем сервер -beforeAll(done => { - server = app.listen(done); +beforeAll(async () => { + // Setup dummy user in DB + await pool.query('CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, google_id VARCHAR(255) UNIQUE, display_name VARCHAR(255), email VARCHAR(255))'); + await pool.query('INSERT INTO users(id, google_id, display_name, email) VALUES(1, \'dummy\', \'Test\', \'test@test.com\') ON CONFLICT DO NOTHING'); + + server = app.listen(); }); -// После выполнения всех тестов, закрываем сервер и пул соединений afterAll(done => { server.close(() => { pool.end(done); @@ -29,4 +43,4 @@ describe('API Tests', () => { expect(response.statusCode).toBe(200); expect(Array.isArray(response.body)).toBe(true); }); -}); \ No newline at end of file +}); diff --git a/app.js b/app.js index 5f04342..f176c08 100644 --- a/app.js +++ b/app.js @@ -3,6 +3,7 @@ const { Pool } = require('pg'); const path = require('path'); const multer = require('multer'); const ExcelJS = require('exceljs'); +const pgFormat = require('pg-format'); const passport = require('passport'); const GoogleStrategy = require('passport-google-oauth20').Strategy; const session = require('express-session'); @@ -658,51 +659,127 @@ app.post('/api/upload-excel', upload.single('excelFile'), ensureAuthenticated, a const client = await pool.connect(); try { await client.query('BEGIN'); + + // 1. Извлекаем данные из строк + const rowData = []; for (const row of rows) { - const name = row.getCell(1).value; - const type = row.getCell(2).value; - const description = row.getCell(3).value; - const houseName = row.getCell(4).value; - const roomName = row.getCell(5).value; - const parentItemName = row.getCell(6).value; - - let houseId = null; - let roomId = null; - let parentItemId = null; - - if (houseName) { - // Ищем дом, принадлежащий текущему пользователю - houseId = await findId('houses', houseName, userId); - if (!houseId) { - // Создаем новый дом, привязывая его к пользователю - const newHouse = await client.query('INSERT INTO houses(name, user_id) VALUES($1, $2) RETURNING id', [houseName, userId]); - houseId = newHouse.rows[0].id; + rowData.push({ + name: row.getCell(1).value, + type: row.getCell(2).value, + description: row.getCell(3).value, + houseName: row.getCell(4).value, + roomName: row.getCell(5).value, + parentItemName: row.getCell(6).value + }); + } + + // Мапы для разрешения ID в памяти + const houseMap = new Map(); // name -> id + const roomMap = new Map(); // name -> id + const itemMap = new Map(); // name -> id + + // 2. Получаем все существующие дома пользователя + const existingHouses = await client.query('SELECT id, name FROM houses WHERE user_id = $1', [userId]); + existingHouses.rows.forEach(h => houseMap.set(h.name, h.id)); + + // Определяем новые дома + const newHouseNames = new Set(); + for (const row of rowData) { + if (row.houseName && !houseMap.has(row.houseName)) { + newHouseNames.add(row.houseName); + } + } + + if (newHouseNames.size > 0) { + const houseInsertData = Array.from(newHouseNames).map(name => [name, userId]); + const insertHousesQuery = pgFormat('INSERT INTO houses (name, user_id) VALUES %L RETURNING id, name', houseInsertData); + const insertedHouses = await client.query(insertHousesQuery); + insertedHouses.rows.forEach(h => houseMap.set(h.name, h.id)); + } + + // 3. Получаем все существующие комнаты пользователя + const existingRooms = await client.query('SELECT id, name, house_id FROM rooms WHERE user_id = $1', [userId]); + // Примечание: в оригинальном коде roomId искался просто по имени. Для надежности маппим по имени, + // но учитываем, что в оригинальном коде findId искал 'rooms' просто по name и user_id. + existingRooms.rows.forEach(r => roomMap.set(r.name, r.id)); + + // Определяем новые комнаты + const newRoomDataMap = new Map(); // name -> [name, houseId, userId] + for (const row of rowData) { + if (row.roomName && row.houseName) { + const houseId = houseMap.get(row.houseName); + if (!roomMap.has(row.roomName) && !newRoomDataMap.has(row.roomName)) { + newRoomDataMap.set(row.roomName, [row.roomName, houseId, userId]); } } + } + + if (newRoomDataMap.size > 0) { + const roomInsertData = Array.from(newRoomDataMap.values()); + const insertRoomsQuery = pgFormat('INSERT INTO rooms (name, house_id, user_id) VALUES %L RETURNING id, name', roomInsertData); + const insertedRooms = await client.query(insertRoomsQuery); + insertedRooms.rows.forEach(r => roomMap.set(r.name, r.id)); + } - if (roomName && houseId) { - // Ищем комнату, принадлежащую текущему пользователю - roomId = await findId('rooms', roomName, userId); - if (!roomId) { - // Создаем новую комнату, привязывая к дому и пользователю - const newRoom = await client.query('INSERT INTO rooms(name, house_id, user_id) VALUES($1, $2, $3) RETURNING id', [roomName, houseId, userId]); - roomId = newRoom.rows[0].id; + // 4. Получаем все существующие предметы пользователя + const existingItems = await client.query('SELECT id, name FROM items WHERE user_id = $1', [userId]); + existingItems.rows.forEach(i => itemMap.set(i.name, i.id)); + + // 5. Строим граф зависимостей для предметов и разрешаем итеративно + let itemsToProcess = [...rowData].filter(r => r.name); + let maxIterations = itemsToProcess.length + 1; + let iterations = 0; + + while (itemsToProcess.length > 0 && iterations < maxIterations) { + let currentGroup = []; + let remainingItems = []; + + + for (const item of itemsToProcess) { + if (!item.parentItemName || itemMap.has(item.parentItemName)) { + currentGroup.push(item); + if (item.name) { + + } + } else { + remainingItems.push(item); } } - if (parentItemName) { - // Ищем родительский предмет ТОЛЬКО среди предметов пользователя - parentItemId = await findId('items', parentItemName, userId); + if (currentGroup.length === 0) { + // Неразрешимые зависимости. Обрабатываем оставшиеся с parent_item_id = null + currentGroup = remainingItems; + remainingItems = []; } - if (name) { - // Вставляем новый предмет, привязывая его к пользователю - await client.query( - 'INSERT INTO items(name, type, description, room_id, parent_item_id, user_id) VALUES($1, $2, $3, $4, $5, $6)', - [name, type, description, roomId, parentItemId, userId] + const itemsInsertData = []; + for (const item of currentGroup) { + let roomId = null; + if (item.roomName && item.houseName) { + roomId = roomMap.get(item.roomName) || null; + } + + let parentItemId = null; + if (item.parentItemName) { + parentItemId = itemMap.get(item.parentItemName) || null; + } + + itemsInsertData.push([item.name, item.type, item.description, roomId, parentItemId, userId]); + } + + if (itemsInsertData.length > 0) { + const insertItemsQuery = pgFormat( + 'INSERT INTO items (name, type, description, room_id, parent_item_id, user_id) VALUES %L RETURNING id, name', + itemsInsertData ); + const insertedItems = await client.query(insertItemsQuery); + insertedItems.rows.forEach(i => itemMap.set(i.name, i.id)); } + + itemsToProcess = remainingItems; + iterations++; } + await client.query('COMMIT'); res.status(200).json({ message: 'Данные успешно импортированы.' }); } catch (dbError) { diff --git a/package-lock.json b/package-lock.json index b427e70..9ab5e02 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "pg": "^8.16.3", + "pg-format": "^1.0.4", "supertest": "^7.1.4" } }, @@ -4949,6 +4950,15 @@ "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", "license": "MIT" }, + "node_modules/pg-format": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", diff --git a/package.json b/package.json index b27d579..c4d3273 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "pg": "^8.16.3", + "pg-format": "^1.0.4", "supertest": "^7.1.4" } } diff --git a/test_perf.js b/test_perf.js new file mode 100644 index 0000000..0e84960 --- /dev/null +++ b/test_perf.js @@ -0,0 +1,85 @@ +const fs = require('fs'); +const path = require('path'); +const request = require('supertest'); + +const http = require('http'); + +// Override IncomingMessage.prototype.isAuthenticated +http.IncomingMessage.prototype.isAuthenticated = function() { + return true; +}; + +// Also we need to ensure req.user exists +Object.defineProperty(http.IncomingMessage.prototype, 'user', { + get: function() { + return { id: 1 }; + } +}); + +const { app, pool } = require('./app'); + +async function setup() { + await pool.query(` + CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + google_id VARCHAR(255) UNIQUE, + display_name VARCHAR(255), + email VARCHAR(255) + ); + CREATE TABLE IF NOT EXISTS houses ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS rooms ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + house_id INTEGER REFERENCES houses(id) ON DELETE CASCADE, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS items ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + type VARCHAR(255), + description TEXT, + room_id INTEGER REFERENCES rooms(id) ON DELETE SET NULL, + parent_item_id INTEGER REFERENCES items(id) ON DELETE SET NULL, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE + ); + `); + + // Create a dummy user + const res = await pool.query('INSERT INTO users(id, google_id, display_name, email) VALUES(1, $1, $2, $3) ON CONFLICT (id) DO UPDATE SET email=EXCLUDED.email RETURNING id', ['dummy_google_id', 'Test User', 'test@example.com']); + return res.rows[0].id; +} + +async function run() { + await setup(); + + // Clear tables before run + await pool.query('DELETE FROM items'); + await pool.query('DELETE FROM rooms'); + await pool.query('DELETE FROM houses'); + + console.log("Starting benchmark..."); + const start = process.hrtime.bigint(); + + const agent = request.agent(app); + const response = await agent + .post('/api/upload-excel') + .attach('excelFile', path.join(__dirname, 'test_large_5000.xlsx')); + + const end = process.hrtime.bigint(); + const durationMs = Number(end - start) / 1_000_000; + + console.log(`Status: ${response.status}`); + console.log(`Response: ${JSON.stringify(response.body)}`); + console.log(`Duration: ${durationMs.toFixed(2)} ms`); + + const itemsCount = await pool.query('SELECT count(*) FROM items'); + console.log(`Items inserted: ${itemsCount.rows[0].count}`); + + process.exit(0); +} + +run().catch(console.error);