⚡ Optimize /api/upload-excel N+1 Query Using Bulk Inserts

__tests__/api.test.js

@@ -1,14 +1,28 @@
 const request = require('supertest');
+const http = require('http');
+
+// Override IncomingMessage.prototype.isAuthenticated BEFORE app is required
+http.IncomingMessage.prototype.isAuthenticated = function() {
+    return true;
+};
+Object.defineProperty(http.IncomingMessage.prototype, 'user', {
+    get: function() {
+        return { id: 1 };
+    }
+});
+
 const { app, pool } = require('../app');
 
 let server;
 
-// Перед выполнением всех тестов, запускаем сервер
-beforeAll(done => {
-  server = app.listen(done);
+beforeAll(async () => {
+    // Setup dummy user in DB
+    await pool.query('CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, google_id VARCHAR(255) UNIQUE, display_name VARCHAR(255), email VARCHAR(255))');
+    await pool.query('INSERT INTO users(id, google_id, display_name, email) VALUES(1, \'dummy\', \'Test\', \'test@test.com\') ON CONFLICT DO NOTHING');
+
+    server = app.listen();
 });
 
-// После выполнения всех тестов, закрываем сервер и пул соединений
 afterAll(done => {
   server.close(() => {
     pool.end(done);
@@ -29,4 +43,4 @@ describe('API Tests', () => {
     expect(response.statusCode).toBe(200);
     expect(Array.isArray(response.body)).toBe(true);
   });
-});
+});

app.js

@@ -3,6 +3,7 @@ const { Pool } = require('pg');
 const path = require('path');
 const multer = require('multer');
 const ExcelJS = require('exceljs');
+const pgFormat = require('pg-format');
 const passport = require('passport');
 const GoogleStrategy = require('passport-google-oauth20').Strategy;
 const session = require('express-session');
@@ -658,51 +659,127 @@ app.post('/api/upload-excel', upload.single('excelFile'), ensureAuthenticated, a
         const client = await pool.connect();
         try {
             await client.query('BEGIN');
+
+            // 1. Извлекаем данные из строк
+            const rowData = [];
             for (const row of rows) {
-                const name = row.getCell(1).value;
-                const type = row.getCell(2).value;
-                const description = row.getCell(3).value;
-                const houseName = row.getCell(4).value;
-                const roomName = row.getCell(5).value;
-                const parentItemName = row.getCell(6).value;
-
-                let houseId = null;
-                let roomId = null;
-                let parentItemId = null;
-
-                if (houseName) {
-                    // Ищем дом, принадлежащий текущему пользователю
-                    houseId = await findId('houses', houseName, userId);
-                    if (!houseId) {
-                        // Создаем новый дом, привязывая его к пользователю
-                        const newHouse = await client.query('INSERT INTO houses(name, user_id) VALUES($1, $2) RETURNING id', [houseName, userId]);
-                        houseId = newHouse.rows[0].id;
+                rowData.push({
+                    name: row.getCell(1).value,
+                    type: row.getCell(2).value,
+                    description: row.getCell(3).value,
+                    houseName: row.getCell(4).value,
+                    roomName: row.getCell(5).value,
+                    parentItemName: row.getCell(6).value
+                });
+            }
+
+            // Мапы для разрешения ID в памяти
+            const houseMap = new Map(); // name -> id
+            const roomMap = new Map(); // name -> id
+            const itemMap = new Map(); // name -> id
+
+            // 2. Получаем все существующие дома пользователя
+            const existingHouses = await client.query('SELECT id, name FROM houses WHERE user_id = $1', [userId]);
+            existingHouses.rows.forEach(h => houseMap.set(h.name, h.id));
+
+            // Определяем новые дома
+            const newHouseNames = new Set();
+            for (const row of rowData) {
+                if (row.houseName && !houseMap.has(row.houseName)) {
+                    newHouseNames.add(row.houseName);
+                }
+            }
+
+            if (newHouseNames.size > 0) {
+                const houseInsertData = Array.from(newHouseNames).map(name => [name, userId]);
+                const insertHousesQuery = pgFormat('INSERT INTO houses (name, user_id) VALUES %L RETURNING id, name', houseInsertData);
+                const insertedHouses = await client.query(insertHousesQuery);
+                insertedHouses.rows.forEach(h => houseMap.set(h.name, h.id));
+            }
+
+            // 3. Получаем все существующие комнаты пользователя
+            const existingRooms = await client.query('SELECT id, name, house_id FROM rooms WHERE user_id = $1', [userId]);
+            // Примечание: в оригинальном коде roomId искался просто по имени. Для надежности маппим по имени,
+            // но учитываем, что в оригинальном коде findId искал 'rooms' просто по name и user_id.
+            existingRooms.rows.forEach(r => roomMap.set(r.name, r.id));
+
+            // Определяем новые комнаты
+            const newRoomDataMap = new Map(); // name -> [name, houseId, userId]
+            for (const row of rowData) {
+                if (row.roomName && row.houseName) {
+                    const houseId = houseMap.get(row.houseName);
+                    if (!roomMap.has(row.roomName) && !newRoomDataMap.has(row.roomName)) {
+                        newRoomDataMap.set(row.roomName, [row.roomName, houseId, userId]);
                     }
                 }
+            }
+
+            if (newRoomDataMap.size > 0) {
+                const roomInsertData = Array.from(newRoomDataMap.values());
+                const insertRoomsQuery = pgFormat('INSERT INTO rooms (name, house_id, user_id) VALUES %L RETURNING id, name', roomInsertData);
+                const insertedRooms = await client.query(insertRoomsQuery);
+                insertedRooms.rows.forEach(r => roomMap.set(r.name, r.id));
+            }
 
-                if (roomName && houseId) {
-                    // Ищем комнату, принадлежащую текущему пользователю
-                    roomId = await findId('rooms', roomName, userId);
-                    if (!roomId) {
-                        // Создаем новую комнату, привязывая к дому и пользователю
-                        const newRoom = await client.query('INSERT INTO rooms(name, house_id, user_id) VALUES($1, $2, $3) RETURNING id', [roomName, houseId, userId]);
-                        roomId = newRoom.rows[0].id;
+            // 4. Получаем все существующие предметы пользователя
+            const existingItems = await client.query('SELECT id, name FROM items WHERE user_id = $1', [userId]);
+            existingItems.rows.forEach(i => itemMap.set(i.name, i.id));
+
+            // 5. Строим граф зависимостей для предметов и разрешаем итеративно
+            let itemsToProcess = [...rowData].filter(r => r.name);
+            let maxIterations = itemsToProcess.length + 1;
+            let iterations = 0;
+
+            while (itemsToProcess.length > 0 && iterations < maxIterations) {
+                let currentGroup = [];
+                let remainingItems = [];
+
+
+                for (const item of itemsToProcess) {
+                    if (!item.parentItemName || itemMap.has(item.parentItemName)) {
+                        currentGroup.push(item);
+                        if (item.name) {
+
+                        }
+                    } else {
+                        remainingItems.push(item);
                     }
                 }
 
-                if (parentItemName) {
-                    // Ищем родительский предмет ТОЛЬКО среди предметов пользователя
-                    parentItemId = await findId('items', parentItemName, userId);
+                if (currentGroup.length === 0) {
+                    // Неразрешимые зависимости. Обрабатываем оставшиеся с parent_item_id = null
+                    currentGroup = remainingItems;
+                    remainingItems = [];
                 }
 
-                if (name) {
-                    // Вставляем новый предмет, привязывая его к пользователю
-                    await client.query(
-                        'INSERT INTO items(name, type, description, room_id, parent_item_id, user_id) VALUES($1, $2, $3, $4, $5, $6)',
-                        [name, type, description, roomId, parentItemId, userId]
+                const itemsInsertData = [];
+                for (const item of currentGroup) {
+                    let roomId = null;
+                    if (item.roomName && item.houseName) {
+                        roomId = roomMap.get(item.roomName) || null;
+                    }
+
+                    let parentItemId = null;
+                    if (item.parentItemName) {
+                        parentItemId = itemMap.get(item.parentItemName) || null;
+                    }
+
+                    itemsInsertData.push([item.name, item.type, item.description, roomId, parentItemId, userId]);
+                }
+
+                if (itemsInsertData.length > 0) {
+                    const insertItemsQuery = pgFormat(
+                        'INSERT INTO items (name, type, description, room_id, parent_item_id, user_id) VALUES %L RETURNING id, name',
+                        itemsInsertData
                     );
+                    const insertedItems = await client.query(insertItemsQuery);
+                    insertedItems.rows.forEach(i => itemMap.set(i.name, i.id));
                 }
+
+                itemsToProcess = remainingItems;
+                iterations++;
             }
+
             await client.query('COMMIT');
             res.status(200).json({ message: 'Данные успешно импортированы.' });
         } catch (dbError) {

package.json

@@ -21,6 +21,7 @@
     "passport": "^0.7.0",
     "passport-google-oauth20": "^2.0.0",
     "pg": "^8.16.3",
+    "pg-format": "^1.0.4",
     "supertest": "^7.1.4"
   }
 }

test_perf.js

@@ -0,0 +1,85 @@
+const fs = require('fs');
+const path = require('path');
+const request = require('supertest');
+
+const http = require('http');
+
+// Override IncomingMessage.prototype.isAuthenticated
+http.IncomingMessage.prototype.isAuthenticated = function() {
+    return true;
+};
+
+// Also we need to ensure req.user exists
+Object.defineProperty(http.IncomingMessage.prototype, 'user', {
+    get: function() {
+        return { id: 1 };
+    }
+});
+
+const { app, pool } = require('./app');
+
+async function setup() {
+    await pool.query(`
+        CREATE TABLE IF NOT EXISTS users (
+            id SERIAL PRIMARY KEY,
+            google_id VARCHAR(255) UNIQUE,
+            display_name VARCHAR(255),
+            email VARCHAR(255)
+        );
+        CREATE TABLE IF NOT EXISTS houses (
+            id SERIAL PRIMARY KEY,
+            name VARCHAR(255) NOT NULL,
+            user_id INTEGER REFERENCES users(id) ON DELETE CASCADE
+        );
+        CREATE TABLE IF NOT EXISTS rooms (
+            id SERIAL PRIMARY KEY,
+            name VARCHAR(255) NOT NULL,
+            house_id INTEGER REFERENCES houses(id) ON DELETE CASCADE,
+            user_id INTEGER REFERENCES users(id) ON DELETE CASCADE
+        );
+        CREATE TABLE IF NOT EXISTS items (
+            id SERIAL PRIMARY KEY,
+            name VARCHAR(255) NOT NULL,
+            type VARCHAR(255),
+            description TEXT,
+            room_id INTEGER REFERENCES rooms(id) ON DELETE SET NULL,
+            parent_item_id INTEGER REFERENCES items(id) ON DELETE SET NULL,
+            user_id INTEGER REFERENCES users(id) ON DELETE CASCADE
+        );
+    `);
+
+    // Create a dummy user
+    const res = await pool.query('INSERT INTO users(id, google_id, display_name, email) VALUES(1, $1, $2, $3) ON CONFLICT (id) DO UPDATE SET email=EXCLUDED.email RETURNING id', ['dummy_google_id', 'Test User', 'test@example.com']);
+    return res.rows[0].id;
+}
+
+async function run() {
+    await setup();
+
+    // Clear tables before run
+    await pool.query('DELETE FROM items');
+    await pool.query('DELETE FROM rooms');
+    await pool.query('DELETE FROM houses');
+
+    console.log("Starting benchmark...");
+    const start = process.hrtime.bigint();
+
+    const agent = request.agent(app);
+    const response = await agent
+        .post('/api/upload-excel')
+        .attach('excelFile', path.join(__dirname, 'test_large_5000.xlsx'));
+
+    const end = process.hrtime.bigint();
+    const durationMs = Number(end - start) / 1_000_000;
+
+    console.log(`Status: ${response.status}`);
+    console.log(`Response: ${JSON.stringify(response.body)}`);
+    console.log(`Duration: ${durationMs.toFixed(2)} ms`);
+
+    const itemsCount = await pool.query('SELECT count(*) FROM items');
+    console.log(`Items inserted: ${itemsCount.rows[0].count}`);
+
+    process.exit(0);
+}
+
+run().catch(console.error);