From faeb0323ba32eb400c2797f58ff45f01553213d0 Mon Sep 17 00:00:00 2001 From: jzitnik-dev Date: Wed, 11 Feb 2026 10:51:02 +0100 Subject: [PATCH] refactor: Some path refactoring --- .gitignore | 6 +++--- Dockerfile | 12 ++---------- scrape/parse.ts | 1 - scrape/parse/v1_v2.ts | 4 ++-- scrape/parse/v3.ts | 4 ++-- scrape/scraper.ts | 32 ++++++++++++++++++-------------- server.ts | 11 +++++++---- tests/test.ts | 6 ++++++ 8 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index a51b950..09480d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ node_modules volume/browser -db -downloads -errors +volume/db +volume/downloads +volume/errors dist # Web diff --git a/Dockerfile b/Dockerfile index 543dee9..5bbd157 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,15 @@ -# Use official Node.js image as base FROM node:18 -# Create app directory WORKDIR /usr/src/app -# Copy package.json and package-lock.json (if available) COPY package*.json ./ -# Install dependencies -RUN npm install +RUN npm ci -# Build RUN npm run build -# Copy app source code COPY . . -# Expose the port your app runs on (optional, depends on your app) EXPOSE 3000 -# Start the app -CMD ["npm", "start"] +CMD ["npm", "run", "serve"] diff --git a/scrape/parse.ts b/scrape/parse.ts index 326df04..97bb810 100644 --- a/scrape/parse.ts +++ b/scrape/parse.ts @@ -15,7 +15,6 @@ import parseV1V2 from "./parse/v1_v2.js"; import parseV3 from "./parse/v3.js"; - export default async function parseThisShit(downloadedFilePath: string) { await parseV1V2(downloadedFilePath); await parseV3(downloadedFilePath); diff --git a/scrape/parse/v1_v2.ts b/scrape/parse/v1_v2.ts index c406a4d..8947ee5 100644 --- a/scrape/parse/v1_v2.ts +++ b/scrape/parse/v1_v2.ts @@ -253,7 +253,7 @@ export default async function parseV1V2(downloadedFilePath: string) { } } - fs.writeFileSync("db/v2.json", JSON.stringify(data, null, 2)); + fs.writeFileSync("volume/db/v2.json", JSON.stringify(data, null, 2)); // Modify the data for v1 const copy = JSON.parse(JSON.stringify(data)); @@ -275,7 +275,7 @@ export default async function parseV1V2(downloadedFilePath: string) { }); }); - fs.writeFileSync("db/v1.json", JSON.stringify(copy, null, 2)) + fs.writeFileSync("volume/db/v1.json", JSON.stringify(copy, null, 2)) } //parseV1V2("db/current.xlsx") diff --git a/scrape/parse/v3.ts b/scrape/parse/v3.ts index dd0bbee..09b41a0 100644 --- a/scrape/parse/v3.ts +++ b/scrape/parse/v3.ts @@ -147,7 +147,7 @@ export default async function parseV3(downloadedFilePath: string) { schedule, }; - fs.writeFileSync("db/v3.json", JSON.stringify(data, null, 2)); + fs.writeFileSync("volume/db/v3.json", JSON.stringify(data, null, 2)); } // @@ -424,4 +424,4 @@ function formatNowTime() { ); } -parseV3("db/current.xlsx") +//parseV3("db/current.xlsx") diff --git a/scrape/scraper.ts b/scrape/scraper.ts index 8899af7..2bd1473 100644 --- a/scrape/scraper.ts +++ b/scrape/scraper.ts @@ -21,12 +21,16 @@ import 'dotenv/config'; const EMAIL = process.env.EMAIL; const PASSWORD = process.env.PASSWORD; const SHAREPOINT_URL = process.env.SHAREPOINT_URL || 'https://spsejecnacz.sharepoint.com/:x:/s/nastenka/ESy19K245Y9BouR5ksciMvgBu3Pn_9EaT0fpP8R6MrkEmg'; -const VOLUME_PATH = path.resolve('./volume/browser'); + +const VOLUME_PATH = path.resolve("./volume/browser"); +const DOWNLOAD_FOLDER = path.resolve("./volume/downloads"); +const ERROR_FOLDER = path.resolve("./volume/errors") +const DB_FOLDER = path.resolve("./volume/db"); async function clearDownloadsFolder() { try { - await fs.promises.rm('./downloads', { recursive: true, force: true }); - await fs.promises.mkdir('./downloads'); + await fs.promises.rm(DOWNLOAD_FOLDER, { recursive: true, force: true }); + await fs.promises.mkdir(DOWNLOAD_FOLDER); } catch (err) { console.error('Error:', err); } @@ -34,28 +38,27 @@ async function clearDownloadsFolder() { async function handleError(page: Page, err: any) { try { - const errorsDir = path.resolve('./errors'); - if (!fs.existsSync(errorsDir)) fs.mkdirSync(errorsDir); + if (!fs.existsSync(ERROR_FOLDER)) fs.mkdirSync(ERROR_FOLDER); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filePath = path.join(errorsDir, `error-${timestamp}.png`); + const filePath = path.join(ERROR_FOLDER, `error-${timestamp}.png`); // @ts-ignore await page.screenshot({ path: filePath, fullPage: true }); console.error(`❌ Error occurred. Screenshot saved: ${filePath}`); // Keep only last 10 screenshots - const files = fs.readdirSync(errorsDir) + const files = fs.readdirSync(ERROR_FOLDER) .map(f => ({ name: f, - time: fs.statSync(path.join(errorsDir, f)).mtime.getTime() + time: fs.statSync(path.join(ERROR_FOLDER, f)).mtime.getTime() })) .sort((a, b) => b.time - a.time); if (files.length > 10) { const oldFiles = files.slice(10); for (const f of oldFiles) { - fs.unlinkSync(path.join(errorsDir, f.name)); + fs.unlinkSync(path.join(ERROR_FOLDER, f.name)); } } } catch (screenshotErr) { @@ -75,13 +78,12 @@ async function handleError(page: Page, err: any) { const pages = await browser.pages(); page = pages[0]; - const downloadPath = path.resolve('./downloads'); - if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath); + if (!fs.existsSync(DOWNLOAD_FOLDER)) fs.mkdirSync(DOWNLOAD_FOLDER); const client = await page.createCDPSession(); await client.send('Page.setDownloadBehavior', { behavior: 'allow', - downloadPath: downloadPath, + downloadPath: DOWNLOAD_FOLDER, }); await page.goto(SHAREPOINT_URL, { waitUntil: 'networkidle2' }); @@ -182,14 +184,16 @@ async function handleError(page: Page, err: any) { return files.length ? path.join(dir, files[0].name) : null; } - const downloadedFilePath = getNewestFile(downloadPath); + const downloadedFilePath = getNewestFile(DOWNLOAD_FOLDER); if (!downloadedFilePath) { throw new Error('No XLSX file found in download folder'); } console.log('Waiting for file:', downloadedFilePath); await waitForFile(downloadedFilePath); - await fs.promises.cp(downloadedFilePath, "db/current.xlsx"); + if (!fs.existsSync(DB_FOLDER)) fs.mkdirSync(DB_FOLDER); + + await fs.promises.cp(downloadedFilePath, path.join(DB_FOLDER, "current.xlsx")); await parseThisShit(downloadedFilePath); await clearDownloadsFolder(); diff --git a/server.ts b/server.ts index 6f6f546..15c2189 100644 --- a/server.ts +++ b/server.ts @@ -20,6 +20,9 @@ import { getCurrentInterval } from "./scheduleRules.js"; import bodyParser from "body-parser"; import cors from "cors"; +const DB_FOLDER = path.join(process.cwd(), "volume", "db"); +const WEB_FOLDER = path.join(process.cwd(), "web", "public"); + const VERSIONS = ["v1", "v2", "v3"]; const PORT = process.env.PORT || 3000; @@ -39,9 +42,9 @@ app.get('/', async (req: Request, res: Response) => { const isBrowser = /Mozilla|Chrome|Firefox|Safari|Edg/.test(userAgent); if (isBrowser) { - res.sendFile(path.join(process.cwd(), "web", "public", "index.html")); + res.sendFile(path.join(WEB_FOLDER, "index.html")); } else { - const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8"); + const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8"); const data = JSON.parse(dataStr); data["status"]["currentUpdateSchedule"] = getCurrentInterval(); @@ -51,7 +54,7 @@ app.get('/', async (req: Request, res: Response) => { }); app.get('/versioned/v1', async (_: Request, res: Response) => { - const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8"); + const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8"); const data = JSON.parse(dataStr); data["status"]["currentUpdateSchedule"] = getCurrentInterval(); @@ -62,7 +65,7 @@ app.get('/versioned/v1', async (_: Request, res: Response) => { VERSIONS.forEach((version) => { app.get(`/versioned/${version}`, async (_: Request, res: Response) => { try { - const filePath = path.join(process.cwd(), "db", `${version}.json`); + const filePath = path.join(DB_FOLDER, `${version}.json`); const dataStr = await fs.readFile(filePath, "utf8"); const data = JSON.parse(dataStr); diff --git a/tests/test.ts b/tests/test.ts index 44c5100..833f835 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -16,6 +16,7 @@ import fs from "fs"; import parseAbsence from "../scrape/utils/parseAbsence.js"; const teachermap: Record = JSON.parse(fs.readFileSync("./tests/teachermap.json", "utf8")); +let passedAll = true; test("Me", [ { @@ -262,6 +263,7 @@ function test(input: string, expectedOutput: any[]) { const res = parseAbsence(input, teachermap); if (!deepEqual(res, expectedOutput)) { + passedAll = false; console.error("ERROR for input: " + input); console.log(JSON.stringify(res, null, 2)); console.log(JSON.stringify(expectedOutput, null, 2)); @@ -308,3 +310,7 @@ function deepEqual(a: any, b: any): boolean { return keysA.every((key) => keysB.includes(key) && deepEqual(a[key], b[key])); } + +if (passedAll) { + console.log("All tests passed"); +}