jzitnik/jecnarozvrh
1
0
Fork 0
Code Actions Activity

refactor: Some path refactoring

Browse Source
This commit is contained in:
Jakub Žitník
2026-02-11 10:51:02 +01:00
parent 9117044f88
commit faeb0323ba
8 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -1,8 +1,8 @@
node_modules node_modules
volume/browser volume/browser
db volume/db
downloads volume/downloads
errors volume/errors
dist dist
# Web # Web

12
Dockerfile
View File

@@ -1,23 +1,15 @@
# Use official Node.js image as base
FROM node:18 FROM node:18
# Create app directory
WORKDIR /usr/src/app WORKDIR /usr/src/app
# Copy package.json and package-lock.json (if available)
COPY package*.json ./ COPY package*.json ./
# Install dependencies RUN npm ci
RUN npm install
# Build
RUN npm run build RUN npm run build
# Copy app source code
COPY . . COPY . .
# Expose the port your app runs on (optional, depends on your app)
EXPOSE 3000 EXPOSE 3000
# Start the app CMD ["npm", "run", "serve"]
CMD ["npm", "start"]

1
scrape/parse.ts
View File

@@ -15,7 +15,6 @@
import parseV1V2 from "./parse/v1_v2.js"; import parseV1V2 from "./parse/v1_v2.js";
import parseV3 from "./parse/v3.js"; import parseV3 from "./parse/v3.js";
export default async function parseThisShit(downloadedFilePath: string) { export default async function parseThisShit(downloadedFilePath: string) {
await parseV1V2(downloadedFilePath); await parseV1V2(downloadedFilePath);
await parseV3(downloadedFilePath); await parseV3(downloadedFilePath);

4
scrape/parse/v1_v2.ts
View File

@@ -253,7 +253,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
} }
} }
fs.writeFileSync("db/v2.json", JSON.stringify(data, null, 2)); fs.writeFileSync("volume/db/v2.json", JSON.stringify(data, null, 2));
// Modify the data for v1 // Modify the data for v1
const copy = JSON.parse(JSON.stringify(data)); const copy = JSON.parse(JSON.stringify(data));
@@ -275,7 +275,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
}); });
}); });
fs.writeFileSync("db/v1.json", JSON.stringify(copy, null, 2)) fs.writeFileSync("volume/db/v1.json", JSON.stringify(copy, null, 2))
} }
//parseV1V2("db/current.xlsx") //parseV1V2("db/current.xlsx")

4
scrape/parse/v3.ts
View File

@@ -147,7 +147,7 @@ export default async function parseV3(downloadedFilePath: string) {
schedule, schedule,
}; };
fs.writeFileSync("db/v3.json", JSON.stringify(data, null, 2)); fs.writeFileSync("volume/db/v3.json", JSON.stringify(data, null, 2));
} }
// //
@@ -424,4 +424,4 @@ function formatNowTime() {
); );
} }
parseV3("db/current.xlsx") //parseV3("db/current.xlsx")

32
scrape/scraper.ts
View File

@@ -21,12 +21,16 @@ import 'dotenv/config';
const EMAIL = process.env.EMAIL; const EMAIL = process.env.EMAIL;
const PASSWORD = process.env.PASSWORD; const PASSWORD = process.env.PASSWORD;
const SHAREPOINT_URL = process.env.SHAREPOINT_URL || 'https://spsejecnacz.sharepoint.com/:x:/s/nastenka/ESy19K245Y9BouR5ksciMvgBu3Pn_9EaT0fpP8R6MrkEmg'; const SHAREPOINT_URL = process.env.SHAREPOINT_URL || 'https://spsejecnacz.sharepoint.com/:x:/s/nastenka/ESy19K245Y9BouR5ksciMvgBu3Pn_9EaT0fpP8R6MrkEmg';
const VOLUME_PATH = path.resolve('./volume/browser');
const VOLUME_PATH = path.resolve("./volume/browser");
const DOWNLOAD_FOLDER = path.resolve("./volume/downloads");
const ERROR_FOLDER = path.resolve("./volume/errors")
const DB_FOLDER = path.resolve("./volume/db");
async function clearDownloadsFolder() { async function clearDownloadsFolder() {
try { try {
await fs.promises.rm('./downloads', { recursive: true, force: true }); await fs.promises.rm(DOWNLOAD_FOLDER, { recursive: true, force: true });
await fs.promises.mkdir('./downloads'); await fs.promises.mkdir(DOWNLOAD_FOLDER);
} catch (err) { } catch (err) {
console.error('Error:', err); console.error('Error:', err);
} }
@@ -34,28 +38,27 @@ async function clearDownloadsFolder() {
async function handleError(page: Page, err: any) { async function handleError(page: Page, err: any) {
try { try {
const errorsDir = path.resolve('./errors'); if (!fs.existsSync(ERROR_FOLDER)) fs.mkdirSync(ERROR_FOLDER);
if (!fs.existsSync(errorsDir)) fs.mkdirSync(errorsDir);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = path.join(errorsDir, `error-${timestamp}.png`); const filePath = path.join(ERROR_FOLDER, `error-${timestamp}.png`);
// @ts-ignore // @ts-ignore
await page.screenshot({ path: filePath, fullPage: true }); await page.screenshot({ path: filePath, fullPage: true });
console.error(`❌ Error occurred. Screenshot saved: ${filePath}`); console.error(`❌ Error occurred. Screenshot saved: ${filePath}`);
// Keep only last 10 screenshots // Keep only last 10 screenshots
const files = fs.readdirSync(errorsDir) const files = fs.readdirSync(ERROR_FOLDER)
.map(f => ({ .map(f => ({
name: f, name: f,
time: fs.statSync(path.join(errorsDir, f)).mtime.getTime() time: fs.statSync(path.join(ERROR_FOLDER, f)).mtime.getTime()
})) }))
.sort((a, b) => b.time - a.time); .sort((a, b) => b.time - a.time);
if (files.length > 10) { if (files.length > 10) {
const oldFiles = files.slice(10); const oldFiles = files.slice(10);
for (const f of oldFiles) { for (const f of oldFiles) {
fs.unlinkSync(path.join(errorsDir, f.name)); fs.unlinkSync(path.join(ERROR_FOLDER, f.name));
} }
} }
} catch (screenshotErr) { } catch (screenshotErr) {
@@ -75,13 +78,12 @@ async function handleError(page: Page, err: any) {
const pages = await browser.pages(); const pages = await browser.pages();
page = pages[0]; page = pages[0];
const downloadPath = path.resolve('./downloads'); if (!fs.existsSync(DOWNLOAD_FOLDER)) fs.mkdirSync(DOWNLOAD_FOLDER);
if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
const client = await page.createCDPSession(); const client = await page.createCDPSession();
await client.send('Page.setDownloadBehavior', { await client.send('Page.setDownloadBehavior', {
behavior: 'allow', behavior: 'allow',
downloadPath: downloadPath, downloadPath: DOWNLOAD_FOLDER,
}); });
await page.goto(SHAREPOINT_URL, { waitUntil: 'networkidle2' }); await page.goto(SHAREPOINT_URL, { waitUntil: 'networkidle2' });
@@ -182,14 +184,16 @@ async function handleError(page: Page, err: any) {
return files.length ? path.join(dir, files[0].name) : null; return files.length ? path.join(dir, files[0].name) : null;
} }
const downloadedFilePath = getNewestFile(downloadPath); const downloadedFilePath = getNewestFile(DOWNLOAD_FOLDER);
if (!downloadedFilePath) { if (!downloadedFilePath) {
throw new Error('No XLSX file found in download folder'); throw new Error('No XLSX file found in download folder');
} }
console.log('Waiting for file:', downloadedFilePath); console.log('Waiting for file:', downloadedFilePath);
await waitForFile(downloadedFilePath); await waitForFile(downloadedFilePath);
await fs.promises.cp(downloadedFilePath, "db/current.xlsx"); if (!fs.existsSync(DB_FOLDER)) fs.mkdirSync(DB_FOLDER);
await fs.promises.cp(downloadedFilePath, path.join(DB_FOLDER, "current.xlsx"));
await parseThisShit(downloadedFilePath); await parseThisShit(downloadedFilePath);
await clearDownloadsFolder(); await clearDownloadsFolder();

11
server.ts
View File

@@ -20,6 +20,9 @@ import { getCurrentInterval } from "./scheduleRules.js";
import bodyParser from "body-parser"; import bodyParser from "body-parser";
import cors from "cors"; import cors from "cors";
const DB_FOLDER = path.join(process.cwd(), "volume", "db");
const WEB_FOLDER = path.join(process.cwd(), "web", "public");
const VERSIONS = ["v1", "v2", "v3"]; const VERSIONS = ["v1", "v2", "v3"];
const PORT = process.env.PORT || 3000; const PORT = process.env.PORT || 3000;
@@ -39,9 +42,9 @@ app.get('/', async (req: Request, res: Response) => {
const isBrowser = /Mozilla|Chrome|Firefox|Safari|Edg/.test(userAgent); const isBrowser = /Mozilla|Chrome|Firefox|Safari|Edg/.test(userAgent);
if (isBrowser) { if (isBrowser) {
res.sendFile(path.join(process.cwd(), "web", "public", "index.html")); res.sendFile(path.join(WEB_FOLDER, "index.html"));
} else { } else {
const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8"); const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8");
const data = JSON.parse(dataStr); const data = JSON.parse(dataStr);
data["status"]["currentUpdateSchedule"] = getCurrentInterval(); data["status"]["currentUpdateSchedule"] = getCurrentInterval();
@@ -51,7 +54,7 @@ app.get('/', async (req: Request, res: Response) => {
}); });
app.get('/versioned/v1', async (_: Request, res: Response) => { app.get('/versioned/v1', async (_: Request, res: Response) => {
const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8"); const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8");
const data = JSON.parse(dataStr); const data = JSON.parse(dataStr);
data["status"]["currentUpdateSchedule"] = getCurrentInterval(); data["status"]["currentUpdateSchedule"] = getCurrentInterval();
@@ -62,7 +65,7 @@ app.get('/versioned/v1', async (_: Request, res: Response) => {
VERSIONS.forEach((version) => { VERSIONS.forEach((version) => {
app.get(`/versioned/${version}`, async (_: Request, res: Response) => { app.get(`/versioned/${version}`, async (_: Request, res: Response) => {
try { try {
const filePath = path.join(process.cwd(), "db", `${version}.json`); const filePath = path.join(DB_FOLDER, `${version}.json`);
const dataStr = await fs.readFile(filePath, "utf8"); const dataStr = await fs.readFile(filePath, "utf8");
const data = JSON.parse(dataStr); const data = JSON.parse(dataStr);

6
tests/test.ts
View File

@@ -16,6 +16,7 @@ import fs from "fs";
import parseAbsence from "../scrape/utils/parseAbsence.js"; import parseAbsence from "../scrape/utils/parseAbsence.js";
const teachermap: Record<string, string> = JSON.parse(fs.readFileSync("./tests/teachermap.json", "utf8")); const teachermap: Record<string, string> = JSON.parse(fs.readFileSync("./tests/teachermap.json", "utf8"));
let passedAll = true;
test("Me", [ test("Me", [
{ {
@@ -262,6 +263,7 @@ function test(input: string, expectedOutput: any[]) {
const res = parseAbsence(input, teachermap); const res = parseAbsence(input, teachermap);
if (!deepEqual(res, expectedOutput)) { if (!deepEqual(res, expectedOutput)) {
passedAll = false;
console.error("ERROR for input: " + input); console.error("ERROR for input: " + input);
console.log(JSON.stringify(res, null, 2)); console.log(JSON.stringify(res, null, 2));
console.log(JSON.stringify(expectedOutput, null, 2)); console.log(JSON.stringify(expectedOutput, null, 2));
@@ -308,3 +310,7 @@ function deepEqual(a: any, b: any): boolean {
return keysA.every((key) => keysB.includes(key) && deepEqual(a[key], b[key])); return keysA.every((key) => keysB.includes(key) && deepEqual(a[key], b[key]));
} }
if (passedAll) {
console.log("All tests passed");
}