jzitnik/jecnarozvrh
1
0
Fork 0
Code Actions Activity

refactor: Some path refactoring

Browse Source
This commit is contained in:
Jakub Žitník
2026-02-11 10:51:02 +01:00
parent 9117044f88
commit faeb0323ba
8 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -1,8 +1,8 @@
node_modules
volume/browser
db
downloads
errors
volume/db
volume/downloads
volume/errors
dist
# Web

12
Dockerfile
View File

@@ -1,23 +1,15 @@
# Use official Node.js image as base
FROM node:18
# Create app directory
WORKDIR /usr/src/app
# Copy package.json and package-lock.json (if available)
COPY package*.json ./
# Install dependencies
RUN npm install
RUN npm ci
# Build
RUN npm run build
# Copy app source code
COPY . .
# Expose the port your app runs on (optional, depends on your app)
EXPOSE 3000
# Start the app
CMD ["npm", "start"]
CMD ["npm", "run", "serve"]

1
scrape/parse.ts
View File

@@ -15,7 +15,6 @@
import parseV1V2 from "./parse/v1_v2.js";
import parseV3 from "./parse/v3.js";
export default async function parseThisShit(downloadedFilePath: string) {
await parseV1V2(downloadedFilePath);
await parseV3(downloadedFilePath);

4
scrape/parse/v1_v2.ts
View File

@@ -253,7 +253,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
}
}
fs.writeFileSync("db/v2.json", JSON.stringify(data, null, 2));
fs.writeFileSync("volume/db/v2.json", JSON.stringify(data, null, 2));
// Modify the data for v1
const copy = JSON.parse(JSON.stringify(data));
@@ -275,7 +275,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
});
});
fs.writeFileSync("db/v1.json", JSON.stringify(copy, null, 2))
fs.writeFileSync("volume/db/v1.json", JSON.stringify(copy, null, 2))
}
//parseV1V2("db/current.xlsx")

4
scrape/parse/v3.ts
View File

@@ -147,7 +147,7 @@ export default async function parseV3(downloadedFilePath: string) {
schedule,
};
fs.writeFileSync("db/v3.json", JSON.stringify(data, null, 2));
fs.writeFileSync("volume/db/v3.json", JSON.stringify(data, null, 2));
}
//
@@ -424,4 +424,4 @@ function formatNowTime() {
);
}
parseV3("db/current.xlsx")
//parseV3("db/current.xlsx")

32
scrape/scraper.ts
View File

@@ -21,12 +21,16 @@ import 'dotenv/config';
const EMAIL = process.env.EMAIL;
const PASSWORD = process.env.PASSWORD;
const SHAREPOINT_URL = process.env.SHAREPOINT_URL || 'https://spsejecnacz.sharepoint.com/:x:/s/nastenka/ESy19K245Y9BouR5ksciMvgBu3Pn_9EaT0fpP8R6MrkEmg';
const VOLUME_PATH = path.resolve('./volume/browser');
const VOLUME_PATH = path.resolve("./volume/browser");
const DOWNLOAD_FOLDER = path.resolve("./volume/downloads");
const ERROR_FOLDER = path.resolve("./volume/errors")
const DB_FOLDER = path.resolve("./volume/db");
async function clearDownloadsFolder() {
try {
await fs.promises.rm('./downloads', { recursive: true, force: true });
await fs.promises.mkdir('./downloads');
await fs.promises.rm(DOWNLOAD_FOLDER, { recursive: true, force: true });
await fs.promises.mkdir(DOWNLOAD_FOLDER);
} catch (err) {
console.error('Error:', err);
}
@@ -34,28 +38,27 @@ async function clearDownloadsFolder() {
async function handleError(page: Page, err: any) {
try {
const errorsDir = path.resolve('./errors');
if (!fs.existsSync(errorsDir)) fs.mkdirSync(errorsDir);
if (!fs.existsSync(ERROR_FOLDER)) fs.mkdirSync(ERROR_FOLDER);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = path.join(errorsDir, `error-${timestamp}.png`);
const filePath = path.join(ERROR_FOLDER, `error-${timestamp}.png`);
// @ts-ignore
await page.screenshot({ path: filePath, fullPage: true });
console.error(`❌ Error occurred. Screenshot saved: ${filePath}`);
// Keep only last 10 screenshots
const files = fs.readdirSync(errorsDir)
const files = fs.readdirSync(ERROR_FOLDER)
.map(f => ({
name: f,
time: fs.statSync(path.join(errorsDir, f)).mtime.getTime()
time: fs.statSync(path.join(ERROR_FOLDER, f)).mtime.getTime()
}))
.sort((a, b) => b.time - a.time);
if (files.length > 10) {
const oldFiles = files.slice(10);
for (const f of oldFiles) {
fs.unlinkSync(path.join(errorsDir, f.name));
fs.unlinkSync(path.join(ERROR_FOLDER, f.name));
}
}
} catch (screenshotErr) {
@@ -75,13 +78,12 @@ async function handleError(page: Page, err: any) {
const pages = await browser.pages();
page = pages[0];
const downloadPath = path.resolve('./downloads');
if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
if (!fs.existsSync(DOWNLOAD_FOLDER)) fs.mkdirSync(DOWNLOAD_FOLDER);
const client = await page.createCDPSession();
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
downloadPath: downloadPath,
downloadPath: DOWNLOAD_FOLDER,
});
await page.goto(SHAREPOINT_URL, { waitUntil: 'networkidle2' });
@@ -182,14 +184,16 @@ async function handleError(page: Page, err: any) {
return files.length ? path.join(dir, files[0].name) : null;
}
const downloadedFilePath = getNewestFile(downloadPath);
const downloadedFilePath = getNewestFile(DOWNLOAD_FOLDER);
if (!downloadedFilePath) {
throw new Error('No XLSX file found in download folder');
}
console.log('Waiting for file:', downloadedFilePath);
await waitForFile(downloadedFilePath);
await fs.promises.cp(downloadedFilePath, "db/current.xlsx");
if (!fs.existsSync(DB_FOLDER)) fs.mkdirSync(DB_FOLDER);
await fs.promises.cp(downloadedFilePath, path.join(DB_FOLDER, "current.xlsx"));
await parseThisShit(downloadedFilePath);
await clearDownloadsFolder();

11
server.ts
View File

@@ -20,6 +20,9 @@ import { getCurrentInterval } from "./scheduleRules.js";
import bodyParser from "body-parser";
import cors from "cors";
const DB_FOLDER = path.join(process.cwd(), "volume", "db");
const WEB_FOLDER = path.join(process.cwd(), "web", "public");
const VERSIONS = ["v1", "v2", "v3"];
const PORT = process.env.PORT || 3000;
@@ -39,9 +42,9 @@ app.get('/', async (req: Request, res: Response) => {
const isBrowser = /Mozilla|Chrome|Firefox|Safari|Edg/.test(userAgent);
if (isBrowser) {
res.sendFile(path.join(process.cwd(), "web", "public", "index.html"));
res.sendFile(path.join(WEB_FOLDER, "index.html"));
} else {
const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8");
const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8");
const data = JSON.parse(dataStr);
data["status"]["currentUpdateSchedule"] = getCurrentInterval();
@@ -51,7 +54,7 @@ app.get('/', async (req: Request, res: Response) => {
});
app.get('/versioned/v1', async (_: Request, res: Response) => {
const dataStr = await fs.readFile(path.join(process.cwd(), "db", "v1.json"), "utf8");
const dataStr = await fs.readFile(path.join(DB_FOLDER, "v1.json"), "utf8");
const data = JSON.parse(dataStr);
data["status"]["currentUpdateSchedule"] = getCurrentInterval();
@@ -62,7 +65,7 @@ app.get('/versioned/v1', async (_: Request, res: Response) => {
VERSIONS.forEach((version) => {
app.get(`/versioned/${version}`, async (_: Request, res: Response) => {
try {
const filePath = path.join(process.cwd(), "db", `${version}.json`);
const filePath = path.join(DB_FOLDER, `${version}.json`);
const dataStr = await fs.readFile(filePath, "utf8");
const data = JSON.parse(dataStr);

6
tests/test.ts
View File

@@ -16,6 +16,7 @@ import fs from "fs";
import parseAbsence from "../scrape/utils/parseAbsence.js";
const teachermap: Record<string, string> = JSON.parse(fs.readFileSync("./tests/teachermap.json", "utf8"));
let passedAll = true;
test("Me", [
{
@@ -262,6 +263,7 @@ function test(input: string, expectedOutput: any[]) {
const res = parseAbsence(input, teachermap);
if (!deepEqual(res, expectedOutput)) {
passedAll = false;
console.error("ERROR for input: " + input);
console.log(JSON.stringify(res, null, 2));
console.log(JSON.stringify(expectedOutput, null, 2));
@@ -308,3 +310,7 @@ function deepEqual(a: any, b: any): boolean {
return keysA.every((key) => keysB.includes(key) && deepEqual(a[key], b[key]));
}
if (passedAll) {
console.log("All tests passed");
}