jzitnik/jecnarozvrh
1
0
Fork 0
Code Actions Activity

refactor: Some path refactoring

Browse Source
This commit is contained in:
Jakub Žitník
2026-02-11 10:51:02 +01:00
parent 9117044f88
commit faeb0323ba
8 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scrape/parse.ts
View File

@@ -15,7 +15,6 @@
import parseV1V2 from "./parse/v1_v2.js";
import parseV3 from "./parse/v3.js";
export default async function parseThisShit(downloadedFilePath: string) {
await parseV1V2(downloadedFilePath);
await parseV3(downloadedFilePath);

4
scrape/parse/v1_v2.ts
View File

@@ -253,7 +253,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
}
}
fs.writeFileSync("db/v2.json", JSON.stringify(data, null, 2));
fs.writeFileSync("volume/db/v2.json", JSON.stringify(data, null, 2));
// Modify the data for v1
const copy = JSON.parse(JSON.stringify(data));
@@ -275,7 +275,7 @@ export default async function parseV1V2(downloadedFilePath: string) {
});
});
fs.writeFileSync("db/v1.json", JSON.stringify(copy, null, 2))
fs.writeFileSync("volume/db/v1.json", JSON.stringify(copy, null, 2))
}
//parseV1V2("db/current.xlsx")

4
scrape/parse/v3.ts
View File

@@ -147,7 +147,7 @@ export default async function parseV3(downloadedFilePath: string) {
schedule,
};
fs.writeFileSync("db/v3.json", JSON.stringify(data, null, 2));
fs.writeFileSync("volume/db/v3.json", JSON.stringify(data, null, 2));
}
//
@@ -424,4 +424,4 @@ function formatNowTime() {
);
}
parseV3("db/current.xlsx")
//parseV3("db/current.xlsx")

32
scrape/scraper.ts
View File

@@ -21,12 +21,16 @@ import 'dotenv/config';
const EMAIL = process.env.EMAIL;
const PASSWORD = process.env.PASSWORD;
const SHAREPOINT_URL = process.env.SHAREPOINT_URL || 'https://spsejecnacz.sharepoint.com/:x:/s/nastenka/ESy19K245Y9BouR5ksciMvgBu3Pn_9EaT0fpP8R6MrkEmg';
const VOLUME_PATH = path.resolve('./volume/browser');
const VOLUME_PATH = path.resolve("./volume/browser");
const DOWNLOAD_FOLDER = path.resolve("./volume/downloads");
const ERROR_FOLDER = path.resolve("./volume/errors")
const DB_FOLDER = path.resolve("./volume/db");
async function clearDownloadsFolder() {
try {
await fs.promises.rm('./downloads', { recursive: true, force: true });
await fs.promises.mkdir('./downloads');
await fs.promises.rm(DOWNLOAD_FOLDER, { recursive: true, force: true });
await fs.promises.mkdir(DOWNLOAD_FOLDER);
} catch (err) {
console.error('Error:', err);
}
@@ -34,28 +38,27 @@ async function clearDownloadsFolder() {
async function handleError(page: Page, err: any) {
try {
const errorsDir = path.resolve('./errors');
if (!fs.existsSync(errorsDir)) fs.mkdirSync(errorsDir);
if (!fs.existsSync(ERROR_FOLDER)) fs.mkdirSync(ERROR_FOLDER);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = path.join(errorsDir, `error-${timestamp}.png`);
const filePath = path.join(ERROR_FOLDER, `error-${timestamp}.png`);
// @ts-ignore
await page.screenshot({ path: filePath, fullPage: true });
console.error(`❌ Error occurred. Screenshot saved: ${filePath}`);
// Keep only last 10 screenshots
const files = fs.readdirSync(errorsDir)
const files = fs.readdirSync(ERROR_FOLDER)
.map(f => ({
name: f,
time: fs.statSync(path.join(errorsDir, f)).mtime.getTime()
time: fs.statSync(path.join(ERROR_FOLDER, f)).mtime.getTime()
}))
.sort((a, b) => b.time - a.time);
if (files.length > 10) {
const oldFiles = files.slice(10);
for (const f of oldFiles) {
fs.unlinkSync(path.join(errorsDir, f.name));
fs.unlinkSync(path.join(ERROR_FOLDER, f.name));
}
}
} catch (screenshotErr) {
@@ -75,13 +78,12 @@ async function handleError(page: Page, err: any) {
const pages = await browser.pages();
page = pages[0];
const downloadPath = path.resolve('./downloads');
if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
if (!fs.existsSync(DOWNLOAD_FOLDER)) fs.mkdirSync(DOWNLOAD_FOLDER);
const client = await page.createCDPSession();
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
downloadPath: downloadPath,
downloadPath: DOWNLOAD_FOLDER,
});
await page.goto(SHAREPOINT_URL, { waitUntil: 'networkidle2' });
@@ -182,14 +184,16 @@ async function handleError(page: Page, err: any) {
return files.length ? path.join(dir, files[0].name) : null;
}
const downloadedFilePath = getNewestFile(downloadPath);
const downloadedFilePath = getNewestFile(DOWNLOAD_FOLDER);
if (!downloadedFilePath) {
throw new Error('No XLSX file found in download folder');
}
console.log('Waiting for file:', downloadedFilePath);
await waitForFile(downloadedFilePath);
await fs.promises.cp(downloadedFilePath, "db/current.xlsx");
if (!fs.existsSync(DB_FOLDER)) fs.mkdirSync(DB_FOLDER);
await fs.promises.cp(downloadedFilePath, path.join(DB_FOLDER, "current.xlsx"));
await parseThisShit(downloadedFilePath);
await clearDownloadsFolder();