vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: lint

Browse Source
This commit is contained in:
karishmas6
2024-08-16 23:43:51 +05:30
parent 87476c3e35
commit b2678759db
1 changed files with 84 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
maxun-core/src/browserSide/scraper.js
View File

@@ -200,7 +200,7 @@ async function clickNextPagination(selector, scrapedData, limit) {
throw new Error("No more items to load or pagination has ended."); throw new Error("No more items to load or pagination has ended.");
} }
//return clicked; // Indicate whether pagination occurred return clicked; // Indicate whether pagination occurred
} }
} }
@@ -339,7 +339,7 @@ async function clickNextPagination(selector, scrapedData, limit) {
* @param {boolean} [config.flexible=false] - Whether to use flexible matching for field selectors * @param {boolean} [config.flexible=false] - Whether to use flexible matching for field selectors
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list * @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
*/ */
window.scrapeList = async function({ listSelector, fields, limit = 10, pagination = null }) { window.scrapeList = async function ({ listSelector, fields, limit = 10, pagination = null }) {
const scrapedData = []; const scrapedData = [];
while (scrapedData.length < limit) { while (scrapedData.length < limit) {
@@ -355,7 +355,6 @@ async function clickNextPagination(selector, scrapedData, limit) {
for (const [label, { selector, attribute }] of Object.entries(fields)) { for (const [label, { selector, attribute }] of Object.entries(fields)) {
const fieldElement = parent.querySelector(selector); const fieldElement = parent.querySelector(selector);
// Depending on the attribute specified, extract the data
if (fieldElement) { if (fieldElement) {
if (attribute === 'innerText') { if (attribute === 'innerText') {
record[label] = fieldElement.innerText.trim(); record[label] = fieldElement.innerText.trim();
@@ -366,7 +365,6 @@ async function clickNextPagination(selector, scrapedData, limit) {
} else if (attribute === 'href') { } else if (attribute === 'href') {
record[label] = fieldElement.href; record[label] = fieldElement.href;
} else { } else {
// Default to attribute retrieval
record[label] = fieldElement.getAttribute(attribute); record[label] = fieldElement.getAttribute(attribute);
} }
} }
@@ -376,19 +374,25 @@ async function clickNextPagination(selector, scrapedData, limit) {
scrapedData.push(record); scrapedData.push(record);
} }
// Check if we need to paginate
if (pagination && scrapedData.length < limit) { if (pagination && scrapedData.length < limit) {
let paginated = false;
switch (pagination.type) { switch (pagination.type) {
case 'scrollDown': case 'scrollDown':
await scrollDownToLoadMore(listSelector, limit); await scrollDownToLoadMore(listSelector, limit);
paginated = true;
break; break;
case 'scrollUp': case 'scrollUp':
await scrollUpToLoadMore(listSelector, limit); await scrollUpToLoadMore(listSelector, limit);
paginated = true;
break; break;
case 'clickNext': case 'clickNext':
await clickNextPagination(pagination.selector, scrapedData, limit); paginated = await clickNextPagination(pagination.selector, scrapedData, limit);
break; break;
case 'clickLoadMore': case 'clickLoadMore':
//await clickLoadMorePagination(pagination.selector); //await clickLoadMorePagination(pagination.selector);
//paginated = true;
break; break;
case 'none': case 'none':
// No more items to load // No more items to load
@@ -397,14 +401,19 @@ async function clickNextPagination(selector, scrapedData, limit) {
console.warn("Unknown pagination type"); console.warn("Unknown pagination type");
break; break;
} }
if (paginated) {
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for content to load await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for content to load
} else {
break; // No further pagination needed
}
} else { } else {
break; // No more items to load or no pagination break; // No more items to load or no pagination
} }
} }
return scrapedData.slice(0, limit); // Return only the limited number of records return scrapedData.slice(0, limit); // Return only the limited number of records
}; };
/** /**