Merge pull request #834 from getmaxun/capflow-revamp

feat(maxun-core): capture flow revamp
This commit is contained in:
Karishma
2025-10-23 23:34:20 +05:30
committed by GitHub
21 changed files with 3194 additions and 1585 deletions

View File

@@ -73,6 +73,15 @@ export default class Interpreter extends EventEmitter {
private cumulativeResults: Record<string, any>[] = [];
private namedResults: Record<string, Record<string, any>> = {};
private screenshotCounter: number = 0;
private serializableDataByType: Record<string, Record<string, any>> = {
scrapeList: {},
scrapeSchema: {}
};
constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>) {
super();
this.workflow = workflow.workflow;
@@ -402,15 +411,37 @@ export default class Interpreter extends EventEmitter {
* Beware of false linter errors - here, we know better!
*/
const wawActions: Record<CustomFunctions, (...args: any[]) => void> = {
screenshot: async (params: PageScreenshotOptions) => {
screenshot: async (
params: PageScreenshotOptions,
nameOverride?: string
) => {
if (this.options.debugChannel?.setActionType) {
this.options.debugChannel.setActionType('screenshot');
this.options.debugChannel.setActionType("screenshot");
}
const screenshotBuffer = await page.screenshot({
...params, path: undefined,
...params,
path: undefined,
});
await this.options.binaryCallback(screenshotBuffer, 'image/png');
const explicitName = (typeof nameOverride === 'string' && nameOverride.trim().length > 0) ? nameOverride.trim() : null;
let screenshotName: string;
if (explicitName) {
screenshotName = explicitName;
} else {
this.screenshotCounter += 1;
screenshotName = `Screenshot ${this.screenshotCounter}`;
}
await this.options.binaryCallback(
{
name: screenshotName,
data: screenshotBuffer,
mimeType: "image/png",
},
"image/png"
);
},
enqueueLinks: async (selector: string) => {
if (this.options.debugChannel?.setActionType) {
@@ -476,21 +507,55 @@ export default class Interpreter extends EventEmitter {
this.cumulativeResults = [];
}
if (this.cumulativeResults.length === 0) {
this.cumulativeResults.push({});
}
const mergedResult = this.cumulativeResults[0];
const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
Object.entries(resultToProcess).forEach(([key, value]) => {
if (value !== undefined) {
mergedResult[key] = value;
if (this.cumulativeResults.length === 0) {
const newRow = {};
Object.entries(resultToProcess).forEach(([key, value]) => {
if (value !== undefined) {
newRow[key] = value;
}
});
this.cumulativeResults.push(newRow);
} else {
const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
if (hasRepeatedKeys) {
const newRow = {};
Object.entries(resultToProcess).forEach(([key, value]) => {
if (value !== undefined) {
newRow[key] = value;
}
});
this.cumulativeResults.push(newRow);
} else {
Object.entries(resultToProcess).forEach(([key, value]) => {
if (value !== undefined) {
lastRow[key] = value;
}
});
}
});
}
console.log("Updated merged result:", mergedResult);
await this.options.serializableCallback([mergedResult]);
const actionType = "scrapeSchema";
const actionName = (schema as any).__name || "Texts";
if (!this.namedResults[actionType]) this.namedResults[actionType] = {};
this.namedResults[actionType][actionName] = this.cumulativeResults;
if (!this.serializableDataByType[actionType]) this.serializableDataByType[actionType] = {};
if (!this.serializableDataByType[actionType][actionName]) {
this.serializableDataByType[actionType][actionName] = [];
}
this.serializableDataByType[actionType][actionName] = [...this.cumulativeResults];
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
},
scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => {
@@ -508,18 +573,62 @@ export default class Interpreter extends EventEmitter {
return;
}
await this.ensureScriptsLoaded(page);
try {
await this.ensureScriptsLoaded(page);
if (this.options.debugChannel?.incrementScrapeListIndex) {
this.options.debugChannel.incrementScrapeListIndex();
}
if (this.options.debugChannel?.incrementScrapeListIndex) {
this.options.debugChannel.incrementScrapeListIndex();
}
if (!config.pagination) {
const scrapeResults: Record<string, any>[] = await page.evaluate((cfg) => window.scrapeList(cfg), config);
await this.options.serializableCallback(scrapeResults);
} else {
const scrapeResults: Record<string, any>[] = await this.handlePagination(page, config);
await this.options.serializableCallback(scrapeResults);
let scrapeResults = [];
if (!config.pagination) {
scrapeResults = await page.evaluate((cfg) => {
try {
return window.scrapeList(cfg);
} catch (error) {
console.warn('ScrapeList evaluation failed:', error.message);
return [];
}
}, config);
} else {
scrapeResults = await this.handlePagination(page, config);
}
if (!Array.isArray(scrapeResults)) {
scrapeResults = [];
}
const actionType = "scrapeList";
const actionName = (config as any).__name || "List";
if (!this.serializableDataByType[actionType]) this.serializableDataByType[actionType] = {};
if (!this.serializableDataByType[actionType][actionName]) {
this.serializableDataByType[actionType][actionName] = [];
}
this.serializableDataByType[actionType][actionName].push(...scrapeResults);
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
} catch (error) {
console.error('ScrapeList action failed completely:', error.message);
const actionType = "scrapeList";
const actionName = (config as any).__name || "List";
if (!this.namedResults[actionType]) this.namedResults[actionType] = {};
this.namedResults[actionType][actionName] = [];
if (!this.serializableDataByType[actionType]) this.serializableDataByType[actionType] = {};
this.serializableDataByType[actionType][actionName] = [];
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
}
},
@@ -595,12 +704,56 @@ export default class Interpreter extends EventEmitter {
for (const step of steps) {
if (this.isAborted) {
this.log('Workflow aborted during step execution', Level.WARN);
return;
}
this.log(`Launching ${String(step.action)}`, Level.LOG);
let stepName: string | null = null;
try {
const debug = this.options.debugChannel;
if (debug?.setActionType) {
debug.setActionType(String(step.action));
}
if ((step as any)?.name) {
stepName = (step as any).name;
} else if (
Array.isArray((step as any)?.args) &&
(step as any).args.length > 0 &&
typeof (step as any).args[0] === "object" &&
"__name" in (step as any).args[0]
) {
stepName = (step as any).args[0].__name;
} else if (
typeof (step as any)?.args === "object" &&
step?.args !== null &&
"__name" in (step as any).args
) {
stepName = (step as any).args.__name;
}
if (!stepName) {
stepName = String(step.action);
}
if (debug && typeof (debug as any).setActionName === "function") {
(debug as any).setActionName(stepName);
}
} catch (err) {
this.log(`Failed to set action name/type: ${(err as Error).message}`, Level.WARN);
}
if (step.action in wawActions) {
// "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
await wawActions[step.action as CustomFunctions](...(params ?? []));
if (step.action === 'screenshot') {
await (wawActions.screenshot as any)(...(params ?? []), stepName ?? undefined);
} else {
await wawActions[step.action as CustomFunctions](...(params ?? []));
}
} else {
if (this.options.debugChannel?.setActionType) {
this.options.debugChannel.setActionType(String(step.action));

View File

@@ -36,6 +36,8 @@ export default class Preprocessor {
what: Joi.array().items({
action: Joi.string().required(),
args: Joi.array().items(Joi.any()),
name: Joi.string(),
actionId: Joi.string()
}).required(),
}),
).required(),

View File

@@ -32,7 +32,9 @@ export type CustomFunctions = 'scrape' | 'scrapeSchema' | 'scroll' | 'screenshot
export type What = {
action: MethodNames<Page> | CustomFunctions,
args?: any[]
args?: any[],
name?: string,
actionId?: string
};
export type PageState = Partial<BaseConditions>;

View File

@@ -1,16 +1,14 @@
import { readFile, readFiles } from "../workflow-management/storage";
import { Router, Request, Response } from 'express';
import { chromium } from "playwright-extra";
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import { requireAPIKey } from "../middlewares/api";
import Robot from "../models/Robot";
import Run from "../models/Run";
const router = Router();
import { getDecryptedProxyConfig } from "../routes/proxy";
import { v4 as uuid } from "uuid";
import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller";
import logger from "../logger";
import { browserPool } from "../server";
import { browserPool, io as serverIo } from "../server";
import { io, Socket } from "socket.io-client";
import { BinaryOutputService } from "../storage/mino";
import { AuthenticatedRequest } from "../routes/record"
@@ -20,8 +18,11 @@ import { WorkflowFile } from "maxun-core";
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet";
import { airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable";
import { sendWebhook } from "../routes/webhook";
chromium.use(stealthPlugin());
const router = Router();
const formatRecording = (recordingData: any) => {
const recordingMeta = recordingData.recording_meta;
const workflow = recordingData.recording.workflow || [];
@@ -334,7 +335,7 @@ function formatRunResponse(run: any) {
id: run.id,
status: run.status,
name: run.name,
robotId: run.robotMetaId, // Renaming robotMetaId to robotId
robotId: run.robotMetaId,
startedAt: run.startedAt,
finishedAt: run.finishedAt,
runId: run.runId,
@@ -342,20 +343,20 @@ function formatRunResponse(run: any) {
runByScheduleId: run.runByScheduleId,
runByAPI: run.runByAPI,
data: {
textData: [],
listData: []
textData: {},
listData: {}
},
screenshots: [] as any[],
};
if (run.serializableOutput) {
if (run.serializableOutput.scrapeSchema && run.serializableOutput.scrapeSchema.length > 0) {
formattedRun.data.textData = run.serializableOutput.scrapeSchema;
}
const output = run.serializableOutput || {};
if (run.serializableOutput.scrapeList && run.serializableOutput.scrapeList.length > 0) {
formattedRun.data.listData = run.serializableOutput.scrapeList;
}
if (output.scrapeSchema && typeof output.scrapeSchema === 'object') {
formattedRun.data.textData = output.scrapeSchema;
}
if (output.scrapeList && typeof output.scrapeList === 'object') {
formattedRun.data.listData = output.scrapeList;
}
if (run.binaryOutput) {
@@ -505,10 +506,30 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
runByAPI: true,
serializableOutput: {},
binaryOutput: {},
retryCount: 0
});
const plainRun = run.toJSON();
try {
const runStartedData = {
runId: plainRun.runId,
robotMetaId: plainRun.robotMetaId,
robotName: plainRun.name,
status: 'running',
startedAt: plainRun.startedAt,
runByUserId: plainRun.runByUserId,
runByScheduleId: plainRun.runByScheduleId,
runByAPI: plainRun.runByAPI || false,
browserId: plainRun.browserId
};
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-started', runStartedData);
logger.log('info', `API run started notification sent for run: ${plainRun.runId} to user-${userId}`);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-started notification for API run ${plainRun.runId}: ${socketError.message}`);
}
return {
browserId,
runId: plainRun.runId,
@@ -525,6 +546,29 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
}
}
async function triggerIntegrationUpdates(runId: string, robotMetaId: string): Promise<void> {
try {
googleSheetUpdateTasks[runId] = {
robotId: robotMetaId,
runId: runId,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[runId] = {
robotId: robotMetaId,
runId: runId,
status: 'pending',
retries: 5,
};
processAirtableUpdates().catch(err => logger.log('error', `Airtable update error: ${err.message}`));
processGoogleSheetUpdates().catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
} catch (err: any) {
logger.log('error', `Failed to update integrations for run: ${runId}: ${err.message}`);
}
}
async function readyForRunHandler(browserId: string, id: string, userId: string){
try {
const result = await executeRun(id, userId);
@@ -565,6 +609,8 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
};
async function executeRun(id: string, userId: string) {
let browser: any = null;
try {
const run = await Run.findOne({ where: { runId: id } });
if (!run) {
@@ -576,6 +622,27 @@ async function executeRun(id: string, userId: string) {
const plainRun = run.toJSON();
if (run.status === 'aborted' || run.status === 'aborting') {
logger.log('info', `API Run ${id} has status ${run.status}, skipping execution`);
return { success: true };
}
if (run.status === 'queued') {
logger.log('info', `API Run ${id} has status 'queued', skipping stale execution - will be handled by recovery`);
return { success: true };
}
const retryCount = plainRun.retryCount || 0;
if (retryCount >= 3) {
logger.log('warn', `API Run ${id} has exceeded max retries (${retryCount}/3), marking as failed`);
await run.update({
status: 'failed',
finishedAt: new Date().toLocaleString(),
log: `Max retries exceeded (${retryCount}/3) - Run permanently failed`
});
return { success: false, error: 'Max retries exceeded' };
}
const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true });
if (!recording) {
return {
@@ -586,7 +653,7 @@ async function executeRun(id: string, userId: string) {
plainRun.status = 'running';
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
browser = browserPool.getRemoteBrowser(plainRun.browserId);
if (!browser) {
throw new Error('Could not access browser');
}
@@ -597,41 +664,33 @@ async function executeRun(id: string, userId: string) {
}
const workflow = AddGeneratedFlags(recording.recording);
browser.interpreter.setRunId(id);
browser.interpreter.setRunId(plainRun.runId);
const interpretationInfo = await browser.interpreter.InterpretRecording(
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings
);
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
await destroyRemoteBrowser(plainRun.browserId, userId);
const updatedRun = await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: interpretationInfo.log.join('\n'),
binaryOutput: uploadedBinaryOutput,
});
// Upload binary output to MinIO and update run with MinIO URLs
const finalRun = await Run.findOne({ where: { runId: id } });
if (finalRun && finalRun.binaryOutput && Object.keys(finalRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(finalRun, finalRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for API run ${id}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for API run ${id}: ${minioError.message}`);
}
}
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
if (finalRun) {
if (finalRun.serializableOutput) {
if (finalRun.serializableOutput.scrapeSchema) {
Object.values(finalRun.serializableOutput.scrapeSchema).forEach((schemaResult: any) => {
if (updatedRun) {
if (updatedRun.dataValues.serializableOutput) {
if (updatedRun.dataValues.serializableOutput.scrapeSchema) {
Object.values(updatedRun.dataValues.serializableOutput.scrapeSchema).forEach((schemaResult: any) => {
if (Array.isArray(schemaResult)) {
totalSchemaItemsExtracted += schemaResult.length;
} else if (schemaResult && typeof schemaResult === 'object') {
@@ -640,8 +699,8 @@ async function executeRun(id: string, userId: string) {
});
}
if (finalRun.serializableOutput.scrapeList) {
Object.values(finalRun.serializableOutput.scrapeList).forEach((listResult: any) => {
if (updatedRun.dataValues.serializableOutput.scrapeList) {
Object.values(updatedRun.dataValues.serializableOutput.scrapeList).forEach((listResult: any) => {
if (Array.isArray(listResult)) {
totalListItemsExtracted += listResult.length;
}
@@ -649,8 +708,8 @@ async function executeRun(id: string, userId: string) {
}
}
if (finalRun.binaryOutput) {
extractedScreenshotsCount = Object.keys(finalRun.binaryOutput).length;
if (updatedRun.dataValues.binaryOutput) {
extractedScreenshotsCount = Object.keys(updatedRun.dataValues.binaryOutput).length;
}
}
@@ -667,17 +726,31 @@ async function executeRun(id: string, userId: string) {
}
)
const parsedOutput =
typeof updatedRun.dataValues.serializableOutput === "string"
? JSON.parse(updatedRun.dataValues.serializableOutput)
: updatedRun.dataValues.serializableOutput || {};
const parsedList =
typeof parsedOutput.scrapeList === "string"
? JSON.parse(parsedOutput.scrapeList)
: parsedOutput.scrapeList || {};
const parsedSchema =
typeof parsedOutput.scrapeSchema === "string"
? JSON.parse(parsedOutput.scrapeSchema)
: parsedOutput.scrapeSchema || {};
const webhookPayload = {
robot_id: plainRun.robotMetaId,
run_id: plainRun.runId,
robot_name: recording.recording_meta.name,
status: 'success',
status: "success",
started_at: plainRun.startedAt,
finished_at: new Date().toLocaleString(),
extracted_data: {
captured_texts: finalRun?.serializableOutput?.scrapeSchema ? Object.values(finalRun.serializableOutput.scrapeSchema).flat() : [],
captured_lists: finalRun?.serializableOutput?.scrapeList || {},
total_rows: totalRowsExtracted,
captured_texts: parsedSchema || {},
captured_lists: parsedList || {},
captured_texts_count: totalSchemaItemsExtracted,
captured_lists_count: totalListItemsExtracted,
screenshots_count: extractedScreenshotsCount
@@ -685,7 +758,7 @@ async function executeRun(id: string, userId: string) {
metadata: {
browser_id: plainRun.browserId,
user_id: userId,
}
},
};
try {
@@ -695,26 +768,7 @@ async function executeRun(id: string, userId: string) {
logger.log('error', `Failed to send webhooks for run ${plainRun.runId}: ${webhookError.message}`);
}
try {
googleSheetUpdateTasks[id] = {
robotId: plainRun.robotMetaId,
runId: id,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[id] = {
robotId: plainRun.robotMetaId,
runId: id,
status: 'pending',
retries: 5,
};
processAirtableUpdates().catch(err => logger.log('error', `Airtable update error: ${err.message}`));
processGoogleSheetUpdates().catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
} catch (err: any) {
logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`);
}
await triggerIntegrationUpdates(plainRun.runId, plainRun.robotMetaId);
return {
success: true,
@@ -728,8 +782,29 @@ async function executeRun(id: string, userId: string) {
await run.update({
status: 'failed',
finishedAt: new Date().toLocaleString(),
log: (run.log ? run.log + '\n' : '') + `Error: ${error.message}\n` + (error.stack ? error.stack : ''),
});
try {
const recording = await Robot.findOne({ where: { 'recording_meta.id': run.robotMetaId }, raw: true });
const failureData = {
runId: run.runId,
robotMetaId: run.robotMetaId,
robotName: recording ? recording.recording_meta.name : 'Unknown Robot',
status: 'failed',
finishedAt: new Date().toLocaleString(),
runByUserId: run.runByUserId,
runByScheduleId: run.runByScheduleId,
runByAPI: run.runByAPI || false,
browserId: run.browserId
};
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-completed', failureData);
logger.log('info', `API run permanently failed notification sent for run: ${run.runId} to user-${userId}`);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-completed notification for permanently failed API run ${run.runId}: ${socketError.message}`);
}
const recording = await Robot.findOne({ where: { 'recording_meta.id': run.robotMetaId }, raw: true });
// Trigger webhooks for run failure
@@ -814,7 +889,7 @@ async function waitForRunCompletion(runId: string, interval: number = 2000) {
if (!run) throw new Error('Run not found');
if (run.status === 'success') {
return run;
return run.toJSON();
} else if (run.status === 'failed') {
throw new Error('Run failed');
}

View File

@@ -65,6 +65,7 @@ const handleWrapper = async (
interface CustomActionEventData {
action: CustomActions;
settings: any;
actionId?: string;
}
/**
@@ -84,23 +85,24 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData, us
* @param page The active page
* @param action The custom action
* @param settings The custom action settings
* @param actionId Optional action ID for tracking and updating specific actions
* @category BrowserManagement
*/
const handleGenerateAction =
async (activeBrowser: RemoteBrowser, page: Page, { action, settings }: CustomActionEventData) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring generate action event: page is closed`);
return;
}
async (activeBrowser: RemoteBrowser, page: Page, { action, settings, actionId }: CustomActionEventData) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring generate action event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.customAction(action, settings, page);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling generate action event: ${message}`);
}
const generator = activeBrowser.generator;
await generator.customAction(action, actionId || '', settings, page);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling generate action event: ${message}`);
}
}
/**
* A wrapper function for handling mousedown event.
@@ -819,6 +821,49 @@ const onDOMWorkflowPair = async (
await handleWrapper(handleWorkflowPair, userId, data);
};
/**
* Handles the remove action event.
* This is called when a user discards a capture action (list or text) that was already emitted to the backend.
* @param activeBrowser - the active remote browser instance
* @param page - the active page of the remote browser
* @param data - the data containing the actionId to remove
* @category BrowserManagement
*/
const handleRemoveAction = async (
activeBrowser: RemoteBrowser,
page: Page,
data: { actionId: string }
) => {
try {
const { actionId } = data;
const generator = activeBrowser.generator;
const removed = generator.removeAction(actionId);
if (removed) {
logger.log("info", `Action ${actionId} successfully removed from workflow`);
} else {
logger.log("debug", `Action ${actionId} not found in workflow`);
}
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling remove action event: ${message}`);
}
};
/**
* A wrapper function for handling the remove action event.
* @param data - the data containing the actionId to remove
* @param userId - the user ID
* @category HelperFunctions
*/
const onRemoveAction = async (
data: { actionId: string },
userId: string
) => {
logger.log("debug", "Handling remove action event emitted from client");
await handleWrapper(handleRemoveAction, userId, data);
};
/**
* Helper function for registering the handlers onto established websocket connection.
* Registers various input handlers.
@@ -831,7 +876,7 @@ const onDOMWorkflowPair = async (
* @returns void
* @category BrowserManagement
*/
const registerInputHandlers = (socket: Socket, userId: string) => {
const registerInputHandlers = (socket: Socket, userId: string) => {
// Register handlers with the socket
socket.on("input:mousedown", (data) => onMousedown(data, userId));
socket.on("input:wheel", (data) => onWheel(data, userId));
@@ -847,6 +892,7 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
socket.on("input:time", (data) => onTimeSelection(data, userId));
socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(data, userId));
socket.on("action", (data) => onGenerateAction(data, userId));
socket.on("removeAction", (data) => onRemoveAction(data, userId));
socket.on("dom:click", (data) => onDOMClickAction(data, userId));
socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId));

View File

@@ -149,14 +149,20 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
let browser = browserPool.getRemoteBrowser(browserId);
const browserWaitStart = Date.now();
let lastLogTime = 0;
let pollAttempts = 0;
const MAX_POLL_ATTEMPTS = 15;
while (!browser && (Date.now() - browserWaitStart) < BROWSER_INIT_TIMEOUT) {
while (!browser && (Date.now() - browserWaitStart) < BROWSER_INIT_TIMEOUT && pollAttempts < MAX_POLL_ATTEMPTS) {
const currentTime = Date.now();
pollAttempts++;
const browserStatus = browserPool.getBrowserStatus(browserId);
if (browserStatus === null) {
throw new Error(`Browser slot ${browserId} does not exist in pool`);
}
if (browserStatus === "failed") {
throw new Error(`Browser ${browserId} initialization failed`);
}
if (currentTime - lastLogTime > 10000) {
logger.log('info', `Browser ${browserId} not ready yet (status: ${browserStatus}), waiting... (${Math.round((currentTime - browserWaitStart) / 1000)}s elapsed)`);
@@ -183,17 +189,25 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
}
const isRunAborted = async (): Promise<boolean> => {
const currentRun = await Run.findOne({ where: { runId: data.runId } });
return currentRun ? (currentRun.status === 'aborted' || currentRun.status === 'aborting') : false;
try {
const currentRun = await Run.findOne({ where: { runId: data.runId } });
return currentRun ? (currentRun.status === 'aborted' || currentRun.status === 'aborting') : false;
} catch (error: any) {
logger.log('error', `Error checking if run ${data.runId} is aborted: ${error.message}`);
return false;
}
};
let currentPage = browser.getCurrentPage();
const pageWaitStart = Date.now();
let lastPageLogTime = 0;
let pageAttempts = 0;
const MAX_PAGE_ATTEMPTS = 15;
while (!currentPage && (Date.now() - pageWaitStart) < BROWSER_PAGE_TIMEOUT) {
while (!currentPage && (Date.now() - pageWaitStart) < BROWSER_PAGE_TIMEOUT && pageAttempts < MAX_PAGE_ATTEMPTS) {
const currentTime = Date.now();
pageAttempts++;
if (currentTime - lastPageLogTime > 5000) {
logger.log('info', `Page not ready for browser ${browserId}, waiting... (${Math.round((currentTime - pageWaitStart) / 1000)}s elapsed)`);
@@ -209,6 +223,26 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
}
logger.log('info', `Starting workflow execution for run ${data.runId}`);
await run.update({
status: 'running',
log: 'Workflow execution started'
});
try {
const startedData = {
runId: data.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording.recording_meta.name,
status: 'running',
startedAt: new Date().toLocaleString()
};
serverIo.of(browserId).emit('run-started', startedData);
serverIo.of('/queued-run').to(`user-${data.userId}`).emit('run-started', startedData);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-started notification for API run ${plainRun.runId}: ${socketError.message}`);
}
// Execute the workflow
const workflow = AddGeneratedFlags(recording.recording);
@@ -231,6 +265,19 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
}
logger.log('info', `Workflow execution completed for run ${data.runId}`);
const binaryOutputService = new BinaryOutputService('maxuncloud-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(
run,
interpretationInfo.binaryOutput
);
// Get the already persisted and credit-validated data from the run record
const finalRun = await Run.findByPk(run.id);
const categorizedOutput = {
scrapeSchema: finalRun?.serializableOutput?.scrapeSchema || {},
scrapeList: finalRun?.serializableOutput?.scrapeList || {}
};
if (await isRunAborted()) {
logger.log('info', `Run ${data.runId} was aborted while processing results, not updating status`);
@@ -240,48 +287,39 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: interpretationInfo.log.join('\n')
log: interpretationInfo.log.join('\n'),
serializableOutput: JSON.parse(JSON.stringify({
scrapeSchema: categorizedOutput.scrapeSchema || {},
scrapeList: categorizedOutput.scrapeList || {},
})),
binaryOutput: uploadedBinaryOutput,
});
// Upload binary output to MinIO and update run with MinIO URLs
const updatedRun = await Run.findOne({ where: { runId: data.runId } });
if (updatedRun && updatedRun.binaryOutput && Object.keys(updatedRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(updatedRun, updatedRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for run ${data.runId}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for run ${data.runId}: ${minioError.message}`);
}
}
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
if (updatedRun) {
if (updatedRun.serializableOutput) {
if (updatedRun.serializableOutput.scrapeSchema) {
Object.values(updatedRun.serializableOutput.scrapeSchema).forEach((schemaResult: any) => {
if (Array.isArray(schemaResult)) {
totalSchemaItemsExtracted += schemaResult.length;
} else if (schemaResult && typeof schemaResult === 'object') {
totalSchemaItemsExtracted += 1;
}
});
}
if (updatedRun.serializableOutput.scrapeList) {
Object.values(updatedRun.serializableOutput.scrapeList).forEach((listResult: any) => {
if (Array.isArray(listResult)) {
totalListItemsExtracted += listResult.length;
}
});
}
if (categorizedOutput) {
if (categorizedOutput.scrapeSchema) {
Object.values(categorizedOutput.scrapeSchema).forEach((schemaResult: any) => {
if (Array.isArray(schemaResult)) {
totalSchemaItemsExtracted += schemaResult.length;
} else if (schemaResult && typeof schemaResult === 'object') {
totalSchemaItemsExtracted += 1;
}
});
}
if (updatedRun.binaryOutput) {
extractedScreenshotsCount = Object.keys(updatedRun.binaryOutput).length;
if (categorizedOutput.scrapeList) {
Object.values(categorizedOutput.scrapeList).forEach((listResult: any) => {
if (Array.isArray(listResult)) {
totalListItemsExtracted += listResult.length;
}
});
}
if (run.binaryOutput) {
extractedScreenshotsCount = Object.keys(run.binaryOutput).length;
}
}
@@ -302,6 +340,21 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
}
);
try {
const completionData = {
runId: data.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording.recording_meta.name,
status: 'success',
finishedAt: new Date().toLocaleString()
};
serverIo.of(browserId).emit('run-completed', completionData);
serverIo.of('/queued-run').to(`user-${data.userId}`).emit('run-completed', completionData);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-completed notification for API run ${plainRun.runId}: ${socketError.message}`);
}
const webhookPayload = {
robot_id: plainRun.robotMetaId,
run_id: data.runId,
@@ -310,12 +363,16 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
started_at: plainRun.startedAt,
finished_at: new Date().toLocaleString(),
extracted_data: {
captured_texts: updatedRun?.serializableOutput?.scrapeSchema ? Object.values(updatedRun.serializableOutput.scrapeSchema).flat() : [],
captured_lists: updatedRun?.serializableOutput?.scrapeList || {},
total_rows: totalRowsExtracted,
captured_texts: Object.keys(categorizedOutput.scrapeSchema || {}).length > 0
? Object.entries(categorizedOutput.scrapeSchema).reduce((acc, [name, value]) => {
acc[name] = Array.isArray(value) ? value : [value];
return acc;
}, {} as Record<string, any[]>)
: {},
captured_lists: categorizedOutput.scrapeList,
captured_texts_count: totalSchemaItemsExtracted,
captured_lists_count: totalListItemsExtracted,
screenshots_count: extractedScreenshotsCount,
screenshots_count: extractedScreenshotsCount
},
metadata: {
browser_id: plainRun.browserId,
@@ -330,26 +387,8 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
logger.log('error', `Failed to send webhooks for run ${data.runId}: ${webhookError.message}`);
}
// Schedule updates for Google Sheets and Airtable
await triggerIntegrationUpdates(plainRun.runId, plainRun.robotMetaId);
// Flush any remaining persistence buffer before emitting socket event
if (browser && browser.interpreter) {
await browser.interpreter.flushPersistenceBuffer();
logger.log('debug', `Flushed persistence buffer before emitting run-completed for run ${data.runId}`);
}
const completionData = {
runId: data.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording.recording_meta.name,
status: 'success',
finishedAt: new Date().toLocaleString()
};
serverIo.of(browserId).emit('run-completed', completionData);
serverIo.of('/queued-run').to(`user-${data.userId}`).emit('run-completed', completionData);
await destroyRemoteBrowser(browserId, data.userId);
logger.log('info', `Browser ${browserId} destroyed after successful run ${data.runId}`);
@@ -416,9 +455,13 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
},
partial_data_extracted: partialDataExtracted,
extracted_data: partialDataExtracted ? {
captured_texts: Object.values(partialUpdateData.serializableOutput?.scrapeSchema || []).flat() || [],
captured_texts: Object.keys(partialUpdateData.serializableOutput?.scrapeSchema || {}).length > 0
? Object.entries(partialUpdateData.serializableOutput.scrapeSchema).reduce((acc, [name, value]) => {
acc[name] = Array.isArray(value) ? value : [value];
return acc;
}, {} as Record<string, any[]>)
: {},
captured_lists: partialUpdateData.serializableOutput?.scrapeList || {},
total_data_points_extracted: partialData?.totalDataPointsExtracted || 0,
captured_texts_count: partialData?.totalSchemaItemsExtracted || 0,
captured_lists_count: partialData?.totalListItemsExtracted || 0,
screenshots_count: partialData?.extractedScreenshotsCount || 0

View File

@@ -9,6 +9,35 @@ const minioClient = new Client({
secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key',
});
async function fixMinioBucketConfiguration(bucketName: string) {
try {
const exists = await minioClient.bucketExists(bucketName);
if (!exists) {
await minioClient.makeBucket(bucketName);
console.log(`Bucket ${bucketName} created.`);
} else {
console.log(`Bucket ${bucketName} already exists.`);
}
const policyJSON = {
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Principal: "*",
Action: ["s3:GetObject"],
Resource: [`arn:aws:s3:::${bucketName}/*`],
},
],
};
await minioClient.setBucketPolicy(bucketName, JSON.stringify(policyJSON));
console.log(`Public-read policy applied to bucket ${bucketName}.`);
} catch (error) {
console.error(`Error configuring bucket ${bucketName}:`, error);
throw error;
}
}
minioClient.bucketExists('maxun-test')
.then((exists) => {
if (exists) {
@@ -81,41 +110,71 @@ class BinaryOutputService {
console.log(`Processing binary output key: ${key}`);
// Check if binaryData has a valid Buffer structure and parse it
if (binaryData && typeof binaryData.data === 'string') {
try {
const parsedData = JSON.parse(binaryData.data);
if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) {
binaryData = Buffer.from(parsedData.data);
} else {
console.error(`Invalid Buffer format for key: ${key}`);
// Convert binary data to Buffer (handles base64, data URI, and old Buffer format)
let bufferData: Buffer | null = null;
if (binaryData && typeof binaryData === 'object' && binaryData.data) {
const dataString = binaryData.data;
if (typeof dataString === 'string') {
try {
if (dataString.startsWith('data:')) {
const base64Match = dataString.match(/^data:([^;]+);base64,(.+)$/);
if (base64Match) {
bufferData = Buffer.from(base64Match[2], 'base64');
console.log(`Converted data URI to Buffer for key: ${key}`);
}
} else {
try {
const parsed = JSON.parse(dataString);
if (parsed?.type === 'Buffer' && Array.isArray(parsed.data)) {
bufferData = Buffer.from(parsed.data);
console.log(`Converted JSON Buffer format for key: ${key}`);
} else {
bufferData = Buffer.from(dataString, 'base64');
console.log(`Converted raw base64 to Buffer for key: ${key}`);
}
} catch {
bufferData = Buffer.from(dataString, 'base64');
console.log(`Converted raw base64 to Buffer for key: ${key}`);
}
}
} catch (error) {
console.error(`Failed to parse binary data for key ${key}:`, error);
continue;
}
} catch (error) {
console.error(`Failed to parse JSON for key: ${key}`, error);
continue;
}
}
// Handle cases where binaryData might not be a Buffer
if (!Buffer.isBuffer(binaryData)) {
console.error(`Binary data for key ${key} is not a valid Buffer.`);
if (!bufferData || !Buffer.isBuffer(bufferData)) {
console.error(`Invalid or empty buffer for key ${key}`);
continue;
}
try {
const minioKey = `${plainRun.runId}/${key}`;
await fixMinioBucketConfiguration(this.bucketName);
await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData);
const minioKey = `${plainRun.runId}/${encodeURIComponent(key.trim().replace(/\s+/g, '_'))}`;
// Construct the public URL for the uploaded object
// todo: use minio endpoint
const publicUrl = `http://localhost:${process.env.MINIO_PORT}/${this.bucketName}/${minioKey}`;
console.log(`Uploading to bucket ${this.bucketName} with key ${minioKey}`);
await minioClient.putObject(
this.bucketName,
minioKey,
bufferData,
bufferData.length,
{ 'Content-Type': binaryData.mimeType || 'image/png' }
);
const publicHost = process.env.MINIO_PUBLIC_HOST || 'http://localhost';
const publicPort = process.env.MINIO_PORT || '9000';
const publicUrl = `${publicHost}:${publicPort}/${this.bucketName}/${minioKey}`;
// Save the public URL in the result object
uploadedBinaryOutput[key] = publicUrl;
console.log(`✅ Uploaded and stored: ${publicUrl}`);
} catch (error) {
console.error(`Error uploading key ${key} to MinIO:`, error);
console.error(`Error uploading key ${key} to MinIO:`, error);
}
}

View File

@@ -726,34 +726,108 @@ export class WorkflowGenerator {
/**
* Generates a pair for the custom action event.
*
* @param action The type of the custom action.
* @param settings The settings of the custom action.
* @param actionId The unique identifier for this action (for updates)
* @param settings The settings of the custom action (may include name and actionId).
* @param page The page to use for obtaining the needed data.
*/
public customAction = async (action: CustomActions, settings: any, page: Page) => {
const pair: WhereWhatPair = {
where: { url: this.getBestUrl(page.url()) },
what: [{
action,
args: settings ? Array.isArray(settings) ? settings : [settings] : [],
}],
}
public customAction = async (action: CustomActions, actionId: string, settings: any, page: Page) => {
try {
let actionSettings = settings;
let actionName: string | undefined;
await this.addPairToWorkflowAndNotifyClient(pair, page);
if (settings && !Array.isArray(settings)) {
actionName = settings.name;
actionSettings = JSON.parse(JSON.stringify(settings));
delete actionSettings.name;
}
if (this.generatedData.lastUsedSelector) {
const elementInfo = await this.getLastUsedSelectorInfo(page, this.generatedData.lastUsedSelector);
const pair: WhereWhatPair = {
where: { url: this.getBestUrl(page.url()) },
what: [{
action,
args: actionSettings
? Array.isArray(actionSettings)
? actionSettings
: [actionSettings]
: [],
...(actionName ? { name: actionName } : {}),
...(actionId ? { actionId } : {}),
}],
};
this.socket.emit('decision', {
pair, actionType: 'customAction',
lastData: {
selector: this.generatedData.lastUsedSelector,
action: this.generatedData.lastAction,
tagName: elementInfo.tagName,
innerText: elementInfo.innerText,
if (actionId) {
const existingIndex = this.workflowRecord.workflow.findIndex(
(workflowPair) =>
Array.isArray(workflowPair.what) &&
workflowPair.what.some((whatItem: any) => whatItem.actionId === actionId)
);
if (existingIndex !== -1) {
const existingPair = this.workflowRecord.workflow[existingIndex];
const existingAction = existingPair.what.find((whatItem: any) => whatItem.actionId === actionId);
const updatedAction = {
...existingAction,
action,
args: Array.isArray(actionSettings)
? actionSettings
: [actionSettings],
name: actionName || existingAction?.name || '',
actionId,
};
this.workflowRecord.workflow[existingIndex] = {
where: JSON.parse(JSON.stringify(existingPair.where)),
what: existingPair.what.map((whatItem: any) =>
whatItem.actionId === actionId ? updatedAction : whatItem
),
};
if (action === 'scrapeSchema' && actionName) {
this.workflowRecord.workflow.forEach((pair, index) => {
pair.what.forEach((whatItem: any, whatIndex: number) => {
if (whatItem.action === 'scrapeSchema' && whatItem.actionId !== actionId) {
this.workflowRecord.workflow[index].what[whatIndex] = {
...whatItem,
name: actionName
};
}
});
});
}
} else {
await this.addPairToWorkflowAndNotifyClient(pair, page);
logger.log("debug", `Added new workflow action: ${action} with actionId: ${actionId}`);
}
});
}
} else {
await this.addPairToWorkflowAndNotifyClient(pair, page);
logger.log("debug", `Added new workflow action: ${action} without actionId`);
}
if (this.generatedData.lastUsedSelector) {
const elementInfo = await this.getLastUsedSelectorInfo(
page,
this.generatedData.lastUsedSelector
);
this.socket.emit('decision', {
pair,
actionType: 'customAction',
lastData: {
selector: this.generatedData.lastUsedSelector,
action: this.generatedData.lastAction,
tagName: elementInfo.tagName,
innerText: elementInfo.innerText,
},
});
}
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling customAction: ${message}`);
}
};
/**
@@ -810,6 +884,48 @@ export class WorkflowGenerator {
}
};
/**
* Removes an action with the given actionId from the workflow.
* Only removes the specific action from the what array, not the entire pair.
* If the what array becomes empty after removal, then the entire pair is removed.
* @param actionId The actionId of the action to remove
* @returns boolean indicating whether an action was removed
*/
public removeAction = (actionId: string): boolean => {
let actionWasRemoved = false;
this.workflowRecord.workflow = this.workflowRecord.workflow
.map((pair) => {
const filteredWhat = pair.what.filter(
(whatItem: any) => whatItem.actionId !== actionId
);
if (filteredWhat.length < pair.what.length) {
actionWasRemoved = true;
if (filteredWhat.length > 0) {
return {
...pair,
what: filteredWhat
};
}
return null;
}
return pair;
})
.filter((pair) => pair !== null) as WhereWhatPair[]; // Remove null entries
if (actionWasRemoved) {
logger.log("info", `Action with actionId ${actionId} removed from workflow`);
} else {
logger.log("debug", `No action found with actionId ${actionId}`);
}
return actionWasRemoved;
};
/**
* Updates the socket used for communication with the client.
* @param socket The socket to be used for communication.

View File

@@ -91,13 +91,16 @@ export class WorkflowInterpreter {
* Storage for different types of serializable data
*/
public serializableDataByType: {
scrapeSchema: any[],
scrapeList: any[],
scrapeSchema: Record<string, any>;
scrapeList: Record<string, any>;
[key: string]: any;
} = {
scrapeSchema: [],
scrapeList: [],
scrapeSchema: {},
scrapeList: {},
};
private currentActionName: string | null = null;
/**
* Track the current action type being processed
*/
@@ -106,7 +109,7 @@ export class WorkflowInterpreter {
/**
* An array of all the binary data extracted from the run.
*/
public binaryData: { mimetype: string, data: string }[] = [];
public binaryData: { name: string; mimeType: string; data: string }[] = [];
/**
* Track current scrapeList index
@@ -259,14 +262,19 @@ export class WorkflowInterpreter {
}
},
binaryCallback: async (data: string, mimetype: string) => {
const binaryItem = { mimetype, data: JSON.stringify(data) };
// For editor mode, we don't have the name yet, so use a timestamp-based name
const binaryItem = {
name: `Screenshot ${Date.now()}`,
mimeType: mimetype,
data: JSON.stringify(data)
};
this.binaryData.push(binaryItem);
// Persist binary data to database
await this.persistBinaryDataToDatabase(binaryItem);
this.socket.emit('binaryCallback', {
data,
this.socket.emit('binaryCallback', {
data,
mimetype,
type: 'captureScreenshot'
});
@@ -364,9 +372,10 @@ export class WorkflowInterpreter {
this.breakpoints = [];
this.interpretationResume = null;
this.currentActionType = null;
this.currentActionName = null;
this.serializableDataByType = {
scrapeSchema: [],
scrapeList: [],
scrapeSchema: {},
scrapeList: {},
};
this.binaryData = [];
this.currentScrapeListIndex = 0;
@@ -409,7 +418,7 @@ export class WorkflowInterpreter {
* Persists binary data to database in real-time
* @private
*/
private persistBinaryDataToDatabase = async (binaryItem: { mimetype: string, data: string }): Promise<void> => {
private persistBinaryDataToDatabase = async (binaryItem: { name: string; mimeType: string; data: string }): Promise<void> => {
if (!this.currentRunId) {
logger.log('debug', 'No run ID available for binary data persistence');
return;
@@ -422,22 +431,29 @@ export class WorkflowInterpreter {
return;
}
const currentBinaryOutput = run.binaryOutput ?
JSON.parse(JSON.stringify(run.binaryOutput)) :
{};
const uniqueKey = `item-${Date.now()}-${Object.keys(currentBinaryOutput).length}`;
const currentBinaryOutput =
run.binaryOutput && typeof run.binaryOutput === 'object'
? JSON.parse(JSON.stringify(run.binaryOutput))
: {};
const baseName = binaryItem.name?.trim() || `Screenshot ${Object.keys(currentBinaryOutput).length + 1}`;
let uniqueName = baseName;
let counter = 1;
while (currentBinaryOutput[uniqueName]) {
uniqueName = `${baseName} (${counter++})`;
}
const updatedBinaryOutput = {
...currentBinaryOutput,
[uniqueKey]: binaryItem
[uniqueName]: binaryItem,
};
await run.update({
binaryOutput: updatedBinaryOutput
});
logger.log('debug', `Persisted binary data for run ${this.currentRunId}: ${binaryItem.mimetype}`);
logger.log('debug', `Persisted binary data for run ${this.currentRunId}: ${binaryItem.name} (${binaryItem.mimeType})`);
} catch (error: any) {
logger.log('error', `Failed to persist binary data in real-time for run ${this.currentRunId}: ${error.message}`);
}
@@ -478,41 +494,101 @@ export class WorkflowInterpreter {
},
incrementScrapeListIndex: () => {
this.currentScrapeListIndex++;
}
},
setActionName: (name: string) => {
this.currentActionName = name;
},
},
serializableCallback: async (data: any) => {
if (this.currentActionType === 'scrapeSchema') {
if (Array.isArray(data) && data.length > 0) {
mergedScrapeSchema = { ...mergedScrapeSchema, ...data[0] };
this.serializableDataByType.scrapeSchema.push(data);
} else {
mergedScrapeSchema = { ...mergedScrapeSchema, ...data };
this.serializableDataByType.scrapeSchema.push([data]);
try {
if (!data || typeof data !== "object") return;
if (!this.currentActionType && Array.isArray(data) && data.length > 0) {
const first = data[0];
if (first && Object.keys(first).some(k => k.toLowerCase().includes("label") || k.toLowerCase().includes("text"))) {
this.currentActionType = "scrapeSchema";
}
}
// Persist the cumulative scrapeSchema data
const cumulativeScrapeSchemaData = Object.keys(mergedScrapeSchema).length > 0 ? [mergedScrapeSchema] : [];
if (cumulativeScrapeSchemaData.length > 0) {
await this.persistDataToDatabase('scrapeSchema', cumulativeScrapeSchemaData);
let typeKey = this.currentActionType || "unknown";
if (this.currentActionType === "scrapeList") {
typeKey = "scrapeList";
} else if (this.currentActionType === "scrapeSchema") {
typeKey = "scrapeSchema";
}
} else if (this.currentActionType === 'scrapeList') {
if (data && Array.isArray(data) && data.length > 0) {
// Use the current index for persistence
await this.persistDataToDatabase('scrapeList', data, this.currentScrapeListIndex);
if (this.currentActionType === "scrapeList" && data.scrapeList) {
data = data.scrapeList;
} else if (this.currentActionType === "scrapeSchema" && data.scrapeSchema) {
data = data.scrapeSchema;
}
this.serializableDataByType.scrapeList[this.currentScrapeListIndex] = data;
}
this.socket.emit('serializableCallback', data);
let actionName = this.currentActionName || "";
if (!actionName) {
if (!Array.isArray(data) && Object.keys(data).length === 1) {
const soleKey = Object.keys(data)[0];
const soleValue = data[soleKey];
if (Array.isArray(soleValue) || typeof soleValue === "object") {
actionName = soleKey;
data = soleValue;
}
}
}
if (!actionName) {
actionName = "Unnamed Action";
}
const flattened = Array.isArray(data)
? data
: (data?.List ?? (data && typeof data === 'object' ? Object.values(data).flat?.() ?? data : []));
if (!this.serializableDataByType[typeKey]) {
this.serializableDataByType[typeKey] = {};
}
this.serializableDataByType[typeKey][actionName] = flattened;
await this.persistDataToDatabase(typeKey, { [actionName]: flattened });
this.socket.emit("serializableCallback", {
type: typeKey,
name: actionName,
data: flattened,
});
this.currentActionType = null;
this.currentActionName = null;
} catch (err: any) {
logger.log('error', `serializableCallback handler failed: ${err.message}`);
}
},
binaryCallback: async (data: string, mimetype: string) => {
const binaryItem = { mimetype, data: JSON.stringify(data) };
this.binaryData.push(binaryItem);
// Persist binary data to database
await this.persistBinaryDataToDatabase(binaryItem);
this.socket.emit('binaryCallback', { data, mimetype });
binaryCallback: async (payload: { name: string; data: Buffer; mimeType: string }) => {
try {
const { name, data, mimeType } = payload;
const base64Data = data.toString("base64");
const binaryItem = {
name,
mimeType,
data: base64Data
};
this.binaryData.push(binaryItem);
await this.persistBinaryDataToDatabase(binaryItem);
this.socket.emit("binaryCallback", {
name,
data: base64Data,
mimeType
});
} catch (err: any) {
logger.log("error", `binaryCallback handler failed: ${err.message}`);
}
}
}
@@ -542,20 +618,13 @@ export class WorkflowInterpreter {
const result = {
log: this.debugMessages,
result: status,
scrapeSchemaOutput: Object.keys(mergedScrapeSchema).length > 0
? { "schema_merged": [mergedScrapeSchema] }
: this.serializableDataByType.scrapeSchema.reduce((reducedObject, item, index) => {
reducedObject[`schema_${index}`] = item;
return reducedObject;
}, {} as Record<string, any>),
scrapeListOutput: this.serializableDataByType.scrapeList.reduce((reducedObject, item, index) => {
reducedObject[`list_${index}`] = item;
return reducedObject;
}, {} as Record<string, any>),
binaryOutput: this.binaryData.reduce((reducedObject, item, index) => {
reducedObject[`item_${index}`] = item;
return reducedObject;
}, {} as Record<string, any>)
scrapeSchemaOutput: this.serializableDataByType.scrapeSchema,
scrapeListOutput: this.serializableDataByType.scrapeList,
binaryOutput: this.binaryData.reduce<Record<string, { data: string; mimeType: string }>>((acc, item) => {
const key = item.name || `Screenshot ${Object.keys(acc).length + 1}`;
acc[key] = { data: item.data, mimeType: item.mimeType };
return acc;
}, {})
}
logger.log('debug', `Interpretation finished`);
@@ -642,19 +711,37 @@ export class WorkflowInterpreter {
const currentSerializableOutput = run.serializableOutput ?
JSON.parse(JSON.stringify(run.serializableOutput)) :
{ scrapeSchema: [], scrapeList: [] };
if (Array.isArray(currentSerializableOutput.scrapeList)) {
currentSerializableOutput.scrapeList = {};
}
if (Array.isArray(currentSerializableOutput.scrapeSchema)) {
currentSerializableOutput.scrapeSchema = {};
}
let hasUpdates = false;
const mergeLists = (target: Record<string, any>, updates: Record<string, any>) => {
for (const [key, val] of Object.entries(updates)) {
const flattened = Array.isArray(val)
? val
: (val?.List ?? (val && typeof val === 'object' ? Object.values(val).flat?.() ?? val : []));
target[key] = flattened;
}
};
for (const item of batchToProcess) {
if (item.actionType === 'scrapeSchema') {
const newSchemaData = Array.isArray(item.data) ? item.data : [item.data];
currentSerializableOutput.scrapeSchema = newSchemaData;
hasUpdates = true;
} else if (item.actionType === 'scrapeList' && typeof item.listIndex === 'number') {
if (!Array.isArray(currentSerializableOutput.scrapeList)) {
currentSerializableOutput.scrapeList = [];
if (!currentSerializableOutput.scrapeSchema || typeof currentSerializableOutput.scrapeSchema !== 'object') {
currentSerializableOutput.scrapeSchema = {};
}
currentSerializableOutput.scrapeList[item.listIndex] = item.data;
mergeLists(currentSerializableOutput.scrapeSchema, item.data);
hasUpdates = true;
} else if (item.actionType === 'scrapeList') {
if (!currentSerializableOutput.scrapeList || typeof currentSerializableOutput.scrapeList !== 'object') {
currentSerializableOutput.scrapeList = {};
}
mergeLists(currentSerializableOutput.scrapeList, item.data);
hasUpdates = true;
}
}

View File

@@ -12,8 +12,8 @@ interface AirtableUpdateTask {
}
interface SerializableOutput {
scrapeSchema?: any[];
scrapeList?: any[];
scrapeSchema?: Record<string, any[]>;
scrapeList?: Record<string, any[]>;
}
const MAX_RETRIES = 3;
@@ -48,47 +48,94 @@ async function refreshAirtableToken(refreshToken: string) {
function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput: Record<string, string>) {
const allRecords: Record<string, any>[] = [];
const schemaData: Array<{key: string, value: any}> = [];
const schemaData: Array<{ Group: string; Field: string; Value: any }> = [];
const listData: any[] = [];
const screenshotData: Array<{key: string, url: string}> = [];
// Collect schema data
if (serializableOutput.scrapeSchema) {
for (const schemaArray of serializableOutput.scrapeSchema) {
if (!Array.isArray(schemaArray)) continue;
for (const schemaItem of schemaArray) {
Object.entries(schemaItem).forEach(([key, value]) => {
if (key && key.trim() !== '' && value !== null && value !== undefined && value !== '') {
schemaData.push({key, value});
}
});
if (Array.isArray(serializableOutput.scrapeSchema)) {
for (const schemaArray of serializableOutput.scrapeSchema) {
if (!Array.isArray(schemaArray)) continue;
for (const schemaItem of schemaArray) {
Object.entries(schemaItem || {}).forEach(([key, value]) => {
if (key && key.trim() !== "" && value !== null && value !== undefined && value !== "") {
schemaData.push({ Group: "Default", Field: key, Value: value });
}
});
}
}
} else if (typeof serializableOutput.scrapeSchema === "object") {
for (const [groupName, schemaArray] of Object.entries(serializableOutput.scrapeSchema)) {
if (!Array.isArray(schemaArray)) continue;
for (const schemaItem of schemaArray) {
Object.entries(schemaItem || {}).forEach(([fieldName, value]) => {
if (fieldName && fieldName.trim() !== "" && value !== null && value !== undefined && value !== "") {
schemaData.push({
Group: groupName,
Field: fieldName,
Value: value,
});
}
});
}
}
}
}
// Collect list data
if (serializableOutput.scrapeList) {
for (const listArray of serializableOutput.scrapeList) {
if (!Array.isArray(listArray)) continue;
listArray.forEach(listItem => {
const hasContent = Object.values(listItem).some(value =>
value !== null && value !== undefined && value !== ''
);
if (hasContent) {
listData.push(listItem);
}
});
if (Array.isArray(serializableOutput.scrapeList)) {
for (const listArray of serializableOutput.scrapeList) {
if (!Array.isArray(listArray)) continue;
listArray.forEach((listItem) => {
const hasContent = Object.values(listItem || {}).some(
(value) => value !== null && value !== undefined && value !== ""
);
if (hasContent) listData.push(listItem);
});
}
} else if (typeof serializableOutput.scrapeList === "object") {
for (const [listName, listArray] of Object.entries(serializableOutput.scrapeList)) {
if (!Array.isArray(listArray)) continue;
listArray.forEach((listItem) => {
const hasContent = Object.values(listItem || {}).some(
(value) => value !== null && value !== undefined && value !== ""
);
if (hasContent) listData.push({ List: listName, ...listItem });
});
}
}
}
// Collect screenshot data
if (binaryOutput && Object.keys(binaryOutput).length > 0) {
Object.entries(binaryOutput).forEach(([key, url]) => {
if (key && key.trim() !== '' && url && url.trim() !== '') {
screenshotData.push({key, url});
}
});
}
// if (binaryOutput && Object.keys(binaryOutput).length > 0) {
// Object.entries(binaryOutput).forEach(([key, rawValue]: [string, any]) => {
// if (!key || key.trim() === "") return;
// let urlString = "";
// // Case 1: old format (string URL)
// if (typeof rawValue === "string") {
// urlString = rawValue;
// }
// // Case 2: new format (object with { url?, data?, mimeType? })
// else if (rawValue && typeof rawValue === "object") {
// const valueObj = rawValue as { url?: string; data?: string; mimeType?: string };
// if (typeof valueObj.url === "string") {
// urlString = valueObj.url;
// } else if (typeof valueObj.data === "string") {
// const mime = valueObj.mimeType || "image/png";
// urlString = `data:${mime};base64,${valueObj.data}`;
// }
// }
// if (typeof urlString === "string" && urlString.trim() !== "") {
// screenshotData.push({ key, url: urlString });
// }
// });
// }
// Mix all data types together to create consecutive records
const maxLength = Math.max(schemaData.length, listData.length, screenshotData.length);
@@ -97,8 +144,9 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
const record: Record<string, any> = {};
if (i < schemaData.length) {
record.Label = schemaData[i].key;
record.Value = schemaData[i].value;
record.Group = schemaData[i].Group;
record.Label = schemaData[i].Field;
record.Value = schemaData[i].Value;
}
if (i < listData.length) {
@@ -120,20 +168,15 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
}
for (let i = maxLength; i < schemaData.length; i++) {
allRecords.push({
Label: schemaData[i].key,
Value: schemaData[i].value
});
allRecords.push({ Label: schemaData[i].Field, Value: schemaData[i].Value });
}
for (let i = maxLength; i < listData.length; i++) {
allRecords.push(listData[i]);
}
for (let i = maxLength; i < screenshotData.length; i++) {
allRecords.push({
Key: screenshotData[i].key,
Screenshot: screenshotData[i].url
Screenshot: screenshotData[i].url,
});
}

View File

@@ -49,25 +49,34 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
const serializableOutput = plainRun.serializableOutput as SerializableOutput;
if (serializableOutput) {
if (serializableOutput.scrapeSchema && serializableOutput.scrapeSchema.length > 0) {
await processOutputType(
robotId,
spreadsheetId,
'Text',
serializableOutput.scrapeSchema,
plainRobot
);
if (serializableOutput.scrapeSchema && typeof serializableOutput.scrapeSchema === "object") {
for (const [groupName, schemaArray] of Object.entries(serializableOutput.scrapeSchema)) {
if (!Array.isArray(schemaArray) || schemaArray.length === 0) continue;
await processOutputType(
robotId,
spreadsheetId,
`Schema - ${groupName}`,
schemaArray,
plainRobot
);
}
}
if (serializableOutput.scrapeList && serializableOutput.scrapeList.length > 0) {
await processOutputType(
robotId,
spreadsheetId,
'List',
serializableOutput.scrapeList,
plainRobot
);
if (serializableOutput.scrapeList && typeof serializableOutput.scrapeList === "object") {
for (const [listName, listArray] of Object.entries(serializableOutput.scrapeList)) {
if (!Array.isArray(listArray) || listArray.length === 0) continue;
await processOutputType(
robotId,
spreadsheetId,
`List - ${listName}`,
listArray,
plainRobot
);
}
}
}
if (plainRun.binaryOutput && Object.keys(plainRun.binaryOutput).length > 0) {
@@ -102,30 +111,27 @@ async function processOutputType(
outputData: any[],
robotConfig: any
) {
for (let i = 0; i < outputData.length; i++) {
const data = outputData[i];
if (!data || data.length === 0) {
console.log(`No data to write for ${outputType}-${i}. Skipping.`);
continue;
}
const sheetName = `${outputType}-${i}`;
await ensureSheetExists(spreadsheetId, sheetName, robotConfig);
let formattedData = data;
if (outputType === 'Text' && data.length > 0) {
const schemaItem = data[0];
formattedData = Object.entries(schemaItem).map(([key, value]) => ({
Label: key,
Value: value
}));
}
await writeDataToSheet(robotId, spreadsheetId, formattedData, sheetName, robotConfig);
console.log(`Data written to ${sheetName} sheet for ${outputType} data`);
const data = outputData;
const sheetName = outputType;
if (!Array.isArray(data) || data.length === 0) {
console.log(`No data to write for ${sheetName}. Skipping.`);
return;
}
await ensureSheetExists(spreadsheetId, sheetName, robotConfig);
const formattedData = data.map(item => {
const flatRow: Record<string, any> = {};
for (const [key, value] of Object.entries(item || {})) {
flatRow[key] =
typeof value === "object" && value !== null ? JSON.stringify(value) : value;
}
return flatRow;
});
await writeDataToSheet(robotId, spreadsheetId, formattedData, sheetName, robotConfig);
console.log(`Data written to ${sheetName} sheet for ${outputType} data`);
}
async function ensureSheetExists(spreadsheetId: string, sheetName: string, robotConfig: any) {

View File

@@ -4,7 +4,7 @@ import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import { io, Socket } from "socket.io-client";
import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-management/controller';
import logger from '../../logger';
import { browserPool } from "../../server";
import { browserPool, io as serverIo } from "../../server";
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integrations/gsheet";
import Robot from "../../models/Robot";
import Run from "../../models/Run";
@@ -46,7 +46,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
};
}
const browserId = createRemoteBrowserForRun( userId);
const browserId = createRemoteBrowserForRun(userId);
const runId = uuid();
const run = await Run.create({
@@ -63,10 +63,30 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
runByScheduleId: uuid(),
serializableOutput: {},
binaryOutput: {},
retryCount: 0
});
const plainRun = run.toJSON();
try {
const runScheduledData = {
runId: plainRun.runId,
robotMetaId: plainRun.robotMetaId,
robotName: plainRun.name,
status: 'scheduled',
startedAt: plainRun.startedAt,
runByUserId: plainRun.runByUserId,
runByScheduleId: plainRun.runByScheduleId,
runByAPI: plainRun.runByAPI || false,
browserId: plainRun.browserId
};
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-scheduled', runScheduledData);
logger.log('info', `Scheduled run notification sent for run: ${plainRun.runId} to user-${userId}`);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-scheduled notification for run ${plainRun.runId}: ${socketError.message}`);
}
return {
browserId,
runId: plainRun.runId,
@@ -83,6 +103,29 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
}
}
async function triggerIntegrationUpdates(runId: string, robotMetaId: string): Promise<void> {
try {
googleSheetUpdateTasks[runId] = {
robotId: robotMetaId,
runId: runId,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[runId] = {
robotId: robotMetaId,
runId: runId,
status: 'pending',
retries: 5,
};
processAirtableUpdates().catch(err => logger.log('error', `Airtable update error: ${err.message}`));
processGoogleSheetUpdates().catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
} catch (err: any) {
logger.log('error', `Failed to update integrations for run: ${runId}: ${err.message}`);
}
}
function AddGeneratedFlags(workflow: WorkflowFile) {
const copy = JSON.parse(JSON.stringify(workflow));
for (let i = 0; i < workflow.workflow.length; i++) {
@@ -95,6 +138,8 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
};
async function executeRun(id: string, userId: string) {
let browser: any = null;
try {
const run = await Run.findOne({ where: { runId: id } });
if (!run) {
@@ -133,6 +178,21 @@ async function executeRun(id: string, userId: string) {
log: plainRun.log ? `${plainRun.log}\nMax retries exceeded (3/3) - Run failed after multiple attempts.` : `Max retries exceeded (3/3) - Run failed after multiple attempts.`
});
try {
const failureSocketData = {
runId: plainRun.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording ? recording.recording_meta.name : 'Unknown Robot',
status: 'failed',
finishedAt: new Date().toLocaleString()
};
serverIo.of(run.browserId).emit('run-completed', failureSocketData);
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-completed', failureSocketData);
} catch (socketError: any) {
logger.log('warn', `Failed to emit failure event in main catch: ${socketError.message}`);
}
return {
success: false,
error: 'Max retries exceeded'
@@ -149,7 +209,22 @@ async function executeRun(id: string, userId: string) {
plainRun.status = 'running';
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
try {
const runStartedData = {
runId: plainRun.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording ? recording.recording_meta.name : 'Unknown Robot',
status: 'running',
startedAt: plainRun.startedAt
};
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-started', runStartedData);
logger.log('info', `Run started notification sent for run: ${plainRun.runId} to user-${userId}`);
} catch (socketError: any) {
logger.log('warn', `Failed to send run-started notification for run ${plainRun.runId}: ${socketError.message}`);
}
browser = browserPool.getRemoteBrowser(plainRun.browserId);
if (!browser) {
throw new Error('Could not access browser');
}
@@ -168,56 +243,52 @@ async function executeRun(id: string, userId: string) {
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings
);
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
const finalRun = await Run.findByPk(run.id);
const categorizedOutput = {
scrapeSchema: finalRun?.serializableOutput?.scrapeSchema || {},
scrapeList: finalRun?.serializableOutput?.scrapeList || {},
};
await destroyRemoteBrowser(plainRun.browserId, userId);
await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: interpretationInfo.log.join('\n'),
binaryOutput: uploadedBinaryOutput
});
// Upload binary output to MinIO and update run with MinIO URLs
const updatedRun = await Run.findOne({ where: { runId: id } });
if (updatedRun && updatedRun.binaryOutput && Object.keys(updatedRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(updatedRun, updatedRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for scheduled run ${id}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for scheduled run ${id}: ${minioError.message}`);
}
}
// Get metrics from persisted data for analytics and webhooks
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
if (updatedRun) {
if (updatedRun.serializableOutput) {
if (updatedRun.serializableOutput.scrapeSchema) {
Object.values(updatedRun.serializableOutput.scrapeSchema).forEach((schemaResult: any) => {
if (Array.isArray(schemaResult)) {
totalSchemaItemsExtracted += schemaResult.length;
} else if (schemaResult && typeof schemaResult === 'object') {
totalSchemaItemsExtracted += 1;
}
});
}
if (updatedRun.serializableOutput.scrapeList) {
Object.values(updatedRun.serializableOutput.scrapeList).forEach((listResult: any) => {
if (Array.isArray(listResult)) {
totalListItemsExtracted += listResult.length;
}
});
}
if (categorizedOutput) {
if (categorizedOutput.scrapeSchema) {
Object.values(categorizedOutput.scrapeSchema).forEach((schemaResult: any) => {
if (Array.isArray(schemaResult)) {
totalSchemaItemsExtracted += schemaResult.length;
} else if (schemaResult && typeof schemaResult === 'object') {
totalSchemaItemsExtracted += 1;
}
});
}
if (updatedRun.binaryOutput) {
extractedScreenshotsCount = Object.keys(updatedRun.binaryOutput).length;
if (categorizedOutput.scrapeList) {
Object.values(categorizedOutput.scrapeList).forEach((listResult: any) => {
if (Array.isArray(listResult)) {
totalListItemsExtracted += listResult.length;
}
});
}
}
if (run.binaryOutput) {
extractedScreenshotsCount = Object.keys(run.binaryOutput).length;
}
const totalRowsExtracted = totalSchemaItemsExtracted + totalListItemsExtracted;
@@ -234,6 +305,21 @@ async function executeRun(id: string, userId: string) {
}
);
try {
const completionData = {
runId: plainRun.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording.recording_meta.name,
status: 'success',
finishedAt: new Date().toLocaleString()
};
serverIo.of(plainRun.browserId).emit('run-completed', completionData);
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-completed', completionData);
} catch (emitError: any) {
logger.log('warn', `Failed to emit success event: ${emitError.message}`);
}
const webhookPayload = {
robot_id: plainRun.robotMetaId,
run_id: plainRun.runId,
@@ -242,16 +328,20 @@ async function executeRun(id: string, userId: string) {
started_at: plainRun.startedAt,
finished_at: new Date().toLocaleString(),
extracted_data: {
captured_texts: updatedRun?.serializableOutput?.scrapeSchema ? Object.values(updatedRun.serializableOutput.scrapeSchema).flat() : [],
captured_lists: updatedRun?.serializableOutput?.scrapeList || {},
total_rows: totalRowsExtracted,
captured_texts: Object.keys(categorizedOutput.scrapeSchema || {}).length > 0
? Object.entries(categorizedOutput.scrapeSchema).reduce((acc, [name, value]) => {
acc[name] = Array.isArray(value) ? value : [value];
return acc;
}, {} as Record<string, any[]>)
: {},
captured_lists: categorizedOutput.scrapeList,
captured_texts_count: totalSchemaItemsExtracted,
captured_lists_count: totalListItemsExtracted,
screenshots_count: extractedScreenshotsCount
},
metadata: {
browser_id: plainRun.browserId,
user_id: userId
user_id: userId,
}
};
@@ -262,26 +352,7 @@ async function executeRun(id: string, userId: string) {
logger.log('error', `Failed to send webhooks for run ${plainRun.runId}: ${webhookError.message}`);
}
try {
googleSheetUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
processAirtableUpdates().catch(err => logger.log('error', `Airtable update error: ${err.message}`));
processGoogleSheetUpdates().catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
} catch (err: any) {
logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`);
}
await triggerIntegrationUpdates(plainRun.runId, plainRun.robotMetaId);
return true;
} catch (error: any) {
logger.log('info', `Error while running a robot with id: ${id} - ${error.message}`);
@@ -320,6 +391,21 @@ async function executeRun(id: string, userId: string) {
} catch (webhookError: any) {
logger.log('error', `Failed to send failure webhooks for run ${run.runId}: ${webhookError.message}`);
}
try {
const failureSocketData = {
runId: run.runId,
robotMetaId: run.robotMetaId,
robotName: recording ? recording.recording_meta.name : 'Unknown Robot',
status: 'failed',
finishedAt: new Date().toLocaleString()
};
serverIo.of(run.browserId).emit('run-completed', failureSocketData);
serverIo.of('/queued-run').to(`user-${userId}`).emit('run-completed', failureSocketData);
} catch (socketError: any) {
logger.log('warn', `Failed to emit failure event in main catch: ${socketError.message}`);
}
}
capture(
'maxun-oss-run-created-scheduled',

View File

@@ -5,13 +5,14 @@ import Canvas from "../recorder/Canvas";
import { Highlighter } from "../recorder/Highlighter";
import { GenericModal } from '../ui/GenericModal';
import { useActionContext } from '../../context/browserActions';
import { useBrowserSteps, TextStep } from '../../context/browserSteps';
import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps';
import { useGlobalInfoStore } from '../../context/globalInfo';
import { useTranslation } from 'react-i18next';
import { AuthContext } from '../../context/auth';
import { coordinateMapper } from '../../helpers/coordinateMapper';
import { useBrowserDimensionsStore } from '../../context/browserDimensions';
import { clientSelectorGenerator, ElementFingerprint } from "../../helpers/clientSelectorGenerator";
import { capturedElementHighlighter } from "../../helpers/capturedElementHighlighter";
import DatePicker from "../pickers/DatePicker";
import Dropdown from "../pickers/Dropdown";
import TimePicker from "../pickers/TimePicker";
@@ -182,10 +183,13 @@ export const BrowserWindow = () => {
count?: number;
} | null>(null);
const [initialAutoFieldIds, setInitialAutoFieldIds] = useState<Set<number>>(new Set());
const [manuallyAddedFieldIds, setManuallyAddedFieldIds] = useState<Set<number>>(new Set());
const { socket } = useSocketStore();
const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode, currentSnapshot } = useGlobalInfoStore();
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
const { addTextStep, addListStep } = useBrowserSteps();
const { addTextStep, addListStep, browserSteps } = useBrowserSteps();
const [currentGroupInfo, setCurrentGroupInfo] = useState<{
isGroupElement: boolean;
@@ -1159,6 +1163,7 @@ export const BrowserWindow = () => {
if (Object.keys(autoFields).length > 0) {
setFields(autoFields);
setInitialAutoFieldIds(new Set(Object.keys(autoFields).map(id => parseInt(id))));
addListStep(
listSelector,
@@ -1195,6 +1200,11 @@ export const BrowserWindow = () => {
cachedListSelector,
pendingNotification,
notify,
createFieldsFromChildSelectors,
currentListId,
currentListActionId,
paginationSelector,
addListStep
]);
useEffect(() => {
@@ -1203,6 +1213,77 @@ export const BrowserWindow = () => {
}
}, [listSelector]);
useEffect(() => {
if (!getList || !listSelector || initialAutoFieldIds.size === 0 || !currentListActionId) return;
const currentListStep = browserSteps.find(
step => step.type === 'list' && step.actionId === currentListActionId
);
if (!currentListStep || currentListStep.type !== 'list' || !currentListStep.fields) return;
const currentFieldIds = new Set(Object.keys(currentListStep.fields).map(id => parseInt(id)));
const newManualIds = new Set<number>();
currentFieldIds.forEach(fieldId => {
if (!initialAutoFieldIds.has(fieldId)) {
newManualIds.add(fieldId);
}
});
if (newManualIds.size !== manuallyAddedFieldIds.size ||
![...newManualIds].every(id => manuallyAddedFieldIds.has(id))) {
setManuallyAddedFieldIds(newManualIds);
}
}, [browserSteps, getList, listSelector, initialAutoFieldIds, currentListActionId, manuallyAddedFieldIds]);
useEffect(() => {
if (!isDOMMode) {
capturedElementHighlighter.clearHighlights();
return;
}
const capturedSelectors: Array<{ selector: string }> = [];
if (getText && currentTextActionId) {
const textSteps = browserSteps.filter(
(step): step is TextStep => step.type === 'text' && step.actionId === currentTextActionId
);
textSteps.forEach(step => {
if (step.selectorObj?.selector) {
capturedSelectors.push({
selector: step.selectorObj.selector,
});
}
});
}
if (getList && listSelector && currentListActionId && manuallyAddedFieldIds.size > 0) {
const listSteps = browserSteps.filter(
step => step.type === 'list' && step.actionId === currentListActionId
) as ListStep[];
listSteps.forEach(listStep => {
if (listStep.fields) {
Object.entries(listStep.fields).forEach(([fieldId, field]: [string, any]) => {
if (manuallyAddedFieldIds.has(parseInt(fieldId)) && field.selectorObj?.selector) {
capturedSelectors.push({
selector: field.selectorObj.selector,
});
}
});
}
});
}
if (capturedSelectors.length > 0) {
capturedElementHighlighter.applyHighlights(capturedSelectors);
} else {
capturedElementHighlighter.clearHighlights();
}
}, [browserSteps, getText, getList, listSelector, currentTextActionId, currentListActionId, isDOMMode, manuallyAddedFieldIds]);
useEffect(() => {
coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height);
}, [viewportInfo, dimensions.width, dimensions.height]);
@@ -1216,7 +1297,6 @@ export const BrowserWindow = () => {
useEffect(() => {
const storedListSelector = sessionStorage.getItem('recordingListSelector');
// Only restore state if it exists in sessionStorage
if (storedListSelector && !listSelector) {
setListSelector(storedListSelector);
}
@@ -1225,7 +1305,6 @@ export const BrowserWindow = () => {
const onMouseMove = (e: MouseEvent) => {
if (canvasRef && canvasRef.current && highlighterData) {
const canvasRect = canvasRef.current.getBoundingClientRect();
// mousemove outside the browser window
if (
e.pageX < canvasRect.left
|| e.pageX > canvasRect.right
@@ -1242,6 +1321,8 @@ export const BrowserWindow = () => {
setFields({});
setCurrentListId(null);
setCachedChildSelectors([]);
setInitialAutoFieldIds(new Set());
setManuallyAddedFieldIds(new Set());
}, []);
useEffect(() => {
@@ -1262,7 +1343,7 @@ export const BrowserWindow = () => {
}
}
}
}, [screenShot, user?.id]);
}, [user?.id]);
useEffect(() => {
if (socket) {
@@ -1456,7 +1537,6 @@ export const BrowserWindow = () => {
}
highlighterUpdateRef.current = now;
// Map the incoming DOMRect from browser coordinates to canvas coordinates
const mappedRect = new DOMRect(
data.rect.x,
data.rect.y,
@@ -1477,17 +1557,14 @@ export const BrowserWindow = () => {
if (limitMode) {
setHighlighterData(null);
} else if (paginationMode) {
// Only set highlighterData if type is not empty, 'none', 'scrollDown', or 'scrollUp'
if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) {
setHighlighterData(mappedData);
} else {
setHighlighterData(null);
}
} else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) {
// Highlight only valid child elements within the listSelector
setHighlighterData(mappedData);
} else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) {
// Handle iframe elements
const isIframeChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes(':>>') &&
childSelector.split(':>>').some(part =>
@@ -1496,7 +1573,6 @@ export const BrowserWindow = () => {
);
setHighlighterData(isIframeChild ? mappedData : null);
} else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) {
// Handle mixed DOM cases with iframes
const selectorParts = mappedData.selector.split(':>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
@@ -1505,7 +1581,6 @@ export const BrowserWindow = () => {
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) {
// Handle Shadow DOM elements
const isShadowChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes('>>') &&
childSelector.split('>>').some(part =>
@@ -1514,7 +1589,6 @@ export const BrowserWindow = () => {
);
setHighlighterData(isShadowChild ? mappedData : null);
} else if (mappedData.selector.includes('>>') && hasValidChildSelectors) {
// Handle mixed DOM cases
const selectorParts = mappedData.selector.split('>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
@@ -1523,15 +1597,12 @@ export const BrowserWindow = () => {
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else {
// If not a valid child in normal mode, clear the highlighter
setHighlighterData(null);
}
} else {
// Set highlighterData for the initial listSelector selection
setHighlighterData(mappedData);
}
} else {
// For non-list steps
setHighlighterData(mappedData);
}
}, [getList, socket, listSelector, paginationMode, paginationType, limitMode]);

View File

@@ -702,12 +702,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
return;
}
if (isInCaptureMode) {
e.preventDefault();
e.stopPropagation();
return;
}
if (!isInCaptureMode) {
const wheelEvent = e as WheelEvent;
const deltaX = Math.round(wheelEvent.deltaX / 10) * 10;

View File

@@ -1,19 +1,13 @@
import React, { useState, useCallback, useEffect, useMemo } from 'react';
import { Button, Paper, Box, TextField, IconButton, Tooltip } from "@mui/material";
import EditIcon from '@mui/icons-material/Edit';
import TextFieldsIcon from '@mui/icons-material/TextFields';
import DocumentScannerIcon from '@mui/icons-material/DocumentScanner';
import { WorkflowFile } from "maxun-core";
import Typography from "@mui/material/Typography";
import { useGlobalInfoStore } from "../../context/globalInfo";
import { PaginationType, useActionContext, LimitType } from '../../context/browserActions';
import { BrowserStep, useBrowserSteps } from '../../context/browserSteps';
import { useSocketStore } from '../../context/socket';
import { ScreenshotSettings } from '../../shared/types';
import InputAdornment from '@mui/material/InputAdornment';
import FormControlLabel from '@mui/material/FormControlLabel';
import FormControl from '@mui/material/FormControl';
import FormLabel from '@mui/material/FormLabel';
import Radio from '@mui/material/Radio';
import RadioGroup from '@mui/material/RadioGroup';
import { getActiveWorkflow } from "../../api/workflow";
@@ -49,13 +43,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
const [showCaptureList, setShowCaptureList] = useState(true);
const [showCaptureScreenshot, setShowCaptureScreenshot] = useState(true);
const [showCaptureText, setShowCaptureText] = useState(true);
const [hoverStates, setHoverStates] = useState<{ [id: string]: boolean }>({});
const [browserStepIdList, setBrowserStepIdList] = useState<number[]>([]);
const [isCaptureTextConfirmed, setIsCaptureTextConfirmed] = useState(false);
const [isCaptureListConfirmed, setIsCaptureListConfirmed] = useState(false);
const { panelHeight } = useBrowserDimensionsStore();
const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, setIsDOMMode, currentSnapshot, setCurrentSnapshot, updateDOMMode, initialUrl, setRecordingUrl } = useGlobalInfoStore();
const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, setIsDOMMode, currentSnapshot, setCurrentSnapshot, updateDOMMode, initialUrl, setRecordingUrl, currentTextGroupName } = useGlobalInfoStore();
const {
getText, startGetText, stopGetText,
getList, startGetList, stopGetList,
@@ -72,7 +62,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
startAction, finishAction
} = useActionContext();
const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId, updateListStepData, updateScreenshotStepData } = useBrowserSteps();
const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps();
const { id, socket } = useSocketStore();
const { t } = useTranslation();
@@ -183,6 +173,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
if (screenshotSteps.length > 0) {
const latestStep = screenshotSteps[screenshotSteps.length - 1];
updateScreenshotStepData(latestStep.id, data.screenshot);
emitActionForStep(latestStep);
}
setCurrentScreenshotActionId('');
@@ -194,7 +185,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
socket.off('directScreenshotCaptured', handleDirectScreenshot);
};
}
}, [socket, id, notify, t, currentScreenshotActionId, updateScreenshotStepData, setCurrentScreenshotActionId]);
}, [socket, id, notify, t, currentScreenshotActionId, updateScreenshotStepData, setCurrentScreenshotActionId, emitActionForStep, browserSteps]);
const extractDataClientSide = useCallback(
(
@@ -271,26 +262,41 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
}
}
},
[isDOMMode, currentSnapshot, updateListStepData, socket, notify]
[isDOMMode, currentSnapshot, updateListStepData, socket, notify, currentWorkflowActionsState]
);
const handleMouseEnter = (id: number) => {
setHoverStates(prev => ({ ...prev, [id]: true }));
};
useEffect(() => {
if (!getList) return;
const handleMouseLeave = (id: number) => {
setHoverStates(prev => ({ ...prev, [id]: false }));
};
const currentListStep = browserSteps.find(
step => step.type === 'list' && step.actionId === currentListActionId
) as (BrowserStep & { type: 'list'; listSelector?: string; fields?: Record<string, any> }) | undefined;
if (!currentListStep || !currentListStep.listSelector || !currentListStep.fields) return;
const fieldCount = Object.keys(currentListStep.fields).length;
if (fieldCount > 0) {
extractDataClientSide(
currentListStep.listSelector,
currentListStep.fields,
currentListStep.id
);
setCurrentWorkflowActionsState({
...currentWorkflowActionsState,
hasScrapeListAction: true
});
}
}, [browserSteps, currentListActionId, getList, extractDataClientSide, setCurrentWorkflowActionsState, currentWorkflowActionsState]);
const handleStartGetText = () => {
setIsCaptureTextConfirmed(false);
const newActionId = `text-${crypto.randomUUID()}`;
setCurrentTextActionId(newActionId);
startGetText();
}
const handleStartGetList = () => {
setIsCaptureListConfirmed(false);
const newActionId = `list-${crypto.randomUUID()}`;
setCurrentListActionId(newActionId);
startGetList();
@@ -302,230 +308,24 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
startGetScreenshot();
};
const handleTextLabelChange = (id: number, label: string, listId?: number, fieldKey?: string) => {
if (listId !== undefined && fieldKey !== undefined) {
// Prevent editing if the field is confirmed
if (confirmedListTextFields[listId]?.[fieldKey]) {
return;
}
updateListTextFieldLabel(listId, fieldKey, label);
} else {
setTextLabels(prevLabels => ({ ...prevLabels, [id]: label }));
}
if (!label.trim()) {
setErrors(prevErrors => ({ ...prevErrors, [id]: t('right_panel.errors.label_required') }));
} else {
setErrors(prevErrors => ({ ...prevErrors, [id]: '' }));
}
};
const handleTextStepConfirm = (id: number) => {
const label = textLabels[id]?.trim();
if (!label) {
setErrors(prevErrors => ({ ...prevErrors, [id]: t('right_panel.errors.label_required') }));
return;
}
const existingLabels = browserSteps
.filter(step =>
step.type === 'text' &&
step.id !== id &&
confirmedTextSteps[step.id] &&
'label' in step &&
step.label
)
.map(step => (step as any).label);
if (existingLabels.includes(label)) {
setErrors(prevErrors => ({
...prevErrors,
[id]: t('right_panel.errors.duplicate_label') || `Label "${label}" already exists. Please use a unique label.`
}));
return;
}
updateBrowserTextStepLabel(id, label);
setConfirmedTextSteps(prev => ({ ...prev, [id]: true }));
setErrors(prevErrors => ({ ...prevErrors, [id]: '' }));
};
const handleTextStepDiscard = (id: number) => {
deleteBrowserStep(id);
setTextLabels(prevLabels => {
const { [id]: _, ...rest } = prevLabels;
return rest;
});
setErrors(prevErrors => {
const { [id]: _, ...rest } = prevErrors;
return rest;
});
};
const handleTextStepDelete = (id: number) => {
deleteBrowserStep(id);
setTextLabels(prevLabels => {
const { [id]: _, ...rest } = prevLabels;
return rest;
});
setConfirmedTextSteps(prev => {
const { [id]: _, ...rest } = prev;
return rest;
});
setErrors(prevErrors => {
const { [id]: _, ...rest } = prevErrors;
return rest;
});
};
const handleListTextFieldConfirm = (listId: number, fieldKey: string) => {
setConfirmedListTextFields(prev => ({
...prev,
[listId]: {
...(prev[listId] || {}),
[fieldKey]: true
}
}));
};
const handleListTextFieldDiscard = (listId: number, fieldKey: string) => {
removeListTextField(listId, fieldKey);
setConfirmedListTextFields(prev => {
const updatedListFields = { ...(prev[listId] || {}) };
delete updatedListFields[fieldKey];
return {
...prev,
[listId]: updatedListFields
};
});
setErrors(prev => {
const { [fieldKey]: _, ...rest } = prev;
return rest;
});
};
const handleListTextFieldDelete = (listId: number, fieldKey: string) => {
removeListTextField(listId, fieldKey);
setConfirmedListTextFields(prev => {
const updatedListFields = { ...(prev[listId] || {}) };
delete updatedListFields[fieldKey];
return {
...prev,
[listId]: updatedListFields
};
});
setErrors(prev => {
const { [fieldKey]: _, ...rest } = prev;
return rest;
});
};
const getTextSettingsObject = useCallback(() => {
const settings: Record<string, {
selector: string;
tag?: string;
[key: string]: any
}> = {};
browserSteps.forEach(step => {
if (browserStepIdList.includes(step.id)) {
return;
}
if (step.type === 'text' && step.label && step.selectorObj?.selector) {
settings[step.label] = {
...step.selectorObj,
selector: step.selectorObj.selector
};
}
setBrowserStepIdList(prevList => [...prevList, step.id]);
});
return settings;
}, [browserSteps, browserStepIdList]);
const stopCaptureAndEmitGetTextSettings = useCallback(() => {
const hasTextStepsForCurrentAction = browserSteps.some(step => step.type === 'text' && step.actionId === currentTextActionId);
if (!hasTextStepsForCurrentAction) {
const currentTextActionStep = browserSteps.find(step => step.type === 'text' && step.actionId === currentTextActionId);
if (!currentTextActionStep) {
notify('error', t('right_panel.errors.no_text_captured'));
return;
}
const hasUnconfirmedTextStepsForCurrentAction = browserSteps.some(step =>
step.type === 'text' &&
step.actionId === currentTextActionId &&
!confirmedTextSteps[step.id]
);
if (hasUnconfirmedTextStepsForCurrentAction) {
notify('error', t('right_panel.errors.confirm_text_fields'));
return;
}
stopGetText();
const settings = getTextSettingsObject();
if (hasTextStepsForCurrentAction) {
socket?.emit('action', { action: 'scrapeSchema', settings });
if (currentTextActionStep) {
emitActionForStep(currentTextActionStep);
}
setIsCaptureTextConfirmed(true);
setCurrentTextActionId('');
resetInterpretationLog();
finishAction('text');
onFinishCapture();
clientSelectorGenerator.cleanup();
}, [stopGetText, getTextSettingsObject, socket, browserSteps, confirmedTextSteps, resetInterpretationLog, finishAction, notify, onFinishCapture, t]);
}, [stopGetText, socket, browserSteps, resetInterpretationLog, finishAction, notify, onFinishCapture, t, currentTextActionId, currentTextGroupName, emitActionForStep]);
const getListSettingsObject = useCallback(() => {
let settings: {
listSelector?: string;
fields?: Record<string, {
selector: string;
tag?: string;
[key: string]: any;
isShadow?: boolean;
}>;
pagination?: {
type: string;
selector?: string;
isShadow?: boolean;
};
limit?: number;
isShadow?: boolean;
} = {};
browserSteps.forEach(step => {
if (step.type === 'list' && step.listSelector && Object.keys(step.fields).length > 0) {
const fields: Record<string, {
selector: string;
tag?: string;
[key: string]: any;
isShadow?: boolean;
}> = {};
Object.entries(step.fields).forEach(([id, field]) => {
if (field.selectorObj?.selector) {
fields[field.label] = {
selector: field.selectorObj.selector,
tag: field.selectorObj.tag,
attribute: field.selectorObj.attribute,
isShadow: field.selectorObj.isShadow
};
}
});
settings = {
listSelector: step.listSelector,
fields: fields,
pagination: {
type: paginationType,
selector: step.pagination?.selector,
isShadow: step.isShadow
},
limit: parseInt(limitType === 'custom' ? customLimit : limitType),
isShadow: step.isShadow
};
}
});
return settings;
}, [browserSteps, paginationType, limitType, customLimit]);
const resetListState = useCallback(() => {
setShowPaginationOptions(false);
@@ -541,32 +341,33 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
}, [stopGetList, resetListState]);
const stopCaptureAndEmitGetListSettings = useCallback(() => {
const settings = getListSettingsObject();
const latestListStep = getLatestListStep(browserSteps);
if (latestListStep && settings) {
if (latestListStep) {
extractDataClientSide(latestListStep.listSelector!, latestListStep.fields, latestListStep.id);
setCurrentWorkflowActionsState({
...currentWorkflowActionsState,
hasScrapeListAction: true
});
socket?.emit('action', { action: 'scrapeList', settings });
emitActionForStep(latestListStep);
handleStopGetList();
setCurrentListActionId('');
resetInterpretationLog();
finishAction('list');
onFinishCapture();
clientSelectorGenerator.cleanup();
} else {
notify('error', t('right_panel.errors.unable_create_settings'));
handleStopGetList();
setCurrentListActionId('');
resetInterpretationLog();
finishAction('list');
onFinishCapture();
clientSelectorGenerator.cleanup();
}
handleStopGetList();
setCurrentListActionId('');
resetInterpretationLog();
finishAction('list');
onFinishCapture();
clientSelectorGenerator.cleanup();
}, [getListSettingsObject, socket, notify, handleStopGetList, resetInterpretationLog, finishAction, onFinishCapture, t, browserSteps, extractDataClientSide]);
const hasUnconfirmedListTextFields = browserSteps.some(step =>
step.type === 'list' &&
step.actionId === currentListActionId &&
Object.entries(step.fields).some(([fieldKey]) =>
!confirmedListTextFields[step.id]?.[fieldKey]
)
);
}, [socket, notify, handleStopGetList, resetInterpretationLog, finishAction, onFinishCapture, t, browserSteps, extractDataClientSide, setCurrentWorkflowActionsState, currentWorkflowActionsState, emitActionForStep]);
const getLatestListStep = (steps: BrowserStep[]) => {
const listSteps = steps.filter(step => step.type === 'list');
@@ -590,19 +391,6 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
return;
}
const hasUnconfirmedListTextFieldsForCurrentAction = browserSteps.some(step =>
step.type === 'list' &&
step.actionId === currentListActionId &&
Object.entries(step.fields).some(([fieldKey]) =>
!confirmedListTextFields[step.id]?.[fieldKey]
)
);
if (hasUnconfirmedListTextFieldsForCurrentAction) {
notify('error', t('right_panel.errors.confirm_all_list_fields'));
return;
}
startPaginationMode();
setShowPaginationOptions(true);
setCaptureStage('pagination');
@@ -613,11 +401,17 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
notify('error', t('right_panel.errors.select_pagination'));
return;
}
const settings = getListSettingsObject();
const paginationSelector = settings.pagination?.selector;
if (['clickNext', 'clickLoadMore'].includes(paginationType) && !paginationSelector) {
notify('error', t('right_panel.errors.select_pagination_element'));
return;
const currentListStepForPagination = browserSteps.find(
step => step.type === 'list' && step.actionId === currentListActionId
) as (BrowserStep & { type: 'list' }) | undefined;
if (currentListStepForPagination) {
const paginationSelector = currentListStepForPagination.pagination?.selector;
if (['clickNext', 'clickLoadMore'].includes(paginationType) && !paginationSelector) {
notify('error', t('right_panel.errors.select_pagination_element'));
return;
}
}
stopPaginationMode();
setShowPaginationOptions(false);
@@ -644,7 +438,6 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
stopLimitMode();
setShowLimitOptions(false);
setIsCaptureListConfirmed(true);
stopCaptureAndEmitGetListSettings();
setCaptureStage('complete');
break;
@@ -653,7 +446,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
setCaptureStage('initial');
break;
}
}, [captureStage, paginationType, limitType, customLimit, startPaginationMode, setShowPaginationOptions, setCaptureStage, getListSettingsObject, notify, stopPaginationMode, startLimitMode, setShowLimitOptions, stopLimitMode, setIsCaptureListConfirmed, stopCaptureAndEmitGetListSettings, t, browserSteps, currentListActionId, confirmedListTextFields]);
}, [captureStage, paginationType, limitType, customLimit, startPaginationMode, setShowPaginationOptions, setCaptureStage, notify, stopPaginationMode, startLimitMode, setShowLimitOptions, stopLimitMode, stopCaptureAndEmitGetListSettings, t, browserSteps, currentListActionId, updateListStepLimit]);
const handleBackCaptureList = useCallback(() => {
switch (captureStage) {
@@ -679,61 +472,28 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
const discardGetText = useCallback(() => {
stopGetText();
if (currentTextActionId) {
const stepsToDelete = browserSteps
.filter(step => step.type === 'text' && step.actionId === currentTextActionId)
.map(step => step.id);
if (currentTextActionId) {
deleteStepsByActionId(currentTextActionId);
setTextLabels(prevLabels => {
const newLabels = { ...prevLabels };
stepsToDelete.forEach(id => {
delete newLabels[id];
});
return newLabels;
});
setErrors(prevErrors => {
const newErrors = { ...prevErrors };
stepsToDelete.forEach(id => {
delete newErrors[id];
});
return newErrors;
});
setConfirmedTextSteps(prev => {
const newConfirmed = { ...prev };
stepsToDelete.forEach(id => {
delete newConfirmed[id];
});
return newConfirmed;
});
if (socket) {
socket.emit('removeAction', { actionId: currentTextActionId });
}
}
setCurrentTextActionId('');
setIsCaptureTextConfirmed(false);
clientSelectorGenerator.cleanup();
notify('error', t('right_panel.errors.capture_text_discarded'));
}, [currentTextActionId, browserSteps, stopGetText, deleteStepsByActionId, notify, t]);
}, [currentTextActionId, browserSteps, stopGetText, deleteStepsByActionId, notify, t, socket]);
const discardGetList = useCallback(() => {
stopGetList();
if (currentListActionId) {
const listStepsToDelete = browserSteps
.filter(step => step.type === 'list' && step.actionId === currentListActionId)
.map(step => step.id);
deleteStepsByActionId(currentListActionId);
setConfirmedListTextFields(prev => {
const newConfirmed = { ...prev };
listStepsToDelete.forEach(id => {
delete newConfirmed[id];
});
return newConfirmed;
});
if (socket) {
socket.emit('removeAction', { actionId: currentListActionId });
}
}
resetListState();
@@ -743,12 +503,14 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
setShowLimitOptions(false);
setCaptureStage('initial');
setCurrentListActionId('');
setIsCaptureListConfirmed(false);
clientSelectorGenerator.cleanup();
notify('error', t('right_panel.errors.capture_list_discarded'));
}, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t]);
}, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t, stopPaginationMode, stopLimitMode, socket]);
const captureScreenshot = (fullPage: boolean) => {
const screenshotCount = browserSteps.filter(s => s.type === 'screenshot').length + 1;
const screenshotName = `Screenshot ${screenshotCount}`;
const screenshotSettings = {
fullPage,
type: 'png' as const,
@@ -756,38 +518,18 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
animations: 'allow' as const,
caret: 'hide' as const,
scale: 'device' as const,
name: screenshotName,
actionId: currentScreenshotActionId
};
socket?.emit('captureDirectScreenshot', screenshotSettings);
socket?.emit('action', { action: 'screenshot', settings: screenshotSettings });
addScreenshotStep(fullPage, currentScreenshotActionId);
stopGetScreenshot();
resetInterpretationLog();
finishAction('screenshot');
clientSelectorGenerator.cleanup();
onFinishCapture();
clientSelectorGenerator.cleanup();
};
const isConfirmCaptureDisabled = useMemo(() => {
if (captureStage !== 'initial') return false;
const hasValidListSelectorForCurrentAction = browserSteps.some(step =>
step.type === 'list' &&
step.actionId === currentListActionId &&
step.listSelector &&
Object.keys(step.fields).length > 0
);
const hasUnconfirmedListTextFieldsForCurrentAction = browserSteps.some(step =>
step.type === 'list' &&
step.actionId === currentListActionId &&
Object.entries(step.fields).some(([fieldKey]) =>
!confirmedListTextFields[step.id]?.[fieldKey]
)
);
return !hasValidListSelectorForCurrentAction || hasUnconfirmedListTextFieldsForCurrentAction;
}, [captureStage, browserSteps, currentListActionId, confirmedListTextFields]);
const theme = useThemeMode();
const isDarkMode = theme.darkMode;
@@ -842,33 +584,20 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
{t('right_panel.buttons.back')}
</Button>
)}
<Tooltip
title={
captureStage !== 'initial' && hasUnconfirmedListTextFields
? t('right_panel.tooltips.confirm_all_list_fields')
: ''
}
placement="top"
arrow
<Button
variant="outlined"
onClick={handleConfirmListCapture}
sx={{
color: '#ff00c3 !important',
borderColor: '#ff00c3 !important',
backgroundColor: 'whitesmoke !important',
}}
>
<span>
<Button
variant="outlined"
onClick={handleConfirmListCapture}
disabled={captureStage !== 'initial' && hasUnconfirmedListTextFields}
sx={{
color: '#ff00c3 !important',
borderColor: '#ff00c3 !important',
backgroundColor: 'whitesmoke !important',
}}
>
{captureStage === 'initial' ? t('right_panel.buttons.confirm_capture') :
captureStage === 'pagination' ? t('right_panel.buttons.confirm_pagination') :
captureStage === 'limit' ? t('right_panel.buttons.confirm_limit') :
t('right_panel.buttons.finish_capture')}
</Button>
</span>
</Tooltip>
</Button>
<Button
variant="outlined"
color="error"
@@ -1056,136 +785,6 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
</Box>
)}
</Box>
<Box>
{browserSteps.map(step => (
<Box key={step.id} onMouseEnter={() => handleMouseEnter(step.id)} onMouseLeave={() => handleMouseLeave(step.id)} sx={{ padding: '10px', margin: '11px', borderRadius: '5px', position: 'relative', background: isDarkMode ? "#1d1c1cff" : 'white', color: isDarkMode ? "white" : 'black' }}>
{
step.type === 'text' && (
<>
<TextField
label={t('right_panel.fields.label')}
value={textLabels[step.id] || step.label || ''}
onChange={(e) => handleTextLabelChange(step.id, e.target.value)}
fullWidth
size="small"
margin="normal"
error={!!errors[step.id]}
helperText={errors[step.id]}
InputProps={{
readOnly: confirmedTextSteps[step.id],
startAdornment: (
<InputAdornment position="start">
<EditIcon />
</InputAdornment>
)
}}
/>
<TextField
label={t('right_panel.fields.data')}
value={step.data}
fullWidth
margin="normal"
InputProps={{
readOnly: confirmedTextSteps[step.id],
startAdornment: (
<InputAdornment position="start">
<TextFieldsIcon />
</InputAdornment>
)
}}
/>
{!confirmedTextSteps[step.id] ? (
<Box display="flex" justifyContent="space-between" gap={2}>
<Button variant="contained" onClick={() => handleTextStepConfirm(step.id)} disabled={!textLabels[step.id]?.trim()}>{t('right_panel.buttons.confirm')}</Button>
<Button variant="contained" color="error" onClick={() => handleTextStepDiscard(step.id)}>{t('right_panel.buttons.discard')}</Button>
</Box>
) : !isCaptureTextConfirmed && (
<Box display="flex" justifyContent="flex-end" gap={2}>
<Button
variant="contained"
color="error"
onClick={() => handleTextStepDelete(step.id)}
>
{t('right_panel.buttons.delete')}
</Button>
</Box>
)}
</>
)}
{step.type === 'screenshot' && (
<Box display="flex" alignItems="center">
<DocumentScannerIcon sx={{ mr: 1 }} />
<Typography>
{step.fullPage ?
t('right_panel.screenshot.display_fullpage') :
t('right_panel.screenshot.display_visible')}
</Typography>
</Box>
)}
{step.type === 'list' && (
Object.entries(step.fields).length === 0 ? (
<Typography>{t('right_panel.messages.list_empty')}</Typography>
) : (
<>
<Typography>{t('right_panel.messages.list_selected')}</Typography>
{Object.entries(step.fields).map(([key, field]) => (
<Box key={key}>
<TextField
label={t('right_panel.fields.field_label')}
value={field.label || ''}
onChange={(e) => handleTextLabelChange(field.id, e.target.value, step.id, key)}
fullWidth
margin="normal"
InputProps={{
readOnly: confirmedListTextFields[field.id]?.[key],
startAdornment: (
<InputAdornment position="start">
<EditIcon />
</InputAdornment>
)
}}
/>
<TextField
label={t('right_panel.fields.field_data')}
value={field.data || ''}
fullWidth
margin="normal"
InputProps={{
readOnly: true,
startAdornment: (
<InputAdornment position="start">
<TextFieldsIcon />
</InputAdornment>
)
}}
/>
{!confirmedListTextFields[step.id]?.[key] && (
<Box display="flex" justifyContent="space-between" gap={2}>
<Button
variant="contained"
onClick={() => handleListTextFieldConfirm(step.id, key)}
disabled={!field.label?.trim()}
>
{t('right_panel.buttons.confirm')}
</Button>
<Button
variant="contained"
color="error"
onClick={() => handleListTextFieldDiscard(step.id, key)}
>
{t('right_panel.buttons.discard')}
</Button>
</Box>
)}
</Box>
))}
</>
)
)}
</Box>
))}
</Box>
</Paper>
);
};

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,4 +1,7 @@
import React, { createContext, useContext, useState } from 'react';
import React, { createContext, useContext, useEffect, useRef, useState } from 'react';
import { useSocketStore } from "./socket";
import { useGlobalInfoStore } from "./globalInfo";
import { useActionContext } from './browserActions';
export interface TextStep {
id: number;
@@ -8,11 +11,13 @@ export interface TextStep {
isShadow?: boolean;
selectorObj: SelectorObject;
actionId?: string;
name?: string;
}
interface ScreenshotStep {
export interface ScreenshotStep {
id: number;
type: 'screenshot';
name?: string;
fullPage: boolean;
actionId?: string;
screenshotData?: string;
@@ -21,6 +26,7 @@ interface ScreenshotStep {
export interface ListStep {
id: number;
type: 'list';
name?: string;
listSelector: string;
isShadow?: boolean;
fields: { [key: string]: TextStep };
@@ -31,112 +37,338 @@ export interface ListStep {
};
limit?: number;
actionId?: string;
data?: any[];
}
export type BrowserStep = TextStep | ScreenshotStep | ListStep;
export interface SelectorObject {
selector: string;
isShadow?: boolean;
tag?: string;
attribute?: string;
isShadow?: boolean;
[key: string]: any;
}
interface BrowserStepsContextType {
browserSteps: BrowserStep[];
addTextStep: (label: string, data: string, selectorObj: SelectorObject, actionId: string) => void;
addListStep: (listSelector: string, fields: { [key: string]: TextStep }, listId: number, actionId: string, pagination?: { type: string; selector: string, isShadow?: boolean }, limit?: number, isShadow?: boolean) => void
addTextStep: (
label: string,
data: string,
selectorObj: SelectorObject,
actionId: string
) => void;
addListStep: (
listSelector: string,
fields: { [key: string]: TextStep },
listId: number,
actionId: string,
pagination?: {
type: string;
selector: string;
isShadow?: boolean;
},
limit?: number,
isShadow?: boolean
) => void;
addScreenshotStep: (fullPage: boolean, actionId: string) => void;
deleteBrowserStep: (id: number) => void;
updateBrowserTextStepLabel: (id: number, newLabel: string) => void;
updateListTextFieldLabel: (listId: number, fieldKey: string, newLabel: string) => void;
updateListTextFieldLabel: (
listId: number,
fieldKey: string,
newLabel: string
) => void;
updateListStepLimit: (listId: number, limit: number) => void;
updateListStepData: (listId: number, extractedData: any[]) => void;
updateListStepName: (listId: number, name: string) => void;
updateScreenshotStepName: (id: number, name: string) => void;
removeListTextField: (listId: number, fieldKey: string) => void;
deleteStepsByActionId: (actionId: string) => void;
updateScreenshotStepData: (id: number, screenshotData: string) => void;
emitActionForStep: (step: BrowserStep) => void;
emitForStepId: (actionId: string, nameOverride?: string) => void;
}
const BrowserStepsContext = createContext<BrowserStepsContextType | undefined>(undefined);
export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const { socket } = useSocketStore();
const { currentTextGroupName } = useGlobalInfoStore();
const [browserSteps, setBrowserSteps] = useState<BrowserStep[]>([]);
const [discardedFields, setDiscardedFields] = useState<Set<string>>(new Set());
const { paginationType, limitType, customLimit } = useActionContext();
const browserStepsRef = useRef<BrowserStep[]>(browserSteps);
useEffect(() => {
browserStepsRef.current = browserSteps;
}, [browserSteps]);
const currentTextGroupNameRef = useRef(currentTextGroupName);
useEffect(() => {
currentTextGroupNameRef.current = currentTextGroupName;
}, [currentTextGroupName]);
const getListSettingsObject = (listStep: ListStep) => {
const fields: Record<string, {
selector: string;
tag?: string;
[key: string]: any;
isShadow?: boolean;
}> = {};
Object.entries(listStep.fields).forEach(([id, field]) => {
if (field.selectorObj?.selector) {
fields[field.label] = {
selector: field.selectorObj.selector,
tag: field.selectorObj.tag,
attribute: field.selectorObj.attribute,
isShadow: field.selectorObj.isShadow
};
}
});
const livePaginationType = paginationType || listStep.pagination?.type || "";
const liveLimit =
limitType === "custom"
? parseInt(customLimit || "0", 10)
: parseInt(limitType || "0", 10);
return {
listSelector: listStep.listSelector,
fields: fields,
pagination: {
type: livePaginationType,
selector: listStep.pagination?.selector,
isShadow: listStep.isShadow
},
limit: liveLimit > 0 ? liveLimit : listStep.limit,
isShadow: listStep.isShadow
};
};
const emitActionForStep = (step: BrowserStep) => {
if (!socket) return;
if (!step.actionId) return;
if (!socket.connected) return;
let action = "";
let settings: any = {};
// Always read the latest steps from the ref to prevent stale data
const latestSteps = browserStepsRef.current;
if (step.type === "list") {
action = "scrapeList";
const baseSettings = getListSettingsObject(step);
settings = {
...baseSettings,
name: step.name || `List Data ${latestSteps.filter(s => s.type === "list").length}`,
};
} else if (step.type === "text") {
action = "scrapeSchema";
const freshTextSteps = latestSteps.filter(
(s): s is TextStep => s.type === "text" && s.actionId === step.actionId
);
// Build schema settings from text steps
const fieldSettings: Record<
string,
{
selector: string;
tag?: string;
[key: string]: any;
}
> = {};
freshTextSteps.forEach((textStep) => {
if (textStep.selectorObj?.selector && textStep.label) {
fieldSettings[textStep.label] = {
selector: textStep.selectorObj.selector,
tag: textStep.selectorObj.tag,
attribute: textStep.selectorObj.attribute,
isShadow: textStep.selectorObj.isShadow,
};
}
});
settings = {
...fieldSettings,
name: currentTextGroupNameRef.current || "Text Data",
};
} else if (step.type === "screenshot") {
action = "screenshot";
const freshScreenshot = latestSteps.find(
(s) => s.type === "screenshot" && s.actionId === step.actionId
) as ScreenshotStep | undefined;
settings = {
name:
step.name ||
freshScreenshot?.name ||
`Screenshot ${latestSteps.filter((s) => s.type === "screenshot").length}`,
type: "png",
caret: "hide",
scale: "device",
timeout: 30000,
fullPage: freshScreenshot?.fullPage ?? step.fullPage ?? true,
animations: "allow",
};
}
socket.emit("action", { action, actionId: step.actionId, settings });
};
const emitForStepId = (actionId: string, nameOverride?: string) => {
const step = browserStepsRef.current.find(s => s.actionId === actionId);
if (!step) return;
let enrichedStep = { ...step };
if (step.type === "text") {
enrichedStep = { ...step, name: currentTextGroupNameRef.current };
}
if (step.type === "screenshot") {
const freshScreenshot = browserStepsRef.current.find(
s => s.type === "screenshot" && s.actionId === actionId
) as ScreenshotStep | undefined;
if (freshScreenshot) {
enrichedStep = { ...freshScreenshot };
if (nameOverride && freshScreenshot.name !== nameOverride) {
enrichedStep.name = nameOverride;
browserStepsRef.current = browserStepsRef.current.map(s =>
s.id === freshScreenshot.id ? { ...s, name: nameOverride } : s
);
setBrowserSteps(prev =>
prev.map(s =>
s.id === freshScreenshot.id ? { ...s, name: nameOverride } : s
)
);
}
}
}
if (step.type === "list") {
const freshList = browserStepsRef.current.find(
s => s.type === "list" && s.actionId === actionId
) as ListStep | undefined;
if (freshList) {
enrichedStep = { ...freshList };
}
}
emitActionForStep(enrichedStep);
};
const addTextStep = (label: string, data: string, selectorObj: SelectorObject, actionId: string) => {
setBrowserSteps(prevSteps => [
...prevSteps,
{ id: Date.now(), type: 'text', label, data, selectorObj, actionId }
]);
setBrowserSteps((prevSteps) => {
const textCount = prevSteps.filter(s => s.type === 'text').length + 1;
const generatedLabel = label || `Label ${textCount}`;
return [
...prevSteps,
{
id: Date.now(),
type: "text",
label: generatedLabel,
data,
selectorObj,
actionId,
},
];
});
};
const addListStep = (
listSelector: string,
newFields: { [key: string]: TextStep },
listId: number,
actionId: string,
pagination?: { type: string; selector: string; isShadow?: boolean },
listSelector: string,
newFields: { [key: string]: TextStep },
listId: number,
actionId: string,
pagination?: {
type: string;
selector: string;
isShadow?: boolean;
},
limit?: number,
isShadow?: boolean
) => {
setBrowserSteps(prevSteps => {
const existingListStepIndex = prevSteps.findIndex(step => step.type === 'list' && step.id === listId);
setBrowserSteps((prevSteps) => {
const existingListStepIndex = prevSteps.findIndex(
(step) => step.type === "list" && step.id === listId
);
if (existingListStepIndex !== -1) {
const updatedSteps = [...prevSteps];
const existingListStep = updatedSteps[existingListStepIndex] as ListStep;
const existingListStep = updatedSteps[
existingListStepIndex
] as ListStep;
// Preserve existing labels for fields
const mergedFields = Object.entries(newFields).reduce((acc, [key, field]) => {
if (!discardedFields.has(`${listId}-${key}`)) {
// If field exists, preserve its label
if (existingListStep.fields[key]) {
acc[key] = {
...field,
label: existingListStep.fields[key].label,
actionId
};
} else {
acc[key] = {
...field,
actionId
};
const mergedFields = Object.entries(newFields).reduce(
(acc, [key, field]) => {
if (!discardedFields.has(`${listId}-${key}`)) {
// If field exists, preserve its label
if (existingListStep.fields[key]) {
acc[key] = {
...field,
label: existingListStep.fields[key].label,
actionId,
};
} else {
acc[key] = {
...field,
actionId,
};
}
}
}
return acc;
}, {} as { [key: string]: TextStep });
return acc;
},
{} as { [key: string]: TextStep }
);
updatedSteps[existingListStepIndex] = {
...existingListStep,
listSelector,
fields: mergedFields,
pagination: pagination || existingListStep.pagination,
limit: limit,
isShadow: isShadow !== undefined ? isShadow : existingListStep.isShadow,
actionId,
isShadow: isShadow !== undefined ? isShadow : existingListStep.isShadow
};
return updatedSteps;
} else {
const fieldsWithActionId = Object.entries(newFields).reduce((acc, [key, field]) => {
acc[key] = {
...field,
actionId
};
return acc;
}, {} as { [key: string]: TextStep });
const fieldsWithActionId = Object.entries(newFields).reduce(
(acc, [key, field]) => {
acc[key] = {
...field,
actionId,
};
return acc;
},
{} as { [key: string]: TextStep }
);
const listCount = prevSteps.filter(s => s.type === 'list').length + 1;
return [
...prevSteps,
{
id: listId,
type: 'list',
listSelector,
fields: fieldsWithActionId,
pagination,
limit,
{
id: listId,
type: "list",
name: `List Data ${listCount}`,
listSelector,
fields: fieldsWithActionId,
pagination,
limit,
actionId,
isShadow
}
},
];
}
});
@@ -165,22 +397,39 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({
);
};
const updateListTextFieldLabel = (listId: number, fieldKey: string, newLabel: string) => {
setBrowserSteps(prevSteps =>
prevSteps.map(step => {
if (step.type === 'list' && step.id === listId) {
// Ensure deep copy of the fields object
const updateListTextFieldLabel = (
listId: number,
fieldKey: string,
newLabel: string
) => {
setBrowserSteps((prevSteps) =>
prevSteps.map((step) => {
if (step.type === "list" && step.id === listId) {
const oldLabel = step.fields[fieldKey].label;
const updatedFields = {
...step.fields,
[fieldKey]: {
...step.fields[fieldKey],
label: newLabel
}
label: newLabel,
},
};
const updatedData = step.data?.map((row: any) => {
if (row[oldLabel] !== undefined) {
const { [oldLabel]: value, ...rest } = row;
return {
...rest,
[newLabel]: value,
};
}
return row;
});
return {
...step,
fields: updatedFields
fields: updatedFields,
data: updatedData,
};
}
return step;
@@ -194,7 +443,7 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({
if (step.type === 'list' && step.id === listId) {
return {
...step,
data: extractedData // Add the extracted data to the step
data: extractedData
};
}
return step;
@@ -217,49 +466,83 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({
};
const updateListStepLimit = (listId: number, limit: number) => {
setBrowserSteps(prevSteps =>
prevSteps.map(step => {
if (step.type === 'list' && step.id === listId) {
return {
...step,
limit: limit
};
}
return step;
})
);
};
const removeListTextField = (listId: number, fieldKey: string) => {
setBrowserSteps(prevSteps =>
prevSteps.map(step => {
if (step.type === 'list' && step.id === listId) {
const { [fieldKey]: _, ...remainingFields } = step.fields;
return {
...step,
fields: remainingFields
limit: limit
};
}
return step;
})
);
setDiscardedFields(prevDiscarded => new Set(prevDiscarded).add(`${listId}-${fieldKey}`));
};
const updateListStepName = (listId: number, name: string) => {
setBrowserSteps((prevSteps) =>
prevSteps.map((step) => {
if (step.type === "list" && step.id === listId) {
return {
...step,
name: name,
};
}
return step;
})
);
};
const updateScreenshotStepName = (id: number, name: string) => {
setBrowserSteps(prevSteps => {
const updated = prevSteps.map(step =>
step.id === id && step.type === 'screenshot'
? { ...step, name }
: step
);
browserStepsRef.current = updated;
return updated;
});
};
const removeListTextField = (listId: number, fieldKey: string) => {
setBrowserSteps((prevSteps) =>
prevSteps.map((step) => {
if (step.type === "list" && step.id === listId) {
const { [fieldKey]: _, ...remainingFields } = step.fields;
return {
...step,
fields: remainingFields,
};
}
return step;
})
);
setDiscardedFields((prevDiscarded) =>
new Set(prevDiscarded).add(`${listId}-${fieldKey}`)
);
};
return (
<BrowserStepsContext.Provider value={{
browserSteps,
addTextStep,
addListStep,
addScreenshotStep,
deleteBrowserStep,
updateBrowserTextStepLabel,
updateListTextFieldLabel,
updateListStepLimit,
updateListStepData,
removeListTextField,
deleteStepsByActionId,
updateScreenshotStepData,
}}>
<BrowserStepsContext.Provider
value={{
browserSteps,
addTextStep,
addListStep,
addScreenshotStep,
deleteBrowserStep,
updateBrowserTextStepLabel,
updateListTextFieldLabel,
updateListStepLimit,
updateListStepData,
updateListStepName,
updateScreenshotStepName,
removeListTextField,
deleteStepsByActionId,
updateScreenshotStepData,
emitActionForStep,
emitForStepId
}}
>
{children}
</BrowserStepsContext.Provider>
);

View File

@@ -139,6 +139,8 @@ interface GlobalInfo {
setCurrentListActionId: (actionId: string) => void;
currentScreenshotActionId: string;
setCurrentScreenshotActionId: (actionId: string) => void;
currentTextGroupName: string;
setCurrentTextGroupName: (name: string) => void;
isDOMMode: boolean;
setIsDOMMode: (isDOMMode: boolean) => void;
currentSnapshot: ProcessedSnapshot | null;
@@ -173,6 +175,7 @@ class GlobalInfoStore implements Partial<GlobalInfo> {
currentTextActionId = '';
currentListActionId = '';
currentScreenshotActionId = '';
currentTextGroupName = 'Text Data';
isDOMMode = false;
currentSnapshot = null;
};
@@ -282,6 +285,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [currentTextActionId, setCurrentTextActionId] = useState<string>('');
const [currentListActionId, setCurrentListActionId] = useState<string>('');
const [currentScreenshotActionId, setCurrentScreenshotActionId] = useState<string>('');
const [currentTextGroupName, setCurrentTextGroupName] = useState<string>('Text Data');
const [isDOMMode, setIsDOMMode] = useState<boolean>(globalInfoStore.isDOMMode);
const [currentSnapshot, setCurrentSnapshot] = useState<ProcessedSnapshot | null>(globalInfoStore.currentSnapshot);
@@ -363,6 +367,8 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
setCurrentListActionId,
currentScreenshotActionId,
setCurrentScreenshotActionId,
currentTextGroupName,
setCurrentTextGroupName,
isDOMMode,
setIsDOMMode,
currentSnapshot,

View File

@@ -0,0 +1,84 @@
/**
* Helper class for managing persistent highlights of captured elements.
* Shows dotted highlights for elements that have been captured but not yet confirmed.
*/
class CapturedElementHighlighter {
private static readonly STYLE_ID = 'maxun-captured-elements-style';
/**
* Apply persistent dotted highlights to captured elements in the DOM iframe
* @param selectors Array of captured element selectors
*/
public applyHighlights(selectors: Array<{ selector: string }>): void {
const iframeDoc = this.getIframeDocument();
if (!iframeDoc) return;
// Remove existing highlights
this.clearHighlights();
// Create CSS rules for each captured selector
const cssRules: string[] = [];
selectors.forEach(({ selector }) => {
const cssSelector = this.getCSSSelector(selector);
if (cssSelector) {
cssRules.push(`
${cssSelector} {
outline: 2px dotted #ff00c3 !important;
outline-offset: 2px !important;
box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.5) !important;
}
`);
}
});
// Inject style element
if (cssRules.length > 0) {
const styleElement = iframeDoc.createElement('style');
styleElement.id = CapturedElementHighlighter.STYLE_ID;
styleElement.textContent = cssRules.join('\n');
iframeDoc.head.appendChild(styleElement);
}
}
/**
* Clear all persistent highlights from the DOM iframe
*/
public clearHighlights(): void {
const iframeDoc = this.getIframeDocument();
if (!iframeDoc) return;
const existingStyle = iframeDoc.getElementById(CapturedElementHighlighter.STYLE_ID);
if (existingStyle) {
existingStyle.remove();
}
}
/**
* Get the iframe document
*/
private getIframeDocument(): Document | null {
const iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement;
return iframeElement?.contentDocument || null;
}
/**
* Convert selector to CSS format for highlighting
*/
private getCSSSelector(selector: string): string {
// Handle XPath selectors by extracting data-mx-id
if (selector.startsWith('//') || selector.startsWith('(//')) {
const mxIdMatch = selector.match(/data-mx-id='([^']+)'/);
if (mxIdMatch) {
return `[data-mx-id='${mxIdMatch[1]}']`;
}
return '';
}
// Already a CSS selector
return selector;
}
}
export const capturedElementHighlighter = new CapturedElementHighlighter();

View File

@@ -2713,52 +2713,6 @@ class ClientSelectorGenerator {
}
}
const testId = element.getAttribute("data-testid");
if (testId && !addPositionToAll) {
const isCommon = this.isAttributeCommonAcrossLists(
element,
"data-testid",
testId,
otherListElements
);
if (isCommon) {
return `${tagName}[@data-testid='${testId}']`;
}
}
if (element.id && !element.id.match(/^\d/) && !addPositionToAll) {
const isCommon = this.isAttributeCommonAcrossLists(
element,
"id",
element.id,
otherListElements
);
if (isCommon) {
return `${tagName}[@id='${element.id}']`;
}
}
if (!addPositionToAll) {
for (const attr of Array.from(element.attributes)) {
if (
attr.name.startsWith("data-") &&
attr.name !== "data-testid" &&
attr.name !== "data-mx-id" &&
attr.value
) {
const isCommon = this.isAttributeCommonAcrossLists(
element,
attr.name,
attr.value,
otherListElements
);
if (isCommon) {
return `${tagName}[@${attr.name}='${attr.value}']`;
}
}
}
}
const position = this.getSiblingPosition(element, parent);
if (addPositionToAll || classes.length === 0) {