Merge branch 'develop' into rect-improve
This commit is contained in:
@@ -283,13 +283,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
|||||||
} else if (attribute === 'innerHTML') {
|
} else if (attribute === 'innerHTML') {
|
||||||
record[label] = fieldElement.innerHTML.trim();
|
record[label] = fieldElement.innerHTML.trim();
|
||||||
} else if (attribute === 'src') {
|
} else if (attribute === 'src') {
|
||||||
// Handle relative 'src' URLs
|
// Handle relative 'src' URLs
|
||||||
const src = fieldElement.getAttribute('src');
|
const src = fieldElement.getAttribute('src');
|
||||||
record[label] = src ? new URL(src, baseUrl).href : null;
|
record[label] = src ? new URL(src, window.location.origin).href : null;
|
||||||
} else if (attribute === 'href') {
|
} else if (attribute === 'href') {
|
||||||
// Handle relative 'href' URLs
|
// Handle relative 'href' URLs
|
||||||
const href = fieldElement.getAttribute('href');
|
const href = fieldElement.getAttribute('href');
|
||||||
record[label] = href ? new URL(href, baseUrl).href : null;
|
record[label] = href ? new URL(href, window.location.origin).href : null;
|
||||||
} else {
|
} else {
|
||||||
record[label] = fieldElement.getAttribute(attribute);
|
record[label] = fieldElement.getAttribute(attribute);
|
||||||
}
|
}
|
||||||
@@ -346,5 +346,5 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
|||||||
|
|
||||||
return results;
|
return results;
|
||||||
};
|
};
|
||||||
|
|
||||||
})(window);
|
})(window);
|
||||||
@@ -121,6 +121,53 @@ export default class Interpreter extends EventEmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// private getSelectors(workflow: Workflow, actionId: number): string[] {
|
||||||
|
// const selectors: string[] = [];
|
||||||
|
|
||||||
|
// // Validate actionId
|
||||||
|
// if (actionId <= 0) {
|
||||||
|
// console.log("No previous selectors to collect.");
|
||||||
|
// return selectors; // Empty array as there are no previous steps
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Iterate from the start up to (but not including) actionId
|
||||||
|
// for (let index = 0; index < actionId; index++) {
|
||||||
|
// const currentSelectors = workflow[index]?.where?.selectors;
|
||||||
|
// console.log(`Selectors at step ${index}:`, currentSelectors);
|
||||||
|
|
||||||
|
// if (currentSelectors && currentSelectors.length > 0) {
|
||||||
|
// currentSelectors.forEach((selector) => {
|
||||||
|
// if (!selectors.includes(selector)) {
|
||||||
|
// selectors.push(selector); // Avoid duplicates
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// console.log("Collected Selectors:", selectors);
|
||||||
|
// return selectors;
|
||||||
|
// }
|
||||||
|
|
||||||
|
private getSelectors(workflow: Workflow): string[] {
|
||||||
|
const selectorsSet = new Set<string>();
|
||||||
|
|
||||||
|
if (workflow.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let index = workflow.length - 1; index >= 0; index--) {
|
||||||
|
const currentSelectors = workflow[index]?.where?.selectors;
|
||||||
|
|
||||||
|
if (currentSelectors && currentSelectors.length > 0) {
|
||||||
|
currentSelectors.forEach((selector) => selectorsSet.add(selector));
|
||||||
|
return Array.from(selectorsSet);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the context object from given Page and the current workflow.\
|
* Returns the context object from given Page and the current workflow.\
|
||||||
* \
|
* \
|
||||||
@@ -130,52 +177,63 @@ export default class Interpreter extends EventEmitter {
|
|||||||
* @param workflow Current **initialized** workflow (array of where-what pairs).
|
* @param workflow Current **initialized** workflow (array of where-what pairs).
|
||||||
* @returns {PageState} State of the current page.
|
* @returns {PageState} State of the current page.
|
||||||
*/
|
*/
|
||||||
private async getState(page: Page, workflow: Workflow): Promise<PageState> {
|
private async getState(page: Page, workflowCopy: Workflow, selectors: string[]): Promise<PageState> {
|
||||||
/**
|
/**
|
||||||
* All the selectors present in the current Workflow
|
* All the selectors present in the current Workflow
|
||||||
*/
|
*/
|
||||||
const selectors = Preprocessor.extractSelectors(workflow);
|
// const selectors = Preprocessor.extractSelectors(workflow);
|
||||||
|
// console.log("Current selectors:", selectors);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
|
* Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
|
||||||
* @param selector Selector to be queried
|
* @param selector Selector to be queried
|
||||||
* @returns True if the targetted element is actionable, false otherwise.
|
* @returns True if the targetted element is actionable, false otherwise.
|
||||||
*/
|
*/
|
||||||
const actionable = async (selector: string): Promise<boolean> => {
|
// const actionable = async (selector: string): Promise<boolean> => {
|
||||||
try {
|
// try {
|
||||||
const proms = [
|
// const proms = [
|
||||||
page.isEnabled(selector, { timeout: 500 }),
|
// page.isEnabled(selector, { timeout: 5000 }),
|
||||||
page.isVisible(selector, { timeout: 500 }),
|
// page.isVisible(selector, { timeout: 5000 }),
|
||||||
];
|
// ];
|
||||||
|
|
||||||
return await Promise.all(proms).then((bools) => bools.every((x) => x));
|
// return await Promise.all(proms).then((bools) => bools.every((x) => x));
|
||||||
} catch (e) {
|
// } catch (e) {
|
||||||
// log(<Error>e, Level.ERROR);
|
// // log(<Error>e, Level.ERROR);
|
||||||
return false;
|
// return false;
|
||||||
}
|
// }
|
||||||
};
|
// };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Object of selectors present in the current page.
|
* Object of selectors present in the current page.
|
||||||
*/
|
*/
|
||||||
const presentSelectors: SelectorArray = await Promise.all(
|
// const presentSelectors: SelectorArray = await Promise.all(
|
||||||
selectors.map(async (selector) => {
|
// selectors.map(async (selector) => {
|
||||||
if (await actionable(selector)) {
|
// if (await actionable(selector)) {
|
||||||
return [selector];
|
// return [selector];
|
||||||
}
|
// }
|
||||||
return [];
|
// return [];
|
||||||
}),
|
// }),
|
||||||
).then((x) => x.flat());
|
// ).then((x) => x.flat());
|
||||||
|
|
||||||
|
const action = workflowCopy[workflowCopy.length - 1];
|
||||||
|
|
||||||
|
// console.log("Next action:", action)
|
||||||
|
|
||||||
|
let url: any = page.url();
|
||||||
|
|
||||||
|
if (action && action.where.url !== url && action.where.url !== "about:blank") {
|
||||||
|
url = action.where.url;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url: page.url(),
|
url,
|
||||||
cookies: (await page.context().cookies([page.url()]))
|
cookies: (await page.context().cookies([page.url()]))
|
||||||
.reduce((p, cookie) => (
|
.reduce((p, cookie) => (
|
||||||
{
|
{
|
||||||
...p,
|
...p,
|
||||||
[cookie.name]: cookie.value,
|
[cookie.name]: cookie.value,
|
||||||
}), {}),
|
}), {}),
|
||||||
selectors: presentSelectors,
|
selectors,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,6 +423,7 @@ export default class Interpreter extends EventEmitter {
|
|||||||
console.log("MERGED results:", mergedResult);
|
console.log("MERGED results:", mergedResult);
|
||||||
|
|
||||||
await this.options.serializableCallback(mergedResult);
|
await this.options.serializableCallback(mergedResult);
|
||||||
|
// await this.options.serializableCallback(scrapeResult);
|
||||||
},
|
},
|
||||||
|
|
||||||
scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => {
|
scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => {
|
||||||
@@ -548,11 +607,31 @@ export default class Interpreter extends EventEmitter {
|
|||||||
return allResults;
|
return allResults;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private getMatchingActionId(workflow: Workflow, pageState: PageState, usedActions: string[]) {
|
||||||
|
for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
|
||||||
|
const step = workflow[actionId];
|
||||||
|
const isApplicable = this.applicable(step.where, pageState, usedActions);
|
||||||
|
console.log("-------------------------------------------------------------");
|
||||||
|
console.log(`Where:`, step.where);
|
||||||
|
console.log(`Page state:`, pageState);
|
||||||
|
console.log(`Match result: ${isApplicable}`);
|
||||||
|
console.log("-------------------------------------------------------------");
|
||||||
|
|
||||||
|
if (isApplicable) {
|
||||||
|
return actionId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async runLoop(p: Page, workflow: Workflow) {
|
private async runLoop(p: Page, workflow: Workflow) {
|
||||||
|
const workflowCopy: Workflow = JSON.parse(JSON.stringify(workflow));
|
||||||
|
|
||||||
// apply ad-blocker to the current page
|
// apply ad-blocker to the current page
|
||||||
await this.applyAdBlocker(p);
|
await this.applyAdBlocker(p);
|
||||||
const usedActions: string[] = [];
|
const usedActions: string[] = [];
|
||||||
|
let selectors: string[] = [];
|
||||||
let lastAction = null;
|
let lastAction = null;
|
||||||
|
let actionId = -1
|
||||||
let repeatCount = 0;
|
let repeatCount = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -561,7 +640,7 @@ export default class Interpreter extends EventEmitter {
|
|||||||
* e.g. via `enqueueLinks`.
|
* e.g. via `enqueueLinks`.
|
||||||
*/
|
*/
|
||||||
p.on('popup', (popup) => {
|
p.on('popup', (popup) => {
|
||||||
this.concurrency.addJob(() => this.runLoop(popup, workflow));
|
this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
|
||||||
});
|
});
|
||||||
|
|
||||||
/* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
|
/* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
|
||||||
@@ -580,8 +659,11 @@ export default class Interpreter extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let pageState = {};
|
let pageState = {};
|
||||||
|
let getStateTest = "Hello";
|
||||||
try {
|
try {
|
||||||
pageState = await this.getState(p, workflow);
|
pageState = await this.getState(p, workflowCopy, selectors);
|
||||||
|
selectors = [];
|
||||||
|
console.log("Empty selectors:", selectors)
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
this.log('The browser has been closed.');
|
this.log('The browser has been closed.');
|
||||||
return;
|
return;
|
||||||
@@ -591,32 +673,52 @@ export default class Interpreter extends EventEmitter {
|
|||||||
this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
|
this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
|
||||||
}
|
}
|
||||||
|
|
||||||
const actionId = workflow.findIndex((step) => {
|
// const actionId = workflow.findIndex((step) => {
|
||||||
const isApplicable = this.applicable(step.where, pageState, usedActions);
|
// const isApplicable = this.applicable(step.where, pageState, usedActions);
|
||||||
console.log(`Where:`, step.where);
|
// console.log("-------------------------------------------------------------");
|
||||||
console.log(`Page state:`, pageState);
|
// console.log(`Where:`, step.where);
|
||||||
console.log(`Match result: ${isApplicable}`);
|
// console.log(`Page state:`, pageState);
|
||||||
return isApplicable;
|
// console.log(`Match result: ${isApplicable}`);
|
||||||
});
|
// console.log("-------------------------------------------------------------");
|
||||||
|
// return isApplicable;
|
||||||
|
// });
|
||||||
|
|
||||||
const action = workflow[actionId];
|
actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
|
||||||
|
|
||||||
|
const action = workflowCopy[actionId];
|
||||||
|
|
||||||
|
console.log("MATCHED ACTION:", action);
|
||||||
|
console.log("MATCHED ACTION ID:", actionId);
|
||||||
this.log(`Matched ${JSON.stringify(action?.where)}`, Level.LOG);
|
this.log(`Matched ${JSON.stringify(action?.where)}`, Level.LOG);
|
||||||
|
|
||||||
if (action) { // action is matched
|
if (action) { // action is matched
|
||||||
if (this.options.debugChannel?.activeId) {
|
if (this.options.debugChannel?.activeId) {
|
||||||
this.options.debugChannel.activeId(actionId);
|
this.options.debugChannel.activeId(actionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
repeatCount = action === lastAction ? repeatCount + 1 : 0;
|
repeatCount = action === lastAction ? repeatCount + 1 : 0;
|
||||||
if (this.options.maxRepeats && repeatCount >= this.options.maxRepeats) {
|
|
||||||
|
console.log("REPEAT COUNT", repeatCount);
|
||||||
|
if (this.options.maxRepeats && repeatCount > this.options.maxRepeats) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
lastAction = action;
|
lastAction = action;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
console.log("Carrying out:", action.what);
|
||||||
await this.carryOutSteps(p, action.what);
|
await this.carryOutSteps(p, action.what);
|
||||||
usedActions.push(action.id ?? 'undefined');
|
usedActions.push(action.id ?? 'undefined');
|
||||||
|
|
||||||
|
workflowCopy.splice(actionId, 1);
|
||||||
|
console.log(`Action with ID ${action.id} removed from the workflow copy.`);
|
||||||
|
|
||||||
|
// const newSelectors = this.getPreviousSelectors(workflow, actionId);
|
||||||
|
const newSelectors = this.getSelectors(workflowCopy);
|
||||||
|
newSelectors.forEach(selector => {
|
||||||
|
if (!selectors.includes(selector)) {
|
||||||
|
selectors.push(selector);
|
||||||
|
}
|
||||||
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.log(<Error>e, Level.ERROR);
|
this.log(<Error>e, Level.ERROR);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ import { io, Socket } from "socket.io-client";
|
|||||||
import { BinaryOutputService } from "../storage/mino";
|
import { BinaryOutputService } from "../storage/mino";
|
||||||
import { AuthenticatedRequest } from "../routes/record"
|
import { AuthenticatedRequest } from "../routes/record"
|
||||||
import {capture} from "../utils/analytics";
|
import {capture} from "../utils/analytics";
|
||||||
|
import { Page } from "playwright";
|
||||||
|
import { WorkflowFile } from "maxun-core";
|
||||||
chromium.use(stealthPlugin());
|
chromium.use(stealthPlugin());
|
||||||
|
|
||||||
const formatRecording = (recordingData: any) => {
|
const formatRecording = (recordingData: any) => {
|
||||||
@@ -533,6 +535,17 @@ function resetRecordingState(browserId: string, id: string) {
|
|||||||
id = '';
|
id = '';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function AddGeneratedFlags(workflow: WorkflowFile) {
|
||||||
|
const copy = JSON.parse(JSON.stringify(workflow));
|
||||||
|
for (let i = 0; i < workflow.workflow.length; i++) {
|
||||||
|
copy.workflow[i].what.unshift({
|
||||||
|
action: 'flag',
|
||||||
|
args: ['generated'],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return copy;
|
||||||
|
};
|
||||||
|
|
||||||
async function executeRun(id: string) {
|
async function executeRun(id: string) {
|
||||||
try {
|
try {
|
||||||
const run = await Run.findOne({ where: { runId: id } });
|
const run = await Run.findOne({ where: { runId: id } });
|
||||||
@@ -560,13 +573,14 @@ async function executeRun(id: string) {
|
|||||||
throw new Error('Could not access browser');
|
throw new Error('Could not access browser');
|
||||||
}
|
}
|
||||||
|
|
||||||
const currentPage = await browser.getCurrentPage();
|
let currentPage = await browser.getCurrentPage();
|
||||||
if (!currentPage) {
|
if (!currentPage) {
|
||||||
throw new Error('Could not create a new page');
|
throw new Error('Could not create a new page');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const workflow = AddGeneratedFlags(recording.recording);
|
||||||
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
||||||
recording.recording, currentPage, plainRun.interpreterSettings
|
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings
|
||||||
);
|
);
|
||||||
|
|
||||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ import { AuthenticatedRequest } from './record';
|
|||||||
import { computeNextRun } from '../utils/schedule';
|
import { computeNextRun } from '../utils/schedule';
|
||||||
import { capture } from "../utils/analytics";
|
import { capture } from "../utils/analytics";
|
||||||
import { tryCatch } from 'bullmq';
|
import { tryCatch } from 'bullmq';
|
||||||
|
import { WorkflowFile } from 'maxun-core';
|
||||||
|
import { Page } from 'playwright';
|
||||||
chromium.use(stealthPlugin());
|
chromium.use(stealthPlugin());
|
||||||
|
|
||||||
export const router = Router();
|
export const router = Router();
|
||||||
@@ -422,6 +424,17 @@ router.get('/runs/run/:id', requireSignIn, async (req, res) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function AddGeneratedFlags(workflow: WorkflowFile) {
|
||||||
|
const copy = JSON.parse(JSON.stringify(workflow));
|
||||||
|
for (let i = 0; i < workflow.workflow.length; i++) {
|
||||||
|
copy.workflow[i].what.unshift({
|
||||||
|
action: 'flag',
|
||||||
|
args: ['generated'],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return copy;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PUT endpoint for finishing a run and saving it to the storage.
|
* PUT endpoint for finishing a run and saving it to the storage.
|
||||||
*/
|
*/
|
||||||
@@ -443,10 +456,11 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re
|
|||||||
|
|
||||||
// interpret the run in active browser
|
// interpret the run in active browser
|
||||||
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
|
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
|
||||||
const currentPage = browser?.getCurrentPage();
|
let currentPage = browser?.getCurrentPage();
|
||||||
if (browser && currentPage) {
|
if (browser && currentPage) {
|
||||||
|
const workflow = AddGeneratedFlags(recording.recording);
|
||||||
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
||||||
recording.recording, currentPage, plainRun.interpreterSettings);
|
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings);
|
||||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||||
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
||||||
await destroyRemoteBrowser(plainRun.browserId);
|
await destroyRemoteBrowser(plainRun.browserId);
|
||||||
|
|||||||
@@ -244,7 +244,12 @@ export class WorkflowInterpreter {
|
|||||||
* @param page The page instance used to interact with the browser.
|
* @param page The page instance used to interact with the browser.
|
||||||
* @param settings The settings to use for the interpretation.
|
* @param settings The settings to use for the interpretation.
|
||||||
*/
|
*/
|
||||||
public InterpretRecording = async (workflow: WorkflowFile, page: Page, settings: InterpreterSettings) => {
|
public InterpretRecording = async (
|
||||||
|
workflow: WorkflowFile,
|
||||||
|
page: Page,
|
||||||
|
updatePageOnPause: (page: Page) => void,
|
||||||
|
settings: InterpreterSettings
|
||||||
|
) => {
|
||||||
const params = settings.params ? settings.params : null;
|
const params = settings.params ? settings.params : null;
|
||||||
delete settings.params;
|
delete settings.params;
|
||||||
|
|
||||||
@@ -262,7 +267,7 @@ export class WorkflowInterpreter {
|
|||||||
this.socket.emit('debugMessage', msg)
|
this.socket.emit('debugMessage', msg)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
serializableCallback: (data: string) => {
|
serializableCallback: (data: any) => {
|
||||||
this.serializableData.push(data);
|
this.serializableData.push(data);
|
||||||
this.socket.emit('serializableCallback', data);
|
this.socket.emit('serializableCallback', data);
|
||||||
},
|
},
|
||||||
@@ -275,6 +280,23 @@ export class WorkflowInterpreter {
|
|||||||
const interpreter = new Interpreter(decryptedWorkflow, options);
|
const interpreter = new Interpreter(decryptedWorkflow, options);
|
||||||
this.interpreter = interpreter;
|
this.interpreter = interpreter;
|
||||||
|
|
||||||
|
interpreter.on('flag', async (page, resume) => {
|
||||||
|
if (this.activeId !== null && this.breakpoints[this.activeId]) {
|
||||||
|
logger.log('debug', `breakpoint hit id: ${this.activeId}`);
|
||||||
|
this.socket.emit('breakpointHit');
|
||||||
|
this.interpretationIsPaused = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.interpretationIsPaused) {
|
||||||
|
this.interpretationResume = resume;
|
||||||
|
logger.log('debug', `Paused inside of flag: ${page.url()}`);
|
||||||
|
updatePageOnPause(page);
|
||||||
|
this.socket.emit('log', '----- The interpretation has been paused -----', false);
|
||||||
|
} else {
|
||||||
|
resume();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
const status = await interpreter.run(page, params);
|
const status = await interpreter.run(page, params);
|
||||||
|
|
||||||
const lastArray = this.serializableData.length > 1
|
const lastArray = this.serializableData.length > 1
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ import Run from "../../models/Run";
|
|||||||
import { getDecryptedProxyConfig } from "../../routes/proxy";
|
import { getDecryptedProxyConfig } from "../../routes/proxy";
|
||||||
import { BinaryOutputService } from "../../storage/mino";
|
import { BinaryOutputService } from "../../storage/mino";
|
||||||
import { capture } from "../../utils/analytics";
|
import { capture } from "../../utils/analytics";
|
||||||
|
import { WorkflowFile } from "maxun-core";
|
||||||
|
import { Page } from "playwright";
|
||||||
chromium.use(stealthPlugin());
|
chromium.use(stealthPlugin());
|
||||||
|
|
||||||
async function createWorkflowAndStoreMetadata(id: string, userId: string) {
|
async function createWorkflowAndStoreMetadata(id: string, userId: string) {
|
||||||
@@ -79,6 +81,17 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function AddGeneratedFlags(workflow: WorkflowFile) {
|
||||||
|
const copy = JSON.parse(JSON.stringify(workflow));
|
||||||
|
for (let i = 0; i < workflow.workflow.length; i++) {
|
||||||
|
copy.workflow[i].what.unshift({
|
||||||
|
action: 'flag',
|
||||||
|
args: ['generated'],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return copy;
|
||||||
|
};
|
||||||
|
|
||||||
async function executeRun(id: string) {
|
async function executeRun(id: string) {
|
||||||
try {
|
try {
|
||||||
const run = await Run.findOne({ where: { runId: id } });
|
const run = await Run.findOne({ where: { runId: id } });
|
||||||
@@ -106,13 +119,15 @@ async function executeRun(id: string) {
|
|||||||
throw new Error('Could not access browser');
|
throw new Error('Could not access browser');
|
||||||
}
|
}
|
||||||
|
|
||||||
const currentPage = await browser.getCurrentPage();
|
let currentPage = await browser.getCurrentPage();
|
||||||
if (!currentPage) {
|
if (!currentPage) {
|
||||||
throw new Error('Could not create a new page');
|
throw new Error('Could not create a new page');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const workflow = AddGeneratedFlags(recording.recording);
|
||||||
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
const interpretationInfo = await browser.interpreter.InterpretRecording(
|
||||||
recording.recording, currentPage, plainRun.interpreterSettings);
|
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings
|
||||||
|
);
|
||||||
|
|
||||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||||
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
||||||
|
|||||||
@@ -15,11 +15,13 @@ import { useGlobalInfoStore } from "../../context/globalInfo";
|
|||||||
import { getStoredRecording } from "../../api/storage";
|
import { getStoredRecording } from "../../api/storage";
|
||||||
import { apiUrl } from "../../apiConfig.js";
|
import { apiUrl } from "../../apiConfig.js";
|
||||||
import Cookies from 'js-cookie';
|
import Cookies from 'js-cookie';
|
||||||
|
|
||||||
interface IntegrationProps {
|
interface IntegrationProps {
|
||||||
isOpen: boolean;
|
isOpen: boolean;
|
||||||
handleStart: (data: IntegrationSettings) => void;
|
handleStart: (data: IntegrationSettings) => void;
|
||||||
handleClose: () => void;
|
handleClose: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IntegrationSettings {
|
export interface IntegrationSettings {
|
||||||
spreadsheetId: string;
|
spreadsheetId: string;
|
||||||
spreadsheetName: string;
|
spreadsheetName: string;
|
||||||
@@ -75,8 +77,7 @@ export const IntegrationSettingsModal = ({
|
|||||||
);
|
);
|
||||||
notify(
|
notify(
|
||||||
"error",
|
"error",
|
||||||
`Error fetching spreadsheet files: ${
|
`Error fetching spreadsheet files: ${error.response?.data?.message || error.message
|
||||||
error.response?.data?.message || error.message
|
|
||||||
}`
|
}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,17 @@
|
|||||||
import React, { useState, useContext } from 'react';
|
import React, { useState, useContext, useEffect } from 'react';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
import styled from "styled-components";
|
import styled from "styled-components";
|
||||||
import { stopRecording } from "../../api/recording";
|
import { stopRecording } from "../../api/recording";
|
||||||
import { useGlobalInfoStore } from "../../context/globalInfo";
|
import { useGlobalInfoStore } from "../../context/globalInfo";
|
||||||
import { IconButton, Menu, MenuItem, Typography, Avatar, Chip, } from "@mui/material";
|
import { IconButton, Menu, MenuItem, Typography, Chip, Button, Modal, Tabs, Tab, Box, Snackbar } from "@mui/material";
|
||||||
import { AccountCircle, Logout, Clear } from "@mui/icons-material";
|
import { AccountCircle, Logout, Clear, YouTube, X, Update, Close } from "@mui/icons-material";
|
||||||
import { useNavigate } from 'react-router-dom';
|
import { useNavigate } from 'react-router-dom';
|
||||||
import { AuthContext } from '../../context/auth';
|
import { AuthContext } from '../../context/auth';
|
||||||
import { SaveRecording } from '../molecules/SaveRecording';
|
import { SaveRecording } from '../molecules/SaveRecording';
|
||||||
import DiscordIcon from '../atoms/DiscordIcon';
|
import DiscordIcon from '../atoms/DiscordIcon';
|
||||||
import { apiUrl } from '../../apiConfig';
|
import { apiUrl } from '../../apiConfig';
|
||||||
import MaxunLogo from "../../assets/maxunlogo.png";
|
import MaxunLogo from "../../assets/maxunlogo.png";
|
||||||
|
import packageJson from "../../../package.json"
|
||||||
|
|
||||||
interface NavBarProps {
|
interface NavBarProps {
|
||||||
recordingName: string;
|
recordingName: string;
|
||||||
@@ -24,6 +25,38 @@ export const NavBar: React.FC<NavBarProps> = ({ recordingName, isRecording }) =>
|
|||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
|
|
||||||
const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null);
|
const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null);
|
||||||
|
const currentVersion = packageJson.version;
|
||||||
|
|
||||||
|
const [open, setOpen] = useState(false);
|
||||||
|
const [latestVersion, setLatestVersion] = useState<string | null>(null);
|
||||||
|
const [tab, setTab] = useState(0);
|
||||||
|
const [isUpdateAvailable, setIsUpdateAvailable] = useState(false);
|
||||||
|
|
||||||
|
const fetchLatestVersion = async (): Promise<string | null> => {
|
||||||
|
try {
|
||||||
|
const response = await fetch("https://api.github.com/repos/getmaxun/maxun/releases/latest");
|
||||||
|
const data = await response.json();
|
||||||
|
const version = data.tag_name.replace(/^v/, ""); // Remove 'v' prefix
|
||||||
|
return version;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to fetch latest version:", error);
|
||||||
|
return null; // Handle errors gracefully
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleUpdateOpen = () => {
|
||||||
|
setOpen(true);
|
||||||
|
fetchLatestVersion();
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleUpdateClose = () => {
|
||||||
|
setOpen(false);
|
||||||
|
setTab(0); // Reset tab to the first tab
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleUpdateTabChange = (event: React.SyntheticEvent, newValue: number) => {
|
||||||
|
setTab(newValue);
|
||||||
|
};
|
||||||
|
|
||||||
const handleMenuOpen = (event: React.MouseEvent<HTMLElement>) => {
|
const handleMenuOpen = (event: React.MouseEvent<HTMLElement>) => {
|
||||||
setAnchorEl(event.currentTarget);
|
setAnchorEl(event.currentTarget);
|
||||||
@@ -50,85 +83,233 @@ export const NavBar: React.FC<NavBarProps> = ({ recordingName, isRecording }) =>
|
|||||||
navigate('/');
|
navigate('/');
|
||||||
};
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const checkForUpdates = async () => {
|
||||||
|
const latestVersion = await fetchLatestVersion();
|
||||||
|
setLatestVersion(latestVersion); // Set the latest version state
|
||||||
|
if (latestVersion && latestVersion !== currentVersion) {
|
||||||
|
setIsUpdateAvailable(true); // Show a notification or highlight the "Upgrade" button
|
||||||
|
}
|
||||||
|
};
|
||||||
|
checkForUpdates();
|
||||||
|
}, []);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<NavBarWrapper>
|
<>
|
||||||
<div style={{
|
{isUpdateAvailable && (
|
||||||
display: 'flex',
|
<Snackbar
|
||||||
justifyContent: 'flex-start',
|
open={isUpdateAvailable}
|
||||||
}}>
|
onClose={() => setIsUpdateAvailable(false)}
|
||||||
<img src={MaxunLogo} width={45} height={40} style={{ borderRadius: '5px', margin: '5px 0px 5px 15px' }} />
|
message={
|
||||||
<div style={{ padding: '11px' }}><ProjectName>Maxun</ProjectName></div>
|
`New version ${latestVersion} available! Click "Upgrade" to update.`
|
||||||
</div>
|
}
|
||||||
{
|
action={
|
||||||
user ? (
|
<>
|
||||||
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'flex-end' }}>
|
<Button
|
||||||
{!isRecording ? (
|
color="primary"
|
||||||
<>
|
size="small"
|
||||||
<IconButton
|
onClick={handleUpdateOpen}
|
||||||
component="a"
|
style={{
|
||||||
href="https://discord.gg/5GbPjBUkws"
|
backgroundColor: '#ff00c3',
|
||||||
target="_blank"
|
color: 'white',
|
||||||
rel="noopener noreferrer"
|
fontWeight: 'bold',
|
||||||
sx={{
|
textTransform: 'none',
|
||||||
|
marginRight: '8px',
|
||||||
|
borderRadius: '5px',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Upgrade
|
||||||
|
</Button>
|
||||||
|
<IconButton
|
||||||
|
size="small"
|
||||||
|
aria-label="close"
|
||||||
|
color="inherit"
|
||||||
|
onClick={() => setIsUpdateAvailable(false)}
|
||||||
|
style={{ color: 'black' }}
|
||||||
|
>
|
||||||
|
<Close />
|
||||||
|
</IconButton>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
ContentProps={{
|
||||||
|
sx: {
|
||||||
|
background: "white",
|
||||||
|
color: "black",
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
|
||||||
|
)}
|
||||||
|
<NavBarWrapper>
|
||||||
|
<div style={{
|
||||||
|
display: 'flex',
|
||||||
|
justifyContent: 'flex-start',
|
||||||
|
}}>
|
||||||
|
<img src={MaxunLogo} width={45} height={40} style={{ borderRadius: '5px', margin: '5px 0px 5px 15px' }} />
|
||||||
|
<div style={{ padding: '11px' }}><ProjectName>Maxun</ProjectName></div>
|
||||||
|
<Chip
|
||||||
|
label={`${currentVersion}`}
|
||||||
|
color="primary"
|
||||||
|
variant="outlined"
|
||||||
|
sx={{ marginTop: '10px' }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{
|
||||||
|
user ? (
|
||||||
|
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'flex-end' }}>
|
||||||
|
{!isRecording ? (
|
||||||
|
<>
|
||||||
|
<Button variant="outlined" onClick={handleUpdateOpen} sx={{
|
||||||
|
marginRight: '40px',
|
||||||
|
color: "#00000099",
|
||||||
|
border: "#00000099 1px solid",
|
||||||
|
'&:hover': { color: '#ff00c3', border: '#ff00c3 1px solid' }
|
||||||
|
}}>
|
||||||
|
<Update sx={{ marginRight: '5px' }} /> Upgrade Maxun
|
||||||
|
</Button>
|
||||||
|
<Modal open={open} onClose={handleUpdateClose}>
|
||||||
|
<Box
|
||||||
|
sx={{
|
||||||
|
position: "absolute",
|
||||||
|
top: "50%",
|
||||||
|
left: "50%",
|
||||||
|
transform: "translate(-50%, -50%)",
|
||||||
|
width: 500,
|
||||||
|
bgcolor: "background.paper",
|
||||||
|
boxShadow: 24,
|
||||||
|
p: 4,
|
||||||
|
borderRadius: 2,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{latestVersion === null ? (
|
||||||
|
<Typography>Checking for updates...</Typography>
|
||||||
|
) : currentVersion === latestVersion ? (
|
||||||
|
<Typography variant="h6" textAlign="center">
|
||||||
|
🎉 You're up to date!
|
||||||
|
</Typography>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Typography variant="body1" textAlign="left" sx={{ marginLeft: '30px' }}>
|
||||||
|
A new version is available: {latestVersion}. Upgrade to the latest version for bug fixes, enhancements and new features!
|
||||||
|
<br />
|
||||||
|
View all the new updates
|
||||||
|
<a href="https://github.com/getmaxun/maxun/releases/" target="_blank" style={{ textDecoration: 'none' }}>{' '}here.</a>
|
||||||
|
</Typography>
|
||||||
|
<Tabs
|
||||||
|
value={tab}
|
||||||
|
onChange={handleUpdateTabChange}
|
||||||
|
sx={{ marginTop: 2, marginBottom: 2 }}
|
||||||
|
centered
|
||||||
|
>
|
||||||
|
<Tab label="Manual Setup Upgrade" />
|
||||||
|
<Tab label="Docker Compose Setup Upgrade" />
|
||||||
|
</Tabs>
|
||||||
|
{tab === 0 && (
|
||||||
|
<Box sx={{ marginLeft: '30px', background: '#cfd0d1', padding: 1, borderRadius: 3 }}>
|
||||||
|
<code style={{ color: 'black' }}>
|
||||||
|
<p>Run the commands below</p>
|
||||||
|
# pull latest changes
|
||||||
|
<br />
|
||||||
|
git pull origin master
|
||||||
|
<br />
|
||||||
|
<br />
|
||||||
|
# install dependencies
|
||||||
|
<br />
|
||||||
|
npm install
|
||||||
|
<br />
|
||||||
|
<br />
|
||||||
|
# start maxun
|
||||||
|
<br />
|
||||||
|
npm run start
|
||||||
|
</code>
|
||||||
|
</Box>
|
||||||
|
)}
|
||||||
|
{tab === 1 && (
|
||||||
|
<Box sx={{ marginLeft: '30px', background: '#cfd0d1', padding: 1, borderRadius: 3 }}>
|
||||||
|
<code style={{ color: 'black' }}>
|
||||||
|
<p>Run the commands below</p>
|
||||||
|
# pull latest docker images
|
||||||
|
<br />
|
||||||
|
docker-compose pull
|
||||||
|
<br />
|
||||||
|
<br />
|
||||||
|
# start maxun
|
||||||
|
<br />
|
||||||
|
docker-compose up -d
|
||||||
|
</code>
|
||||||
|
</Box>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
</Modal>
|
||||||
|
<iframe src="https://ghbtns.com/github-btn.html?user=getmaxun&repo=maxun&type=star&count=true&size=large" frameBorder="0" scrolling="0" width="170" height="30" title="GitHub"></iframe>
|
||||||
|
<IconButton onClick={handleMenuOpen} sx={{
|
||||||
display: 'flex',
|
display: 'flex',
|
||||||
alignItems: 'center',
|
alignItems: 'center',
|
||||||
borderRadius: '5px',
|
borderRadius: '5px',
|
||||||
padding: '8px',
|
padding: '8px',
|
||||||
marginRight: '30px',
|
marginRight: '10px',
|
||||||
}}
|
'&:hover': { backgroundColor: 'white', color: '#ff00c3' }
|
||||||
>
|
}}>
|
||||||
<DiscordIcon sx={{ marginRight: '5px' }} />
|
<AccountCircle sx={{ marginRight: '5px' }} />
|
||||||
</IconButton>
|
<Typography variant="body1">{user.email}</Typography>
|
||||||
<iframe src="https://ghbtns.com/github-btn.html?user=getmaxun&repo=maxun&type=star&count=true&size=large" frameBorder="0" scrolling="0" width="170" height="30" title="GitHub"></iframe>
|
</IconButton>
|
||||||
<IconButton onClick={handleMenuOpen} sx={{
|
<Menu
|
||||||
display: 'flex',
|
anchorEl={anchorEl}
|
||||||
alignItems: 'center',
|
open={Boolean(anchorEl)}
|
||||||
borderRadius: '5px',
|
onClose={handleMenuClose}
|
||||||
padding: '8px',
|
anchorOrigin={{
|
||||||
marginRight: '10px',
|
vertical: 'bottom',
|
||||||
'&:hover': { backgroundColor: 'white', color: '#ff00c3' }
|
horizontal: 'right',
|
||||||
}}>
|
}}
|
||||||
<AccountCircle sx={{ marginRight: '5px' }} />
|
transformOrigin={{
|
||||||
<Typography variant="body1">{user.email}</Typography>
|
vertical: 'top',
|
||||||
</IconButton>
|
horizontal: 'right',
|
||||||
<Menu
|
}}
|
||||||
anchorEl={anchorEl}
|
PaperProps={{ sx: { width: '180px' } }}
|
||||||
open={Boolean(anchorEl)}
|
>
|
||||||
onClose={handleMenuClose}
|
<MenuItem onClick={() => { handleMenuClose(); logout(); }}>
|
||||||
anchorOrigin={{
|
<Logout sx={{ marginRight: '5px' }} /> Logout
|
||||||
vertical: 'bottom',
|
</MenuItem>
|
||||||
horizontal: 'right',
|
<MenuItem onClick={() => {
|
||||||
}}
|
window.open('https://discord.gg/5GbPjBUkws', '_blank');
|
||||||
transformOrigin={{
|
}}>
|
||||||
vertical: 'top',
|
<DiscordIcon sx={{ marginRight: '5px' }} /> Discord
|
||||||
horizontal: 'right',
|
</MenuItem>
|
||||||
}}
|
<MenuItem onClick={() => {
|
||||||
>
|
window.open('https://www.youtube.com/@MaxunOSS/videos?ref=app', '_blank');
|
||||||
<MenuItem onClick={() => { handleMenuClose(); logout(); }}>
|
}}>
|
||||||
<Logout sx={{ marginRight: '5px' }} /> Logout
|
<YouTube sx={{ marginRight: '5px' }} /> YouTube
|
||||||
</MenuItem>
|
</MenuItem>
|
||||||
</Menu>
|
<MenuItem onClick={() => {
|
||||||
</>
|
window.open('https://x.com/maxun_io?ref=app', '_blank');
|
||||||
) : (
|
}}>
|
||||||
<>
|
<X sx={{ marginRight: '5px' }} /> Twiiter (X)
|
||||||
<IconButton onClick={goToMainMenu} sx={{
|
</MenuItem>
|
||||||
borderRadius: '5px',
|
</Menu>
|
||||||
padding: '8px',
|
</>
|
||||||
background: 'red',
|
) : (
|
||||||
color: 'white',
|
<>
|
||||||
marginRight: '10px',
|
<IconButton onClick={goToMainMenu} sx={{
|
||||||
'&:hover': { color: 'white', backgroundColor: 'red' }
|
borderRadius: '5px',
|
||||||
}}>
|
padding: '8px',
|
||||||
<Clear sx={{ marginRight: '5px' }} />
|
background: 'red',
|
||||||
Discard
|
color: 'white',
|
||||||
</IconButton>
|
marginRight: '10px',
|
||||||
<SaveRecording fileName={recordingName} />
|
'&:hover': { color: 'white', backgroundColor: 'red' }
|
||||||
</>
|
}}>
|
||||||
)}
|
<Clear sx={{ marginRight: '5px' }} />
|
||||||
</div>
|
Discard
|
||||||
) : ""
|
</IconButton>
|
||||||
}
|
<SaveRecording fileName={recordingName} />
|
||||||
</NavBarWrapper>
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
) : ""
|
||||||
|
}
|
||||||
|
</NavBarWrapper>
|
||||||
|
</>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import React, { FC, useState } from 'react';
|
import React, { FC, useState } from 'react';
|
||||||
import { Stack, Button, IconButton, Tooltip, Chip, Badge } from "@mui/material";
|
import { Stack, Button, IconButton, Tooltip, Badge } from "@mui/material";
|
||||||
import { AddPair, deletePair, UpdatePair } from "../../api/workflow";
|
import { AddPair, deletePair, UpdatePair } from "../../api/workflow";
|
||||||
import { WorkflowFile } from "maxun-core";
|
import { WorkflowFile } from "maxun-core";
|
||||||
import { ClearButton } from "../atoms/buttons/ClearButton";
|
import { ClearButton } from "../atoms/buttons/ClearButton";
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import React, { useState, useEffect } from 'react';
|
import React, { useState, useEffect } from 'react';
|
||||||
import { GenericModal } from "../atoms/GenericModal";
|
import { GenericModal } from "../atoms/GenericModal";
|
||||||
import { TextField, Typography, Box, Button, Chip } from "@mui/material";
|
import { TextField, Typography, Box, Button } from "@mui/material";
|
||||||
import { modalStyle } from "./AddWhereCondModal";
|
import { modalStyle } from "./AddWhereCondModal";
|
||||||
import { useGlobalInfoStore } from '../../context/globalInfo';
|
import { useGlobalInfoStore } from '../../context/globalInfo';
|
||||||
import { duplicateRecording, getStoredRecording } from '../../api/storage';
|
import { duplicateRecording, getStoredRecording } from '../../api/storage';
|
||||||
|
|||||||
Reference in New Issue
Block a user