292 lines
8.0 KiB
JavaScript
292 lines
8.0 KiB
JavaScript
/**
|
|
* Example usage of Playwright with human-like behavior
|
|
* Demonstrates various scraping scenarios
|
|
*/
|
|
|
|
import { chromium } from 'playwright';
|
|
import {
|
|
getHumanizedContext,
|
|
humanClick,
|
|
humanType,
|
|
humanScroll,
|
|
simulateReading,
|
|
randomDelay,
|
|
randomMouseMovements
|
|
} from './human-behavior.js';
|
|
|
|
/**
|
|
* Example 1: Simple Google search with human behavior
|
|
*/
|
|
async function exampleGoogleSearch() {
|
|
console.log('\n=== Example 1: Google Search ===\n');
|
|
|
|
const browser = await chromium.launch({ headless: false, slowMo: 50 });
|
|
const context = await getHumanizedContext(browser);
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
// Navigate to Google
|
|
console.log('Navigating to Google...');
|
|
await page.goto('https://www.google.com');
|
|
await randomDelay(1000, 2000);
|
|
|
|
// Move mouse around naturally
|
|
await randomMouseMovements(page, 2);
|
|
|
|
// Search for something
|
|
console.log('Performing search...');
|
|
const searchBox = 'textarea[name="q"], input[name="q"]';
|
|
await humanClick(page, searchBox);
|
|
await humanType(page, searchBox, 'laptop repair toronto', {
|
|
minDelay: 70,
|
|
maxDelay: 180,
|
|
mistakes: 0.03
|
|
});
|
|
|
|
await randomDelay(500, 1200);
|
|
await page.keyboard.press('Enter');
|
|
|
|
// Wait for results
|
|
await page.waitForLoadState('networkidle');
|
|
await randomDelay(1500, 2500);
|
|
|
|
// Scroll through results
|
|
console.log('Scrolling through results...');
|
|
await humanScroll(page, {
|
|
scrollCount: 3,
|
|
minScroll: 150,
|
|
maxScroll: 400,
|
|
randomDirection: true
|
|
});
|
|
|
|
// Extract result count
|
|
const resultCount = await page.locator('div.g').count();
|
|
console.log(`✅ Found ${resultCount} search results\n`);
|
|
|
|
// Simulate reading
|
|
await simulateReading(page, 3000);
|
|
|
|
} finally {
|
|
await page.close();
|
|
await context.close();
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Example 2: Reddit scraping with natural behavior
|
|
*/
|
|
async function exampleRedditScraping() {
|
|
console.log('\n=== Example 2: Reddit Scraping ===\n');
|
|
|
|
const browser = await chromium.launch({ headless: false, slowMo: 50 });
|
|
const context = await getHumanizedContext(browser);
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
// Navigate to subreddit
|
|
console.log('Navigating to r/toronto...');
|
|
await page.goto('https://www.reddit.com/r/toronto');
|
|
await randomDelay(2000, 3000);
|
|
|
|
// Random mouse movements (looking around)
|
|
await randomMouseMovements(page, 3);
|
|
|
|
// Scroll naturally
|
|
console.log('Scrolling through posts...');
|
|
await humanScroll(page, {
|
|
scrollCount: 4,
|
|
minScroll: 200,
|
|
maxScroll: 500,
|
|
minDelay: 1000,
|
|
maxDelay: 2500
|
|
});
|
|
|
|
// Extract post titles
|
|
const posts = await page.evaluate(() => {
|
|
const postElements = document.querySelectorAll('[data-testid="post-container"]');
|
|
return Array.from(postElements).slice(0, 10).map(post => {
|
|
const titleEl = post.querySelector('h3');
|
|
return titleEl ? titleEl.innerText : null;
|
|
}).filter(Boolean);
|
|
});
|
|
|
|
console.log(`\n📝 Found ${posts.length} posts:`);
|
|
posts.forEach((title, i) => {
|
|
console.log(` ${i + 1}. ${title.substring(0, 60)}...`);
|
|
});
|
|
|
|
// Simulate reading
|
|
await simulateReading(page, 4000);
|
|
|
|
} finally {
|
|
await page.close();
|
|
await context.close();
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Example 3: Multi-step navigation with human behavior
|
|
*/
|
|
async function exampleMultiStepNavigation() {
|
|
console.log('\n=== Example 3: Multi-Step Navigation ===\n');
|
|
|
|
const browser = await chromium.launch({ headless: false, slowMo: 50 });
|
|
const context = await getHumanizedContext(browser);
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
// Step 1: Go to Hacker News
|
|
console.log('Step 1: Navigating to Hacker News...');
|
|
await page.goto('https://news.ycombinator.com');
|
|
await randomDelay(1500, 2500);
|
|
await randomMouseMovements(page, 2);
|
|
|
|
// Step 2: Scroll and read
|
|
console.log('Step 2: Scrolling and reading...');
|
|
await humanScroll(page, { scrollCount: 2 });
|
|
await simulateReading(page, 3000);
|
|
|
|
// Step 3: Click on first story
|
|
console.log('Step 3: Clicking on a story...');
|
|
const firstStory = '.titleline > a';
|
|
await page.waitForSelector(firstStory);
|
|
|
|
// Get the story title first
|
|
const storyTitle = await page.locator(firstStory).first().innerText();
|
|
console.log(` Clicking: "${storyTitle.substring(0, 50)}..."`);
|
|
|
|
await humanClick(page, firstStory);
|
|
await randomDelay(2000, 3000);
|
|
|
|
// Step 4: Interact with the new page
|
|
console.log('Step 4: Exploring the article...');
|
|
await humanScroll(page, {
|
|
scrollCount: 3,
|
|
minScroll: 200,
|
|
maxScroll: 600
|
|
});
|
|
|
|
await simulateReading(page, 4000);
|
|
|
|
console.log('✅ Multi-step navigation completed\n');
|
|
|
|
} finally {
|
|
await page.close();
|
|
await context.close();
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Example 4: Demonstrating different mouse movement patterns
|
|
*/
|
|
async function exampleMousePatterns() {
|
|
console.log('\n=== Example 4: Mouse Movement Patterns ===\n');
|
|
|
|
const browser = await chromium.launch({ headless: false, slowMo: 30 });
|
|
const context = await getHumanizedContext(browser);
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
await page.goto('https://www.example.com');
|
|
await randomDelay(1000, 1500);
|
|
|
|
console.log('Demonstrating various mouse patterns...');
|
|
|
|
// Pattern 1: Random movements
|
|
console.log(' 1. Random scanning...');
|
|
await randomMouseMovements(page, 5);
|
|
|
|
// Pattern 2: Slow deliberate movements
|
|
console.log(' 2. Deliberate movements...');
|
|
const viewport = page.viewportSize();
|
|
for (let i = 0; i < 3; i++) {
|
|
const target = {
|
|
x: Math.random() * viewport.width,
|
|
y: Math.random() * viewport.height
|
|
};
|
|
await page.mouse.move(target.x, target.y);
|
|
await randomDelay(800, 1500);
|
|
}
|
|
|
|
// Pattern 3: Hovering over elements
|
|
console.log(' 3. Hovering over link...');
|
|
const link = await page.locator('a').first();
|
|
const box = await link.boundingBox();
|
|
if (box) {
|
|
await page.mouse.move(
|
|
box.x + box.width / 2,
|
|
box.y + box.height / 2
|
|
);
|
|
await randomDelay(1000, 2000);
|
|
}
|
|
|
|
console.log('✅ Mouse patterns demonstration completed\n');
|
|
|
|
} finally {
|
|
await page.close();
|
|
await context.close();
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run all examples
|
|
*/
|
|
async function runAllExamples() {
|
|
console.log('\n' + '='.repeat(60));
|
|
console.log('PLAYWRIGHT HUMAN BEHAVIOR EXAMPLES');
|
|
console.log('='.repeat(60));
|
|
|
|
const examples = [
|
|
{ name: 'Google Search', fn: exampleGoogleSearch },
|
|
{ name: 'Reddit Scraping', fn: exampleRedditScraping },
|
|
{ name: 'Multi-Step Navigation', fn: exampleMultiStepNavigation },
|
|
{ name: 'Mouse Patterns', fn: exampleMousePatterns }
|
|
];
|
|
|
|
console.log('\nAvailable examples:');
|
|
examples.forEach((ex, i) => {
|
|
console.log(` ${i + 1}. ${ex.name}`);
|
|
});
|
|
|
|
const args = process.argv.slice(2);
|
|
|
|
if (args.length === 0) {
|
|
console.log('\nUsage: node scripts/example-usage.js [example-number]');
|
|
console.log('Example: node scripts/example-usage.js 1\n');
|
|
console.log('Running all examples...\n');
|
|
|
|
for (const example of examples) {
|
|
await example.fn();
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
}
|
|
} else {
|
|
const exampleNum = parseInt(args[0]) - 1;
|
|
if (exampleNum >= 0 && exampleNum < examples.length) {
|
|
await examples[exampleNum].fn();
|
|
} else {
|
|
console.log(`\n❌ Invalid example number. Choose 1-${examples.length}\n`);
|
|
}
|
|
}
|
|
|
|
console.log('\n' + '='.repeat(60));
|
|
console.log('ALL EXAMPLES COMPLETED');
|
|
console.log('='.repeat(60) + '\n');
|
|
}
|
|
|
|
// Run examples
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
runAllExamples().catch(console.error);
|
|
}
|
|
|
|
export {
|
|
exampleGoogleSearch,
|
|
exampleRedditScraping,
|
|
exampleMultiStepNavigation,
|
|
exampleMousePatterns
|
|
};
|
|
|