Foreclosure Leads for Free: How I Scrape County Auctions with Node.js

npm install puppeteer puppeteer-core @sparticuz/chromium

const puppeteer = require('puppeteer');
const chromium = require('@sparticuz/chromium');

const getRecords = async() => {
};
getRecords();

const browser = await puppeteer.launch({
  args: [
    ...chromium.args,
    '--disable-features=site-per-process',
    '--disable-dev-shm-usage'],
  defaultViewport: chromium.defaultViewport,
  executablePath: await chromium.executablePath(),
  headless: chromium.headless,
  ignoreHTTPSErrors: true,
  timeout: 120000,
});

try {
  // This is where you will place the rest of
  // your code.
} catch (error) {
  // Logs an error if anything goes wrong.
  console.log(error);
} finally {
  // Closes the browser and exits the process
  // after code finishes running.
  browser.close();
  process.exit();
};

// Open a new browser tab in Chromium.
const page = await browser.newPage();
// Go to Foreclosure's calendar and wait for
// content to load.
await page.goto('https://pinellas.realforeclose.com/index.cfm?zaction=USER&zmethod=CALENDAR', {
  waitUntil: 'domcontentloaded'
});

await page.waitForFunction(
  // Wait until this function returns TRUE.
  selector => !!document.querySelector(selector).innerText,
  // Options and target selector.
  {}, 'span.CALMSG > span.CALACT'
);
// Get all the boxes inside of the calendar
// that represent sale days.
const saleDayHandles = await page.$$('div.CALSELF, div.CALSELT');

// Create an empty Array we can push Auction
// Days to.
let nextAuctionDays = [];
// Loop through Auction Days and get days
// when there are Auctions available.
for (const auctionDay of auctionDays) {
  // Find the number of Auctions available.
  const availableAuctions = await page.evaluate(el => {
    // Check if the number is greater than 0 and
    // return it if it is.
    if (el.querySelector('span.CALACT').innerText > 0) {
      return el.querySelector('span.CALACT').innerText;
    };
    // Return null if number is lower or equal to 0
    return null;
  }, auctionDay);
  // If availableAuctions is not null, push the
  // Auction Day to nextAuctionDays.
  if (availableAuctions) {
    nextAuctionDays.push(auctionDay);
  };
};
// Click the first Auction Day that has
// Auctions available.
await nextAuctionDays[0].click();

// Create a variable containing TRUE    
let condition = true;
  
// Make an object you can use to store
// the Auctions you find in each page.
const object = [];
// Create a do... while loop that runs
// while condition is still true.
do {
  // This is where will you place the rest
  // of your code.
} while (condition)

// Wait until the header containing the
// Auction Date contains text.
await page.waitForFunction(
  selector => !!document.querySelector(selector).innerText,
  {}, 'div.BLHeaderDateDisplay'
);
// Create a variable containing the
// current Auction Date.
const currentAuction = await page.evaluate(selector => {
  return document.querySelector(selector).innerText;
}, 'div.BLHeaderDateDisplay'); 

// Hold off until the amount of pages
// is available.
await page.waitForFunction(
  selector => !!document.querySelector(selector).innerText,
  {}, 'span#maxWA'
);
// Collect the amount of pages.
const pages = await page.evaluate(selector => {
  return document.querySelector(selector).innerHTML;
}, 'span#maxWA');
// Repeat code until all pages have 
// been visited.
for (let i = 1; i <= pages; i++) {
  // This is where you will continue to
  // write code. I'll let you know when
  // to stop adding code to this loop.
};

// Find all the boxes containing Auction
// information inside.
const saleContainers = await page.$$('div.Head_W > div#Area_W > div.AUCTION_ITEM');
// Go through each box and call your
// extractDetails() function. Read
// below for more details.
for (const saleContainer of sale Containers) {
  object.push(
    await extractDetails(page, saleContainer);
  };
};

// Create an asynchronous function that
// receives a Puppeteer Page and a sale
// container.
const extractDetails = async (page, saleContainer) => {
  // Utilize Page.evaluate() to execute functions
  // within the browser context.
  const details = await page.evaluate(el => {
    // The content is inside of a Table,
    // extract the cells inside that Table.
    const details = el.querySelectorAll('div.AUCTION_DETAILS td');
    // Extract the relevant data from the
    // container. 
    return {
      sale_date: el.querySelector('div.AUCTION_STATS > div.ASTAT_MSGB').innerText,
      sale_case: details[1]?.innerText || '',
      judgement_amount: details[2]?.innerText || '',
      parcel_id: details[3]?.innerText || '',
      property_address: details[5] ? [ details[4].innerText, details[5].innerText.replace('-', '') ].join(', ') : '',
      assessed_value: details[6]?.innerText || '',
      plaintiff_max_bid: details[7]?.innerText || ''
    };
  }, saleContainer);
};

// Check if the loop hasn't reached
// the last page.
if (i < pages) {
  // Wait until the Next button is
  // available to click.
  await page.waitForSelector('div.Head_W > div.PageFrame > span.PageRight');
  
  // Click the Next button to go to the
  // following page.
  await page.click('div.Head_W > div.PageFrame > span.PageRight');
    
  // Hold on until the element that
  // contains the current page number
  // is not the same as the current iteration.
  await page.waitForFunction('document.getElementById("curPWA").value != '+'"i"');
};

process.stdout.write(`\rYou've succesfully scraped ${object.length} Auctions.\n`);