Heroku sudden spike in memory usage with Puppeteer

  heroku, javascript, mongodb, node.js, puppeteer

Had this commit yesterday where the process would go straight to Heroku’s memory limit giving me an R15 error – It worked really well in my testing and also on heroku until it gets to a random number of checked items, at which it throws the error. The interesting part is that right after the error, i get another R15 one, which says i am using only 22.2% of available memory.

Here is the code giving me the error:

let lenPlug = await client.db("PluginBoutique").collection("Kaye").countDocuments({});
let lenGuild = await client.db("Guilds").collection("Kaye").countDocuments({});
let lenSpit = await client.db("Spitfire").collection("Kaye").countDocuments({});
console.log("Plugins in database: " + lenPlug + " on Plugin Boutique and " + lenSpit + " on Spitfire" + "nGuilds in database: " + lenGuild + 'n');

async function sendNote(Embed, lenGuild) {
  for (let j = 0; j < lenGuild; j++) {
    let guildId = await client.db("Guilds").collection("Kaye").distinct("id", {
      _id: j
    });
    let channelId = await client.db("Guilds").collection("Kaye").distinct("channel", {
      _id: j
    });
    try {
      await bot.guilds.cache.get(guildId[0]).channels.cache.get(channelId[0]).send(Embed);
    } catch {
      console.log("Could not send message to " + channelId + " ");
    }
  }
  return;
}

function percentage(price, discount) {
  let price_int = parseInt(price.substring(1), 10);
  let discount_int = parseInt(discount.substring(1), 10);
  return 100 - parseInt(discount_int * 100 / price_int, 10);
}

function percentageColor(percentage) {
  if (percentage < 10)
    return '#36B3E2';
  if (percentage < 25)
    return '#36E26B';
  if (percentage < 50)
    return '#FAB633';
  return '#E23E36';
}

const browser = await puppeteer.launch(args);

const page = await browser.newPage();
await page.setRequestInterception(true);
await page.on('request', (req) => config(req));
await page.setDefaultNavigationTimeout(0);

let existsDiscountSpit = false;
for (let i = 25; i < lenSpit; i++) {
  let url = (await client.db("Spitfire").collection("Kaye").distinct("url", {
    _id: i
  })).toString();
  let hasDiscount = await client.db("Spitfire").collection("Kaye").distinct("hasDiscount", {
    _id: i
  });
  await page.goto(url, {
    waitUntil: 'networkidle2',
  }).then(async() => {
    try {
      const el4 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.one-third.lap--one-whole.palm--one-whole > p > span.view-product-current-price.view-product-discounted-price', {
        timeout: 5000
      });
      const txt4 = await el4.getProperty('textContent');
      const price = await txt4.jsonValue();
      const el5 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.one-third.lap--one-whole.palm--one-whole > p > span.view-product-was-price', {
        timeout: 5000
      });
      const txt5 = await el5.getProperty('textContent');
      const oldprice = await txt5.jsonValue();
      const el1 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.two-thirds.lap--one-whole.palm--one-whole > div > h1', {
        timeout: 5000
      });
      const txt1 = await el1.getProperty('textContent');
      const name = await txt1.jsonValue();
      const image = await page.$$eval('.image:nth-child(1) > picture img[src]', image => image.map(image => image.getAttribute('src')));
      if (hasDiscount == 0) {
        existsDiscountSpit = true;
        await client.db("Spitfire").collection("Kaye").replaceOne({
          _id: i
        }, {
          url: url,
          hasDiscount: 1
        });
        const percentage = percentage(oldprice, price)
        const color = percentageColor(percentage)
        const Embed = new discord.MessageEmbed()
          .setColor(color)
          .setTitle("<:kaye:828581356392808458> " + name)
          .setURL(url)
          .addField('u200B', name + " was **" + oldprice + "** and now is **" + price + "** with a **" + percentage + "%** discount on **[Spitfire](https://www.spitfireaudio.com/)**! " + messages[Math.floor(Math.random() * messages.length)])
          .setImage(image[0])
          .setFooter('kaye-app')
        sendNote(Embed, lenGuild);
        console.log("Discount found for url " + url + "!");
      } else {
        console.log("Already notified about plugin url " + url + "!")
      }
    } catch {
      console.log("No discount found for url " + url + "!");
      if (hasDiscount == 1)
        await client.db("Spitfire").collection("Kaye").replaceOne({
          _id: i
        }, {
          url: url,
          hasDiscount: 0
        });
    }
  });
}
if (!existsDiscountSpit) {
  const Embed = new discord.MessageEmbed()
    .setColor('#d6d3ce')
    .setTitle("<:kaye:828581356392808458> Didn't find discounts on Spitfire today!")
    .setURL("https://www.spitfireaudio.com/")
    .setFooter('kaye-app')
  sendNote(Embed, lenGuild);
}

Here is a snippet of the Heroku logs:

2021-04-19T07:54:58.701763+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-brass-professional/!
2021-04-19T07:55:03.337663+00:00 app[worker.1]: The script uses approximately 21.2 MB
2021-04-19T07:55:03.337669+00:00 app[worker.1]:
2021-04-19T07:55:07.262079+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-orchestra/!
2021-04-19T07:55:08.337457+00:00 app[worker.1]: The script uses approximately 21.35 MB
2021-04-19T07:55:08.337464+00:00 app[worker.1]:
2021-04-19T07:55:13.339770+00:00 app[worker.1]: The script uses approximately 21.4 MB
2021-04-19T07:55:13.339772+00:00 app[worker.1]:
2021-04-19T07:55:14.889726+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-orchestra-professional/!
2021-04-19T07:55:16.146914+00:00 heroku[worker.1]: Process running mem=512M(100.0%)
2021-04-19T07:55:16.193131+00:00 heroku[worker.1]: Error R15 (Memory quota vastly exceeded)
2021-04-19T07:55:16.197917+00:00 heroku[worker.1]: Stopping process with SIGKILL
2021-04-19T07:55:16.212670+00:00 heroku[worker.1]: Process running mem=115M(22.2%)
2021-04-19T07:55:16.266903+00:00 heroku[worker.1]: Error R15 (Memory quota vastly exceeded)
2021-04-19T07:55:16.297708+00:00 heroku[worker.1]: Stopping process with SIGKILL
2021-04-19T07:55:16.308694+00:00 heroku[worker.1]: Process exited with status 137

NOTE: I also have a function that logs the memory usage

Thanks!

Source: Ask Javascript Questions

LEAVE A COMMENT