ウェブでのラッピング

const { sanitizeEntity } = require('strapi-utils');

module.exports = {
  /**
   * Retrieve a record.
   *
   * @return {Object}
   */

  async findOne(ctx) {
    const { slug } = ctx.params;

    const entity = await strapi.services.scraper.findOne({ slug });
    return sanitizeEntity(entity, { model: strapi.models.scraper });
  },
};

{
  "routes": [
    ...
    {
      "method": "GET",
      "path": "/scrapers/:slug",
      "handler": "scraper.findOne",
      "config": {
        "policies": []
      }
    },
    ... 
    ]
}

const main = async () => {
  // Fetch the correct scraper thanks to the slug
  const slug = "jamstack-org"
  const scraper = await strapi.query('scraper').findOne({
    slug: slug
  });

  console.log(scraper);

  // If the scraper doesn't exists, is disabled or doesn't have a frequency then we do nothing
  if (scraper == null || !scraper.enabled || !scraper.frequency)
    return
}

exports.main = main;

'use strict';

/**
 * An asynchronous bootstrap function that runs before
 * your application gets started.
 *
 * This gives you an opportunity to set up your data model,
 * run jobs, or perform some special logic.
 *
 * See more details here: https://strapi.io/documentation/developer-docs/latest/concepts/configurations.html#bootstrap
 */
const jamstack = require('../../scripts/scrapers/jamstack.js')

module.exports = () => {
  jamstack.main()
};

'use strict'

yarn add cron-parser chalk

'use strict'

const parser = require('cron-parser');

const scraperCanRun = async (scraper) => {
  const frequency = parser.parseExpression(scraper.frequency);
  const current_date = parseInt((new Date().getTime() / 1000));
  let next_execution_at = ""

  if (scraper.next_execution_at){
    next_execution_at = scraper.next_execution_at
  }
  else {
    next_execution_at = (frequency.next().getTime() / 1000);
    await strapi.query('scraper').update({
      id: scraper.id
    }, {
      next_execution_at: next_execution_at
    });
  }

  if (next_execution_at <= current_date){
    await strapi.query('scraper').update({
      id: scraper.id
    }, {
      next_execution_at: (frequency.next().getTime() / 1000)
    });
    return true
  }
  return false
}

module.exports = { scraperCanRun }

'use strict'

...

const getAllSG = async (scraper) => {
  const existingSG = await strapi.query('site-generator').find({
    _limit: 1000,
    scraper: scraper.id
  }, ["name"]);
  const allSG = existingSG.map(x => x.name);
  console.log(`Site generators in database: \t${chalk.blue(allSG.length)}`);

  return allSG;
}

module.exports = { getAllSG, scraperCanRun }

'use strict'

...

const getDate = async () => {
  const today = new Date();
  const date  = today.getFullYear()+'-'+(today.getMonth()+1)+'-'+today.getDate();
  const time  = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
  return date+' '+time;
}

module.exports = { getDate, getAllSG, scraperCanRun }

'use strict'

...

const getReport = async (newSG) => {
  return { newSG: newSG, date: await getDate()}
}

module.exports = { getReport, getDate, getAllSG, scraperCanRun }

yarn add puppeteer cheerio

'use strict'

const chalk = require('chalk');
const puppeteer = require('puppeteer');
const {
  getReport,
  getDate,
  getAllSG,
  scraperCanRun
} = require('./utils/utils.js')

let report = {}
let errors = []
let newSG = 0

const scrape = async () => {
  console.log("Scrape function");
}

const main = async () => {
  // Fetch the correct scraper thanks to the slug
  const slug = "jamstack-org"
  const scraper = await strapi.query('scraper').findOne({
    slug: slug
  });

  // If the scraper doesn't exists, is disabled or doesn't have a frequency then we do nothing
  // if (scraper == null || !scraper.enabled || !scraper.frequency){
  //   console.log(`${chalk.red("Exit")}: (Your scraper may does not exist, is not activated or does not have a frequency field filled in)`);
  //   return
  // }

  // const canRun = await scraperCanRun(scraper);
  // if (canRun && scraper.enabled){
    const allSG = await getAllSG(scraper)
    await scrape(allSG, scraper)
  // }
}

exports.main = main;

'use strict'

const chalk = require('chalk');
const cheerio = require('cheerio');
const puppeteer = require('puppeteer');
const {
  getReport,
  getDate,
  getAllSG,
  scraperCanRun
} = require('./utils/utils.js')
const {
  createSiteGenerators,
  updateScraper
} = require('./utils/query.js')

let report = {}
let errors = []
let newSG = 0

const scrape = async (allSG, scraper) => {
  const url = "https://jamstack.org/generators/"
  const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
  const page = await browser.newPage();

  try {
    await page.goto(url)
  } catch (e) {
    console.log(`${chalk.red("Error")}: (${url})`);
    errors.push({
      context: "Page navigation",
      url: url,
      date: await getDate()
    })
    return
  }

  const expression = "//div[@class='generator-card flex flex-col h-full']"
  const elements = await page.$x(expression);
  await page.waitForXPath(expression, { timeout: 3000 })

  const promise = new Promise((resolve, reject) => {
    elements.forEach(async (element) => {
      let card = await page.evaluate(el => el.innerHTML, element);
      let $ = cheerio.load(card)
      const name = $('.text-xl').text().trim() || null;
      // Skip this iteration if the sg is already in db
      if (allSG.includes(name))
        return;
      const stars = $('span:contains("stars")').parent().text().replace("stars", "").trim() || null;
      const forks = $('span:contains("forks")').parent().text().replace("forks", "").trim() || null;
      const issues = $('span:contains("issues")').parent().text().replace("issues", "").trim() || null;
      const description = $('.text-sm.mb-4').text().trim() || null;
      const language = $('dt:contains("Language:")').next().text().trim() || null;
      const template = $('dt:contains("Templates:")').next().text().trim() || null;
      const license = $('dt:contains("License:")').next().text().trim() || null;
      const deployLink = $('a:contains("Deploy")').attr('href') || null;

      await createSiteGenerators(
        name,
        stars,
        forks,
        issues,
        description,
        language,
        template,
        license,
        deployLink,
        scraper
      )
      newSG += 1;
    });
  });

  promise.then(async () => {
    await page.close()
    await browser.close();
  });
}

const main = async () => {
  // Fetch the correct scraper thanks to the slug
  const slug = "jamstack-org"
  const scraper = await strapi.query('scraper').findOne({
    slug: slug
  });

  // If the scraper doesn't exists, is disabled or doesn't have a frequency then we do nothing
  // if (scraper == null || !scraper.enabled || !scraper.frequency){
  //   console.log(`${chalk.red("Exit")}: (Your scraper may does not exist, is not activated or does not have a frequency field filled in)`);
  //   return
  // }

  // const canRun = await scraperCanRun(scraper);
  // if (canRun && scraper.enabled){
  const allSG = await getAllSG(scraper)
  await scrape(allSG, scraper)
  report = await getReport(newSG);
  // }
}

exports.main = main;

'use strict'

const chalk = require('chalk');

const createSiteGenerators = async (name, stars, forks, issues, description, language, template, license, deployLink, scraper) => {

  try {
    const entry = await strapi.query('site-generator').create({
      name: name,
      stars: stars,
      forks: forks,
      issues: issues,
      description: description,
      language: language,
      templates: template,
      license: license,
      deploy_to_netlify_link: deployLink,
      scraper: scraper.id
    })
  } catch (e) {
    console.log(e);
  }
}

const updateScraper = async (scraper, report, errors) => {
  await strapi.query('scraper').update({
    id: scraper.id
  }, {
    report: report,
    error: errors,
  });

  console.log(`Job done for: ${chalk.green(scraper.name)}`);
}

module.exports = {
  createSiteGenerators,
  updateScraper,
}

'use strict'

const chalk = require('chalk');
const cheerio = require('cheerio');
const puppeteer = require('puppeteer');
const {
  getReport,
  getDate,
  getAllSG,
  scraperCanRun
} = require('./utils/utils.js')
const {
  createSiteGenerators,
  updateScraper
} = require('./utils/query.js')

let report = {}
let errors = []
let newSG = 0

const scrape = async (allSG, scraper) => {
  const url = "https://jamstack.org/generators/"
  const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
  const page = await browser.newPage();

  try {
    await page.goto(url)
  } catch (e) {
    console.log(`${chalk.red("Error")}: (${url})`);
    errors.push({
      context: "Page navigation",
      url: url,
      date: await getDate()
    })
    return
  }

  const expression = "//div[@class='generator-card flex flex-col h-full']"
  const elements = await page.$x(expression);
  await page.waitForXPath(expression, { timeout: 3000 })

  const promise = new Promise((resolve, reject) => {
    elements.forEach(async (element) => {
      let card = await page.evaluate(el => el.innerHTML, element);
      let $ = cheerio.load(card)
      const name = $('.text-xl').text().trim() || null;
            // Skip this iteration if the sg is already in db
      if (allSG.includes(name))
        return;
      const stars = $('span:contains("stars")').parent().text().replace("stars", "").trim() || null;
      const forks = $('span:contains("forks")').parent().text().replace("forks", "").trim() || null;
      const issues = $('span:contains("issues")').parent().text().replace("issues", "").trim() || null;
      const description = $('.text-sm.mb-4').text().trim() || null;
      const language = $('dt:contains("Language:")').next().text().trim() || null;
      const template = $('dt:contains("Templates:")').next().text().trim() || null;
      const license = $('dt:contains("License:")').next().text().trim() || null;
      const deployLink = $('a:contains("Deploy")').attr('href') || null;

      await createSiteGenerators(
        name,
        stars,
        forks,
        issues,
        description,
        language,
        template,
        license,
        deployLink,
        scraper
      )
      newSG += 1;
    });
  });

  promise.then(async () => {
    await page.close()
    await browser.close();
  });
}

const main = async () => {
  // Fetch the correct scraper thanks to the slug
  const slug = "jamstack-org"
  const scraper = await strapi.query('scraper').findOne({
    slug: slug
  });

  // If the scraper doesn't exists, is disabled or doesn't have a frequency then we do nothing
  if (scraper == null || !scraper.enabled || !scraper.frequency){
    console.log(`${chalk.red("Exit")}: (Your scraper may does not exist, is not activated or does not have a frequency field filled in)`);
    return
  }

  const canRun = await scraperCanRun(scraper);
  if (canRun && scraper.enabled){
    const allSG = await getAllSG(scraper)
    await scrape(allSG, scraper)
    report = await getReport(newSG);
    await updateScraper(scraper, report, errors)
  }
}

exports.main = main;

'use strict'

/**
 * An asynchronous bootstrap function that runs before
 * your application gets started.
 *
 * This gives you an opportunity to set up your data model,
 * run jobs, or perform some special logic.
 *
 * See more details here: https://strapi.io/documentation/developer-docs/latest/concepts/configurations.html#bootstrap
 */

module.exports = () => {};