This commit is contained in:
sam 2024-03-10 11:07:24 +13:00
parent fcbe2db038
commit 5d6cb78cdf
2 changed files with 3 additions and 3 deletions

View file

@ -23,10 +23,10 @@ async function crawl(url, ignoreExisting = false) {
console.log(`[${res.status} ${res.statusText}] ${url}`);
if(res.status == 200) {
const html = await res.text();
const $ = cheerio.load(html);
const html = await res.text();
const $ = cheerio.load(html);
const title = $("title").text();
const text = $.text().replace(/\s+/g, " ").trim();
const text = $.text().replace(/\s+/g, " ").trim();
console.log(`[TITLE] ${title} (${url})`);

View file