19 lines
417 B
TypeScript
19 lines
417 B
TypeScript
|
import { WebScraperDataProvider } from "./scraper/WebScraper";
|
||
|
|
|
||
|
|
async function example() {
|
||
|
|
const example = new WebScraperDataProvider();
|
||
|
|
|
||
|
|
await example.setOptions({
|
||
|
|
mode: "crawl",
|
||
|
|
urls: ["https://mendable.ai"],
|
||
|
|
crawlerOptions: {},
|
||
|
|
});
|
||
|
|
const docs = await example.getDocuments(false);
|
||
|
|
docs.map((doc) => {
|
||
|
|
console.log(doc.metadata.sourceURL);
|
||
|
|
});
|
||
|
|
console.log(docs.length);
|
||
|
|
}
|
||
|
|
|
||
|
|
// example();
|