gpt4 book ai didi

javascript - 从 Node 中的另一个模块导入的异步数据无法解析,我该如何解决这个问题?

转载 作者:太空宇宙 更新时间:2023-11-04 01:58:04 24 4
gpt4 key购买 nike

这是收集和导出异步数据的模块:scraper.js

const express = require('express')
const cheerio = require('cheerio')
const request = require("tinyreq")
const fs = require('fs')
const _ = require('lodash')
const uuid = require('uuid/v4')
const async = require('async')


const mental_models = {
url: 'https://www.farnamstreetblog.com/mental-models/',
data: {}
}
const decision_making = {
url: 'https://www.farnamstreetblog.com/smart-decisions/',
data: {}
}
const cognitive_bias = {
url: 'https://betterhumans.coach.me/cognitive-bias-cheat-sheet-55a472476b18',
data: {}
}
const DATA_URLS = [mental_models, decision_making, cognitive_bias]

const filterScrape = async (source, params) => {

let filtered_data = {
topics: [],
content: [],
additional_content: []
}

let response = await scrape(source)

try {
let $ = cheerio.load(response)

params.forEach((elem) => {
let headers = ['h1', 'h2', 'h3']
if ($(elem) && headers.includes(elem)) {

let topic = {}
let content = {}
let id = uuid()
topic.id = id
topic.text = $(elem).text()

if ($(elem).closest('p')) {
content.text = $(elem).closest('p').text()
content.id = id
}
filtered_data.topics.push(topic)
filtered_data.content.push(content)
} else if ($(elem) && !headers.includes(elem)) {

let content = {}
let id = uuid()
content.text = $(elem).text()
content.id = id
filtered_data.additional_content.push(content)

} else {

}
})

}
catch (err) {
console.log(err)
}

return filtered_data

}

const scrape = (source) => {
return new Promise((resolve, reject) => {
request(source.url, function (err, body) {
if (err) {
reject(err)
return
}
resolve(body)
})
})
}

const DATA = _.map(DATA_URLS, async (source) => {

let params = ['h1', 'h2', 'h3', 'p']
let new_data = await filterScrape(source, params)

try {
source.data = new_data
}
catch (err) {
console.log(err)
}

})
module.exports = DATA

这是导入数据的模块:neural.js

const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const DATA = require('./scraper')

console.log(DATA)

显然没有发生太多事情,我已经删除了代码,因为变量无法解析。记录时,它会记录一个 promise ,但该 promise 不会解决。然而,在导入的模块中,promise 被记录然后解析。是什么赋予了?我应该导入一个解析数据的函数吗?

最佳答案

当然最好导入该函数,但是它不会改变代码中的问题:

const DATA = _.map(DATA_URLS, async (source) => {

Lodash 不支持异步迭代 - 所以你需要一些其他方法,一种是使用最新的 Nodejs 版本 (10.x) 并利用异步迭代 - 但这不会使用异步代码的全部功能。

您还可以使用scramjet - 我公司支持的框架。上面的代码将采用以下形式:

const {DataStream} = require("scramjet");
const DATA_URLS = [mental_models, decision_making, cognitive_bias];

module.exports = async () => DataStream.fromArray(DATA_URLS)
.setOptions({maxParallel: 2}) // if you need to limit that at all.
.map(async ({url}) => {
let params = ['h1', 'h2', 'h3', 'p']
let data = await filterScrape(source, params);

return { url, data };
})
.toArray();

另一个文件将采用以下形式:

const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const scraper = require('./scraper')

(async (){
const DATA = await scraper();
console.log(DATA); // or do whatever else you were expecting...
})();

关于javascript - 从 Node 中的另一个模块导入的异步数据无法解析,我该如何解决这个问题?,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/47115435/

24 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com