javascript - NodeJS - 直接流式传输到 Azure 数据湖

Question

我得到了下面的示例，其中我使用流式传输下载 zip 文件。效果很好。

但我有一个挑战。我需要下载此文件并直接发送到 Azure，而不在本地保存此下载。可能吗？

看代码:

const { createWriteStream } = require("fs");
const stream = require("stream");
const { promisify } = require("util");
const pipeline = promisify(stream.pipeline);

const url = "http://....../file.zip";
const fileName = "filedownloaded.zip";

const downloadStream = got.stream(url);
const fileWriterStream = createWriteStream(fileName);

downloadStream.on("downloadProgress", ({ transferred, total, percent }) => {
  const percentage = Math.round(percent * 100);
  console.error(`progress: ${transferred}/${total} (${percentage}%)`);
});

(async () => {
    try {
      await pipeline(downloadStream, fileWriterStream)
      console.log(`File downloaded to ${fileName}`);
    } catch (error) {
      console.error(`Something went wrong. ${error.message}`);
    }
  })();

我应该使用缓冲区来做到这一点吗？我的意思是，我怎样才能将该文件发送到那里？有人做过类似的事情吗？

这是在 Azure Datalake 上创建容器、文件夹和文件的代码

const http = require('http');
var unzip = require('unzip');
const { DataLakeServiceClient, StorageSharedKeyCredential } = require("@azure/storage-file-datalake");

// Load the .env file if it exists
require("dotenv").config();

const sharedKeyCredential = 
     new StorageSharedKeyCredential(process.env.ACCOUNT_NAME, process.env.ACCOUNT_KEY);
const datalakeServiceClient = new DataLakeServiceClient(
      `https://${process.env.ACCOUNT_NAME}.dfs.core.windows.net`, sharedKeyCredential);

async function CreateFileSystem(fileSystemName) {
  const fileSystemClient = datalakeServiceClient.getFileSystemClient(fileSystemName);
  const createResponse = await fileSystemClient.create(); 
  return {response: createResponse, container: fileSystemClient} 
}

async function CreateDirectory(fileSystemClient, directoryName) {
  const directoryClient = fileSystemClient.getDirectoryClient(directoryName);
  const result = await directoryClient.create();
  return result
}

async function DeleteDirectory(fileSystemClient, directoryName) {
  const directoryClient = fileSystemClient.getDirectoryClient(directoryName); 
  const result = await directoryClient.delete();
  return result
}

async function UploadFile(fileSystemClient, from, fileName ) {
  const fs = require('fs') 
  var content = "";
  fs.readFile('mytestfile.txt', (err, data) => { 
      if (err) throw err;
      content = data.toString();
  }) 
  const fileClient = fileSystemClient.getFileClient("directoryexample2/uploaded-file.txt");
  await fileClient.create();
  await fileClient.append(content, 0, content.length);
  await fileClient.flush(content.length);

}

const main = async () => {
  const fs  =  await CreateFileSystem("filesystemexample2");
  const dir = await CreateDirectory(fs.container, "directoryexample2");
  await UploadFile(fs.container)
}

console.log("Starting ...")
main();

Answer 1

因为看起来您只是在读取文件缓冲区的字符串表示形式并将其提供给您的 UploadFile 函数。您可以尝试仅将远程文件读取为文本:

const content = await got(url).text();

然后直接调用您的上传文件逻辑

const fileClient = fileSystemClient.getFileClient("directoryexample2/uploaded-file.txt");
await fileClient.create();
await fileClient.append(content, 0, content.length);
await fileClient.flush(content.length);