javascript - ftp 目录下载触发最大调用堆栈超出错误

Question

我目前正在使用 NodeJS 编写备份脚本。该脚本使用 FTP/FTPS 递归下载目录及其文件和子目录。我正在使用basic-ftp包进行 FTP 调用。

当我尝试下载包含大量子目录的大目录时，出现超出最大调用堆栈大小错误，但我不知道发生这种情况的原因和位置。我没有看到任何无限循环或任何丢失的返回调用。经过几个小时的调试，我没有更多的想法。

我不使用 basic-ftp 中的 downloadDirTo 方法，因为我不想在发生错误后停止下载。当发生错误时，它应该继续运行，并将错误添加到日志文件中。

存储库位于:https://github.com/julianpoemp/webspace-backup .

一旦 FTPManager 准备就绪，我就调用 doBackup 方法(请参阅 BackupManager 中的方法)。该方法调用FTPManager中定义的downloadFolder方法。

export class BackupManager {

    private ftpManager: FtpManager;

    constructor() {
        osLocale().then((locale) => {
            ConsoleOutput.info(`locale is ${locale}`);
            moment.locale(locale);
        }).catch((error) => {
            ConsoleOutput.error(error);
        });

        this.ftpManager = new FtpManager(AppSettings.settings.backup.root, {
            host: AppSettings.settings.server.host,
            port: AppSettings.settings.server.port,
            user: AppSettings.settings.server.user,
            password: AppSettings.settings.server.password,
            pasvTimeout: AppSettings.settings.server.pasvTimeout
        });

        this.ftpManager.afterManagerIsReady().then(() => {
            this.doBackup();
        }).catch((error) => {
            ConsoleOutput.error(error);
        });
    }

    public doBackup() {
        let errors = '';
        if (fs.existsSync(path.join(AppSettings.appPath, 'errors.log'))) {
            fs.unlinkSync(path.join(AppSettings.appPath, 'errors.log'));
        }
        if (fs.existsSync(path.join(AppSettings.appPath, 'statistics.txt'))) {
            fs.unlinkSync(path.join(AppSettings.appPath, 'statistics.txt'));
        }
        const subscr = this.ftpManager.error.subscribe((message: string) => {
            ConsoleOutput.error(`${moment().format('L LTS')}: ${message}`);
            const line = `${moment().format('L LTS')}:\t${message}\n`;
            errors += line;
            fs.appendFile(path.join(AppSettings.appPath, 'errors.log'), line, {
                encoding: 'Utf8'
            }, () => {
            });
        });

        let name = AppSettings.settings.backup.root.substring(0, AppSettings.settings.backup.root.lastIndexOf('/'));
        name = name.substring(name.lastIndexOf('/') + 1);
        const downloadPath = (AppSettings.settings.backup.downloadPath === '') ? AppSettings.appPath : AppSettings.settings.backup.downloadPath;

        ConsoleOutput.info(`Remote path: ${AppSettings.settings.backup.root}\nDownload path: ${downloadPath}\n`);

        this.ftpManager.statistics.started = Date.now();
        this.ftpManager.downloadFolder(AppSettings.settings.backup.root, path.join(downloadPath, name)).then(() => {
            this.ftpManager.statistics.ended = Date.now();
            this.ftpManager.statistics.duration = (this.ftpManager.statistics.ended - this.ftpManager.statistics.started) / 1000 / 60;

            ConsoleOutput.success('Backup finished!');
            const statistics = `\n-- Statistics: --
Started: ${moment(this.ftpManager.statistics.started).format('L LTS')}
Ended: ${moment(this.ftpManager.statistics.ended).format('L LTS')}
Duration: ${this.ftpManager.getTimeString(this.ftpManager.statistics.duration * 60 * 1000)} (H:m:s)

Folders: ${this.ftpManager.statistics.folders}
Files: ${this.ftpManager.statistics.files}
Errors: ${errors.split('\n').length - 1}`;

            ConsoleOutput.log('\n' + statistics);
            fs.writeFileSync(path.join(AppSettings.appPath, 'statistics.txt'), statistics, {
                encoding: 'utf-8'
            });
            if (errors !== '') {
                ConsoleOutput.error(`There are errors. Please read the errors.log file for further information.`);
            }
            subscr.unsubscribe();
            this.ftpManager.close();
        }).catch((error) => {
            ConsoleOutput.error(error);
            this.ftpManager.close();
        });
    }
}

import * as ftp from 'basic-ftp';
import {FileInfo} from 'basic-ftp';
import * as Path from 'path';
import * as fs from 'fs';
import {Subject} from 'rxjs';
import {FtpEntry, FTPFolder} from './ftp-entry';
import {ConsoleOutput} from './ConsoleOutput';
import moment = require('moment');

export class FtpManager {
    private isReady = false;
    private _client: ftp.Client;
    private currentDirectory = '';

    public readyChange: Subject<boolean>;
    public error: Subject<string>;
    private connectionOptions: FTPConnectionOptions;

    public statistics = {
        folders: 0,
        files: 0,
        started: 0,
        ended: 0,
        duration: 0
    };

    private recursives = 0;

    constructor(path: string, options: FTPConnectionOptions) {
        this._client = new ftp.Client();
        this._client.ftp.verbose = false;
        this.readyChange = new Subject<boolean>();
        this.error = new Subject<string>();
        this.currentDirectory = path;
        this.connectionOptions = options;


        this.connect().then(() => {
            this.isReady = true;
            this.gotTo(path).then(() => {
                this.onReady();
            }).catch((error) => {
                ConsoleOutput.error('ERROR: ' + error);
                this.onConnectionFailed();
            });
        });
    }

    private connect(): Promise<void> {
        return new Promise<void>((resolve, reject) => {
            this._client.access({
                host: this.connectionOptions.host,
                user: this.connectionOptions.user,
                password: this.connectionOptions.password,
                secure: true
            }).then(() => {
                resolve();
            }).catch((error) => {
                reject(error);
            });
        });
    }

    private onReady = () => {
        this.isReady = true;
        this.readyChange.next(true);
    };

    private onConnectionFailed() {
        this.isReady = false;
        this.readyChange.next(false);
    }

    public close() {
        this._client.close();
    }

    public async gotTo(path: string) {
        return new Promise<void>((resolve, reject) => {
            if (this.isReady) {
                ConsoleOutput.info(`open ${path}`);
                this._client.cd(path).then(() => {
                    this._client.pwd().then((dir) => {
                        this.currentDirectory = dir;
                        resolve();
                    }).catch((error) => {
                        reject(error);
                    });
                }).catch((error) => {
                    reject(error);
                });
            } else {
                reject(`FTPManager is not ready. gotTo ${path}`);
            }
        });
    }

    public async listEntries(path: string): Promise<FileInfo[]> {
        if (this.isReady) {
            return this._client.list(path);
        } else {
            throw new Error('FtpManager is not ready. list entries');
        }
    }

    public afterManagerIsReady(): Promise<void> {
        return new Promise<void>((resolve, reject) => {
            if (this.isReady) {
                resolve();
            } else {
                this.readyChange.subscribe(() => {
                        resolve();
                    },
                    (error) => {
                        reject(error);
                    },
                    () => {
                    });
            }
        });
    }

    public async downloadFolder(remotePath: string, downloadPath: string) {
        this.recursives++;

        if (this.recursives % 100 === 99) {
            ConsoleOutput.info('WAIT');
            await this.wait(0);
        }

        if (!fs.existsSync(downloadPath)) {
            fs.mkdirSync(downloadPath);
        }

        try {
            const list = await this.listEntries(remotePath);
            for (const fileInfo of list) {
                if (fileInfo.isDirectory) {
                    const folderPath = remotePath + fileInfo.name + '/';
                    try {
                        await this.downloadFolder(folderPath, Path.join(downloadPath, fileInfo.name));
                        this.statistics.folders++;
                        ConsoleOutput.success(`${this.getCurrentTimeString()}===> Directory downloaded: ${remotePath}\n`);
                    } catch (e) {
                        this.error.next(e);
                    }
                } else if (fileInfo.isFile) {
                    try {
                        const filePath = remotePath + fileInfo.name;
                        if (this.recursives % 100 === 99) {
                            ConsoleOutput.info('WAIT');
                            await this.wait(0);
                        }
                        await this.downloadFile(filePath, downloadPath, fileInfo);
                    } catch (e) {
                        this.error.next(e);
                    }
                }
            }
            return true;
        } catch (e) {
            this.error.next(e);
            return true;
        }
    }

    public async downloadFile(path: string, downloadPath: string, fileInfo: FileInfo) {
        this.recursives++;
        if (fs.existsSync(downloadPath)) {
            const handler = (info) => {
                let procent = Math.round((info.bytes / fileInfo.size) * 10000) / 100;
                if (isNaN(procent)) {
                    procent = 0;
                }
                let procentStr = '';
                if (procent < 10) {
                    procentStr = '__';
                } else if (procent < 100) {
                    procentStr = '_';
                }
                procentStr += procent.toFixed(2);

                ConsoleOutput.log(`${this.getCurrentTimeString()}---> ${info.type} (${procentStr}%): ${info.name}`);
            };

            if (this._client.closed) {
                try {
                    await this.connect();
                } catch (e) {
                    throw new Error(e);
                }
            }
            this._client.trackProgress(handler);
            try {
                await this._client.downloadTo(Path.join(downloadPath, fileInfo.name), path);
                this._client.trackProgress(undefined);
                this.statistics.files++;
                return true;
            } catch (e) {
                throw new Error(e);
            }
        } else {
            throw new Error('downloadPath does not exist');
        }
    }

    public chmod(path: string, permission: string): Promise<void> {
        return new Promise<void>((resolve, reject) => {
            this._client.send(`SITE CHMOD ${permission} ${path}`).then(() => {
                console.log(`changed chmod of ${path} to ${permission}`);
                resolve();
            }).catch((error) => {
                reject(error);
            });
        });
    }

    public getCurrentTimeString(): string {
        const duration = Date.now() - this.statistics.started;
        return moment().format('L LTS') + ' | Duration: ' + this.getTimeString(duration) + ' ';
    }

    public getTimeString(timespan: number) {
        if (timespan < 0) {
            timespan = 0;
        }

        let result = '';
        const minutes: string = this.formatNumber(this.getMinutes(timespan), 2);
        const seconds: string = this.formatNumber(this.getSeconds(timespan), 2);
        const hours: string = this.formatNumber(this.getHours(timespan), 2);

        result += hours + ':' + minutes + ':' + seconds;

        return result;
    }

    private formatNumber = (num, length): string => {
        let result = '' + num.toFixed(0);
        while (result.length < length) {
            result = '0' + result;
        }
        return result;
    };

    private getSeconds(timespan: number): number {
        return Math.floor(timespan / 1000) % 60;
    }

    private getMinutes(timespan: number): number {
        return Math.floor(timespan / 1000 / 60) % 60;
    }

    private getHours(timespan: number): number {
        return Math.floor(timespan / 1000 / 60 / 60);
    }

    public async wait(time: number): Promise<void> {
        return new Promise<void>((resolve) => {
            setTimeout(() => {
                resolve();
            }, time);
        });
    }
}


export interface FTPConnectionOptions {
    host: string;
    port: number;
    user: string;
    password: string;
    pasvTimeout: number;
}

Answer 1

问题

在 FtpManager.downloadFolder 函数中，我看到使用 await 递归调用相同的 downloadFolder 方法。您的超出最大调用堆栈错误可能来自那里，因为您的初始调用需要在遍历所有子目录时将所有内容保留在内存中。

建议的解决方案

您可以使用如下算法设置队列系统，而不是递归地等待所有内容:

• 将当前文件夹添加到队列
• 虽然该队列不为空:
  • 获取队列中的第一个文件夹(并将其从中删除)
  • 列出其中的所有条目
  • 下载所有文件
  • 将所有子文件夹添加到队列

这允许您循环下载大量文件夹，而不是使用递归。每个循环迭代将独立运行，这意味着根目录下载的结果不会依赖于其中的deeeeeep文件树。

使用队列管理器

NodeJS 有很多队列管理器模块，它们允许您实现并发、超时等。我过去使用过的一个简单地命名为 queue 。它有很多有用的功能，但需要做更多的工作才能在您的项目中实现。因此，对于这个答案，我没有使用外部队列模块，以便您可以看到其背后的逻辑。请随意搜索队列、作业、并发...

示例

我想直接在您自己的代码中实现该逻辑，但我不使用 Typescript，所以我想我应该制作一个简单的文件夹复制功能，它使用相同的逻辑。

注意:为了简单起见，我没有添加任何错误处理，这只是一个概念证明!您可以找到一个使用此here on my Github的演示项目.

这是我的做法:

const fs = require('fs-extra');
const Path = require('path');

class CopyManager {
  constructor() {
    // Create a queue accessible by all methods
    this.folderQueue = [];
  }

  /**
   * Copies a directory
   * @param {String} remotePath
   * @param {String} downloadPath
   */
  async copyFolder(remotePath, downloadPath) {
    // Add the folder to the queue
    this.folderQueue.push({ remotePath, downloadPath });
    // While the queue contains folders to download
    while (this.folderQueue.length > 0) {
      // Download them
      const { remotePath, downloadPath } = this.folderQueue.shift();
      console.log(`Copy directory: ${remotePath} to ${downloadPath}`);
      await this._copyFolderAux(remotePath, downloadPath);
    }
  }

  /**
   * Private internal method which copies the files from a folder,
   * but if it finds subfolders, simply adds them to the folderQueue
   * @param {String} remotePath
   * @param {String} downloadPath
   */
  async _copyFolderAux(remotePath, downloadPath) {
    await fs.mkdir(downloadPath);
    const list = await this.listEntries(remotePath);
    for (const fileInfo of list) {
      if (fileInfo.isDirectory) {
        const folderPath = Path.join(remotePath, fileInfo.name);
        const targetPath = Path.join(downloadPath, fileInfo.name);
        // Push the folder to the queue
        this.folderQueue.push({ remotePath: folderPath, downloadPath: targetPath });
      } else if (fileInfo.isFile) {
        const filePath = Path.join(remotePath, fileInfo.name);
        await this.copyFile(filePath, downloadPath, fileInfo);
      }
    }
  }

  /**
   * Copies a file
   * @param {String} filePath
   * @param {String} downloadPath
   * @param {Object} fileInfo
   */
  async copyFile(filePath, downloadPath, fileInfo) {
    const targetPath = Path.join(downloadPath, fileInfo.name);
    console.log(`Copy file: ${filePath} to ${targetPath}`);
    return await fs.copy(filePath, targetPath);
  }

  /**
   * Lists entries from a folder
   * @param {String} remotePath
   */
  async listEntries(remotePath) {
    const fileNames = await fs.readdir(remotePath);
    return Promise.all(
      fileNames.map(async name => {
        const stats = await fs.lstat(Path.join(remotePath, name));
        return {
          name,
          isDirectory: stats.isDirectory(),
          isFile: stats.isFile()
        };
      })
    );
  }
}

module.exports = CopyManager;