java - 将大型二进制数据写入文件

Question

您好，我正在开展一个编写我们自己的霍夫曼编码的项目。我目前在将二进制 1 和 0 写入输出文件时遇到问题。它适用于较小的输入文件，但对于非常大的文件，它不会将任何内容写入输出文件。负责写入的方法是 compress 方法。任何帮助，将不胜感激。谢谢!

package proj3;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Scanner;
import java.util.PriorityQueue;

public class Project3 {

//variables for PriorityQueue and Huffman Tree
private static PriorityQueue<BinaryNode<Character>> queue;
private static BinaryNode<Character> huffTree;
private static Map<Character, String> table = new LinkedHashMap<Character, String>();

/**
 * Method for creating Huffman Tree
 * @param counts Map that contains all characters and their frequencies
 * @return the Huffman Tree
 */
public static BinaryNode<Character> makeTree(Map<Character, Integer> counts)
{
    queue = new PriorityQueue<BinaryNode<Character>>();

    for(Character c : counts.keySet())
    {
        BinaryNode<Character> tree = new BinaryNode<Character>(c, counts.get(c), null, null);
        queue.add(tree);
    }

    while(!queue.isEmpty())
    {
        if(queue.size() >= 2)
        {   
            BinaryNode<Character> n1 = queue.remove();
            BinaryNode<Character> n2 = queue.remove();
            Integer weight = n1.getFreq() + n2.getFreq();
            huffTree = new BinaryNode<Character>(null, weight, n1, n2);
            queue.add(huffTree);
        }
        if(queue.size() == 1)
        {
            return queue.remove();
        }
    }
    return huffTree;
}

public static void encode(BinaryNode<Character> node, String s)
{
    if(!node.isLeaf())
    {
        encode(node.getLeft(), s + "0");
        encode(node.getRight(), s + "1");
    }
    else
    {
        table.put(node.getElement(), s);
    }
}

public static void compress(String in, String out) throws IOException
{
    try 
    {
        File outFile = new File(out);
        FileOutputStream compressedFile = new FileOutputStream(outFile);
        Scanner infile = new Scanner(new FileInputStream(in));
        while(infile.hasNext())
        {
            infile.useDelimiter("");
            String str = infile.next();
            Character character = str.charAt(0);
            for(Character c : table.keySet())
            {
                if(c == character){
                    compressedFile.write(table.get(c).getBytes());
                    compressedFile.flush();
                }
            }
        }
        for(Byte b : table.get('^').getBytes())
        {
            compressedFile.write(b);
        }
        infile.close();
        compressedFile.close();
    }
    catch (FileNotFoundException e) 
    {
        System.err.println("File not found.");
        e.printStackTrace();
    }
}

public static void decompress(String s)
{

}

public static void printEncodings(Map<Character, String> m)
{
    ArrayList<Character> chars = new ArrayList<Character>();

    System.out.println("Character Encodings");
    System.out.println("---------------------");
    for(Character c : m.keySet())
    {
        chars.add(c);
        Collections.sort(chars);
    }
    for(Character c : chars)
    {
        System.out.print(c + "\t" + m.get(c) + "\n");
    }
    System.out.println();
    System.out.println("Total Characters: " + chars.size());
}

/**
 * Method for creating map with character and its frequencies
 * @param s the file name to be opened
 * @return the Map containing characters and frequencies
 */
public static Map<Character, Integer> charCount(String s){

    Map<Character, Integer> counts = new LinkedHashMap<Character, Integer>();
    ArrayList<Character> chars = new ArrayList<Character>();

    try {
        Scanner file = new Scanner(new FileInputStream(s));
        while(file.hasNext()){
            file.useDelimiter("");
            String str = file.next();
            Character c = str.charAt(0);
            if(counts.containsKey(c)){
                counts.put(c, counts.get(c) + 1);
            }
            else{
                counts.put(c, 1);
            }
        }
        counts.put('^', 1);
        System.out.println("Character Frequencies");
        System.out.println("---------------------");
        for(Character c : counts.keySet())
        {
            chars.add(c);
            Collections.sort(chars);
        }
        for(Character c : chars){
            System.out.println(c + "\t" + counts.get(c));
        }
        System.out.println();
        System.out.println("Total characters: " + chars.size() + "\n");
        file.close();
    } 
    catch (FileNotFoundException e) {
        System.err.println("File not found.");
        System.exit(0);
    }
    return counts;
}

public static void main(String[] args){

    if(args.length != 3)
    {
        throw new IllegalArgumentException("Invalid number of arguments.");
    }
    encode(makeTree(charCount(args[0])), "");
    printEncodings(table);
    try {
        compress(args[0], args[1]);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

}

Answer 1

我很确定您在 compress() 方法中遇到了性能/内存问题。编码在您的主要方法中打印正常，但我相信文件输出会卡住。我可以在您的代码中看到至少三种优化的可能性:

1. 您正在使用Scanner逐字符读取输入文件，但不使用 Scanner 提供的任何解析功能。尝试使用 InputStreamReader相反。

2. 您正在循环遍历霍夫曼表中的键集并检查是否相等。您可以简单地使用当前字符来获取它映射到的字符串并省略循环。

3. 您无需循环字节数组即可将其写入输出文件。 FileOutputStream的write()方法可以将整个字节数组作为参数。

凭借我拙劣的 Java 技能，我宁愿以如下方式实现它；请注意，这是未经测试的代码，因为我没有您的 BinaryNode 类:

public static void compress(String in, String out) throws IOException
{
    try 
        {
            File outFile = new File(out);
            FileOutputStream compressedFile = new FileOutputStream(outFile);

            // 1. Use a Reader instead of a Scanner;
            // make sure to use the correct charset
            FileInputStream fis= new FileInputStream(in);
            Reader reader = new InputStreamReader(fis,        
                Charset.forName("US-ASCII"));

            // use BufferedReader for even better performance
            Reader bufferedReader = new BufferedReader(reader);

            int r;
            while ((r = bufferedReader.read()) != -1) {
                char ch= (char) r;

                // 2. Get the string for this character directly instead of
                // looping the keySet and checking for equivalence
                String s= table.get(ch);
                if (s != null) {
                    // 3. Write entire array of bytes instead of 
                    // looping and writing bytes one by one
                    compressedFile.write(s.getBytes());
                }
            }
            fis.close();
            compressedFile.close();
        }
    ...