java - 将大型 JSON 对象存储到 java 文件中？

Question

在将 JSON 对象存储到文件时，我不断收到 Java heap space 错误。输入文件为 180 MB。

我正在创建一个 JSON 对象，它由 JSON 对象的 JSON 数组组成。在处理不太大的文件时，我总是会遇到此错误。
将此类对象存储到文件的最佳方法是什么？

public class ProcessData {

    public static void createJson(String path) throws Exception 
    {
        BufferedReader br = new BufferedReader(new FileReader(path));
        FileWriter fw = new FileWriter("restaurants.json");

        try 
        {
            JSONObject storeObj = new JSONObject();
            JSONArray restaurantArray = new JSONArray();

            String line;
            StringBuilder sb = new StringBuilder();

            while ((line = br.readLine()) != null) 
            {
                JSONObject obj = new JSONObject();
                String[] vals = line.split("\\t");
                obj.put("id", vals[0]);
                String[] tempvals = vals[1].split("\\|");
                String[] details = tempvals[0].split(",");

                obj.put("name", details[0]);
                sb.setLength(0);
                sb.append(details[2]);
                sb.append(", ");
                sb.append(details[3]);
                sb.append(", ");
                sb.append(details[1]);
                sb.append(", ");
                sb.append(details[4]);

                String address = sb.toString();
                address.replace("\\s+", " ");
                obj.put("address", address);
                JSONArray arr = new JSONArray();

                for (int i = 0; i < tempvals.length; ++i) 
                {
                    JSONObject objtemp = new JSONObject();
                    details = tempvals[i].split(",(?=([^\"]*\"[^\"]*\")*[^\"]*$)");

                    objtemp.put("inspectionDate", details[7]);
                    objtemp.put("code", details[9]);
                    objtemp.put("description", details[10]);
                    objtemp.put("score", details[12]);
                    objtemp.put("grade", details[13]);
                    objtemp.put("gradeDate", details[14]);
                    arr.add(objtemp);
                }
                obj.put("violationDetails", arr);
                restaurantArray.add(obj);
          }
          storeObj.put("restaurants", restaurantArray);
          fw.write(storeObj.toString());
        } 
        catch (Exception e) 
        {
            e.printStackTrace();
        } 
        finally 
        {
            fw.flush();
            fw.close();
            br.close();
        }
    }

    public static void main(String args[]) throws Exception {
        try{
        createJson("output/part-r-00000");
        }
        catch(Exception e){
            e.printStackTrace();
        }
    }
}

Answer 1

我尝试从您的函数中删除具体的 JSONObject 部分并手动创建您的 JSON 字符串并将其直接写入文件而不缓冲到您的对象中。

以下是您函数的更新代码:

public static void createJson(String path) throws Exception 
{
    BufferedReader br = new BufferedReader(new FileReader(path));
    FileWriter fw = new FileWriter("restaurants.json");
    try 
    {
        String line;
        StringBuilder sb = new StringBuilder();

        fw.write("{\"restaurants\":");
        line = br.readLine();
        while (line != null) 
        {
            fw.write("[{");
            String[] vals = line.split("\\t");

            fw.write("{");
            fw.write("\"id\":\""+vals[0]+"\",");

            String[] tempvals = vals[1].split("\\|");
            String[] details = tempvals[0].split(",");

            fw.write("\"name\":\""+details[0]+"\",");

            sb.setLength(0);
            sb.append(details[2]);
            sb.append(", ");
            sb.append(details[3]);
            sb.append(", ");
            sb.append(details[1]);
            sb.append(", ");
            sb.append(details[4]);

            String address = sb.toString();
            address.replace("\\s+", " ");

            fw.write("\"address\":\""+address+"\",");

            fw.write("\"violationDetails\":[");
            StringBuilder sbStr = new StringBuilder();
            for (int i = 0; i < tempvals.length; ++i) 
            {
                details = tempvals[i].split(",(?=([^\"]*\"[^\"]*\")*[^\"]*$)");

                sbStr.append("{");
                sbStr.append("\"inspectionDate\":\""+details[7]+"\",");
                sbStr.append("\"code\":\""+details[9]+"\",");
                sbStr.append("\"description\":\""+details[10]+"\",");
                sbStr.append("\"score\":\""+details[12]+"\",");
                sbStr.append("\"grade\":\""+details[13]+"\",");
                sbStr.append("\"gradeDate\":\""+details[14]+"\",");
                sbStr.append("},");
            }
            if(tempvals.length>0)
                sbStr.substring(0, sbStr.length()-1);

            fw.write(sbStr.toString()+"]");

            line = br.readLine();
            if(line != null)
                fw.write("}],");
            else
                fw.write("}]");
        }
        fw.write("}");
    } 
    catch (Exception e) 
    {
        e.printStackTrace();
    } 
    finally 
    {
        fw.flush();
        fw.close();
        br.close();
    }
}

我没有检查过真实的输入文件数据，所以请检查你的正确数据。

我希望这能解决您的问题。

文件中的 JSON 输出可能如下所示:

{ "restaurants": [ { "id": "id", "name": "name", "address": "Address", "violationDetails": [ { "inspectionDate": "details7", "code": "details9", "description": "details10", "score": "details12", "grade": "details13", "gradeDate": "details14" } ] } ] }