java - 在多线程应用程序中使用 VTD

Question

我有巨大的 xml，因此我有很多 xpath 来从 xml 中提取条目。所以我尝试生成多个线程，以便每个 xpath 可以在不同的线程中进行评估。但是我收到以下错误是可以给出一个公平想法的代码片段，为了简洁起见，我在这里使用了一个非常小的 xml。我正在创建 3 个线程并在 10 个任务中排队。

import java.io.File;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import com.ximpleware.AutoPilot;
import com.ximpleware.EOFException;
import com.ximpleware.EncodingException;
import com.ximpleware.EntityException;
import com.ximpleware.ParseException;
import com.ximpleware.VTDGen;
import com.ximpleware.VTDNav;

public class MultiThread {
    public static void main(String args[]) throws InterruptedException, ExecutionException, EncodingException, EOFException, EntityException, ParseException
    {
        String str="<library><booked>book</booked> <book id=\"1\"> <title>Googled By God</title>  </book> </library>";
        File f = new File("/home/cloudera/wos.xml");
        byte[] ba =null;;
        ExecutorService executor = Executors.newFixedThreadPool(3);
        List<Task> extractorTasks = new ArrayList<Task>();
        VTDGen vg = new VTDGen();
        vg.setDoc(str.getBytes());
        vg.parse(false);


        //add 10 tasks
        for(int i=0;i<10;i++)
        {

            Task d = new Task(str.getBytes(),vg,"/library/book/title");
            extractorTasks.add(d);
        }

        List<Future<String>> output = executor.invokeAll(extractorTasks);
        executor.shutdown();    
    }
}
class Task implements Callable<String> {

    VTDGen vg = null;
    String xpath = "";
    byte [] ba=null;
    AutoPilot ap = null;
    Task(byte[] _ba,VTDGen _vg,String _xpath)
    {
        ba = _ba;
        vg = _vg;
        xpath = _xpath;
    }
    public String call() throws Exception 
    {

        String title = "";
        try 
        {
            /* if we uncomment below 3 lines, all works well, thats becuase we are reparsing the whole document*/
            //vg = new VTDGen();
            //vg.setDoc(ba);
            //vg.parse(false);

            VTDNav vn = vg.getNav();
            ap = new AutoPilot(vn);
            ap.selectXPath(xpath);

            //Get all the titles and print each of those
            while(ap.evalXPath() != -1)
            {
                //getText will return the index of  the VTDRecord
                int titleIndex = vn.getText();
                //Get the text of the VTDRecord
                title = vn.toNormalizedString(titleIndex);
                System.out.println("Title is "+title);
            }

            vn.toElement(VTDNav.ROOT);

        }  
        catch (Exception e) {
            e.printStackTrace();
        }

        return title;
    }

}

Answer 1

我找到了修复方法。我将 VTDNav 存储在变量中并将其“重复项”传递给每个任务。 GetNav() 调用会清除内部状态，这可能还会导致 VTDnav 失效，因此保留导航器的副本并传递导航器的副本就可以了。

import java.io.File;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import com.ximpleware.AutoPilot;
import com.ximpleware.EOFException;
import com.ximpleware.EncodingException;
import com.ximpleware.EntityException;
import com.ximpleware.ParseException;
import com.ximpleware.VTDGen;
import com.ximpleware.VTDNav;

public class MultiThread {
    public static void main(String args[]) throws InterruptedException, ExecutionException, EncodingException, EOFException, EntityException, ParseException
    {
        String str="<library><booked>book</booked> <book id=\"1\"> <title>Googled By God</title>  </book> </library>";
        File f = new File("/home/cloudera/wos.xml");
        byte[] ba =null;;
        ExecutorService executor = Executors.newFixedThreadPool(5);
        List<Task> extractorTasks = new ArrayList<Task>();
        VTDGen vg = new VTDGen();
        vg.setDoc(str.getBytes());
        vg.parse(false);

        //The GetNav() call cleans internal state , so keep a copy of VTDNav
        VTDNav vn = vg.getNav();

        for(int i=0;i<100;i++)
        {
            //pass the duplicates of navigator
            Task d = new Task(str.getBytes(),vn.duplicateNav(),"/library/book/title");
            extractorTasks.add(d);
        }

        List<Future<String>> output = executor.invokeAll(extractorTasks);
        executor.shutdown();    
    }
}
class Task implements Callable<String> {

    VTDGen vg = null;
    String xpath = "";
    byte [] ba=null;
    VTDNav vn = null;
    AutoPilot ap = null;
    Task(byte[] _ba,VTDNav _vn,String _xpath)
    {
        ba = _ba;
        vn = _vn;
        xpath = _xpath;
    }
    public String call() throws Exception 
    {

        String title = "";
        try 
        {
            ap = new AutoPilot(vn);
            //Thread.sleep(500);
            ap.selectXPath(xpath);

            //Get all the titles and print each of those
            while(ap.evalXPath() != -1)
            {
                //getText will return the index of  the VTDRecord
                int titleIndex = vn.getText();
                //Get the text of the VTDRecord
                title = vn.toNormalizedString(titleIndex);
                System.out.println("Title is "+title);
            }

            //if(vn.toElement(VTDNav.ROOT) == true)
            //  System.out.println("to element failed");

        }  
        catch (Exception e) {
            e.printStackTrace();
            System.out.println("Message is "+e.getMessage());
        }

        return title;
    }

}