java - 如何在 JavaFX 中高频率显示图像？

Question

我的应用程序生成热图图像的速度与 CPU 的速度一样快(大约每秒 30-60 张)，我想在单个“实时热图中”显示它们。在 AWT/Swing 中，我可以将它们绘制到 JPanel 中，这非常有用。最近，我切换到 JavaFX 并希望在这里实现相同的目标；起初，我尝试使用 Canvas ，速度很慢但还可以，但存在严重的内存泄漏问题，导致应用程序崩溃。现在，我尝试了 ImageView组件 - 这显然太慢了，因为图像变得非常滞后(在每次新迭代中使用 ImageView.setImage)。据我了解，setImage 不保证图像在函数完成时实际显示。

我的印象是我在错误的轨道上，以不适合的方式使用这些组件。如何每秒显示 30-60 张图像？

编辑:一个非常简单的测试应用程序。您将需要 JHeatChart图书馆。请注意，在台式机上，我获得了大约 70-80 FPS，并且可视化效果还不错且流畅，但在较小的 raspberry pi(我的目标机器)上，我获得了大约 30 FPS，但可视化非常卡住。

package sample;

import javafx.application.Application;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.ImageView;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;
import org.tc33.jheatchart.HeatChart;

import java.awt.*;
import java.awt.geom.AffineTransform;
import java.awt.image.AffineTransformOp;
import java.awt.image.BufferedImage;
import java.util.LinkedList;

public class Main extends Application {
ImageView imageView = new ImageView();
final int scale = 15;

@Override
public void start(Stage primaryStage) {
    Thread generator = new Thread(() -> {
        int col = 0;
        LinkedList<Long> fps = new LinkedList<>();
        while (true) {
            fps.add(System.currentTimeMillis());
            double[][] matrix = new double[48][128];
            for (int i = 0; i < 48; i++) {
                for (int j = 0; j < 128; j++) {
                    matrix[i][j] = col == j ? Math.random() : 0;
                }
            }
            col = (col + 1) % 128;

            HeatChart heatChart = new HeatChart(matrix, 0, 1);
            heatChart.setShowXAxisValues(false);
            heatChart.setShowYAxisValues(false);
            heatChart.setLowValueColour(java.awt.Color.black);
            heatChart.setHighValueColour(java.awt.Color.white);
            heatChart.setAxisThickness(0);
            heatChart.setChartMargin(0);
            heatChart.setCellSize(new Dimension(1, 1));

            long currentTime = System.currentTimeMillis();
            fps.removeIf(elem -> currentTime - elem > 1000);
            System.out.println(fps.size());

            imageView.setImage(SwingFXUtils.toFXImage((BufferedImage) scale(heatChart.getChartImage(), scale), null));
        }
    });

    VBox box = new VBox();
    box.getChildren().add(imageView);

    Scene scene = new Scene(box, 1920, 720);
    primaryStage.setScene(scene);
    primaryStage.show();

    generator.start();
}


public static void main(String[] args) {
    launch(args);
}

private static Image scale(Image image, int scale) {
    BufferedImage res = new BufferedImage(image.getWidth(null) * scale, image.getHeight(null) * scale,
            BufferedImage.TYPE_INT_ARGB);
    AffineTransform at = new AffineTransform();
    at.scale(scale, scale);
    AffineTransformOp scaleOp =
            new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);

    return scaleOp.filter((BufferedImage) image, res);
}

Answer 1

您的代码从后台线程更新 UI，这是绝对不允许的。您需要确保从 FX Application Thread 进行更新。您还想尝试“限制”实际的 UI 更新，使其在每次 JavaFX 帧渲染时不超过一次。最简单的方法是使用 AnimationTimer，每次渲染帧时都会调用其 handle() 方法。

这是执行此操作的代码版本:

import java.awt.Dimension;
import java.awt.Image;
import java.awt.geom.AffineTransform;
import java.awt.image.AffineTransformOp;
import java.awt.image.BufferedImage;
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicReference;

import org.tc33.jheatchart.HeatChart;

import javafx.animation.AnimationTimer;
import javafx.application.Application;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.ImageView;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;

public class Main extends Application {
    ImageView imageView = new ImageView();
    final int scale = 15;

    @Override
    public void start(Stage primaryStage) {

        AtomicReference<BufferedImage> image = new AtomicReference<>();

        Thread generator = new Thread(() -> {
            int col = 0;
            LinkedList<Long> fps = new LinkedList<>();
            while (true) {
                fps.add(System.currentTimeMillis());
                double[][] matrix = new double[48][128];
                for (int i = 0; i < 48; i++) {
                    for (int j = 0; j < 128; j++) {
                        matrix[i][j] = col == j ? Math.random() : 0;
                    }
                }
                col = (col + 1) % 128;

                HeatChart heatChart = new HeatChart(matrix, 0, 1);
                heatChart.setShowXAxisValues(false);
                heatChart.setShowYAxisValues(false);
                heatChart.setLowValueColour(java.awt.Color.black);
                heatChart.setHighValueColour(java.awt.Color.white);
                heatChart.setAxisThickness(0);
                heatChart.setChartMargin(0);
                heatChart.setCellSize(new Dimension(1, 1));

                long currentTime = System.currentTimeMillis();
                fps.removeIf(elem -> currentTime - elem > 1000);
                System.out.println(fps.size());

                image.set((BufferedImage) scale(heatChart.getChartImage(), scale));

            }
        });

        VBox box = new VBox();
        box.getChildren().add(imageView);

        Scene scene = new Scene(box, 1920, 720);
        primaryStage.setScene(scene);
        primaryStage.show();

        generator.setDaemon(true);
        generator.start();

        AnimationTimer animation = new AnimationTimer() {

            @Override
            public void handle(long now) {
                BufferedImage img = image.getAndSet(null);
                if (img != null) {
                    imageView.setImage(SwingFXUtils.toFXImage(img, null));
                }
            }

        };

        animation.start();
    }

    public static void main(String[] args) {
        launch(args);
    }

    private static Image scale(Image image, int scale) {
        BufferedImage res = new BufferedImage(image.getWidth(null) * scale, image.getHeight(null) * scale,
                BufferedImage.TYPE_INT_ARGB);
        AffineTransform at = new AffineTransform();
        at.scale(scale, scale);
        AffineTransformOp scaleOp = new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);

        return scaleOp.filter((BufferedImage) image, res);
    }
}

使用 AtomicReference 包装缓冲图像确保它在两个线程之间安全共享。

在我的机器上，这每秒生成大约 130 张图像；请注意，并非所有内容都会显示，因为每次 JavaFX 图形框架显示一帧(通常限制在 60fps)时，只会显示最新的一个。

如果您想确保显示生成的所有图像，即您通过 JavaFX 帧速率限制图像生成，那么您可以使用 BlockingQueue 来存储图片:

    // AtomicReference<BufferedImage> image = new AtomicReference<>();

    // Size of the queue is a trade-off between memory consumption
    // and smoothness (essentially works as a buffer size)
    BlockingQueue<BufferedImage> image = new ArrayBlockingQueue<>(5);

    // ...

    // image.set((BufferedImage) scale(heatChart.getChartImage(), scale));
    try {
        image.put((BufferedImage) scale(heatChart.getChartImage(), scale));
    } catch (InterruptedException exc) {
        Thread.currentThread.interrupt();
    }

和

        @Override
        public void handle(long now) {
            BufferedImage img = image.poll();
            if (img != null) {
                imageView.setImage(SwingFXUtils.toFXImage(img, null));
            }
        }

代码效率很低，因为您在每次迭代时都会生成一个新矩阵、新的 HeatChart 等。这会导致在堆上创建许多对象并迅速丢弃，这会导致 GC 运行过于频繁，尤其是在小内存机器上。也就是说，我在将最大堆大小设置为 64MB (-Xmx64m) 的情况下运行此程序，它仍然运行良好。您可以优化代码，但使用 AnimationTimer 可以更快地生成图像，不会对 JavaFX 框架造成任何额外压力。我建议调查使用 HeatChart 的可变性(即 setZValues())以避免创建太多对象，和/或使用 PixelBuffer直接将数据写入 ImageView (这需要在 FX 应用程序线程上完成)。

这是一个不同的例子，它(几乎)完全减少了对象的创建，使用一个屏幕外的 int[] 数组来计算数据，并使用一个屏幕上的 int[]数组来显示它。有一些低级线程细节可确保屏幕上的阵列仅在一致状态下可见。屏幕上的数组用于底层 PixelBuffer，后者又用于 WritableImage。

此类生成图像数据:

import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;

public class ImageGenerator {

    private final int width;
    private final int height;


    // Keep two copies of the data: one which is not exposed
    // that we modify on the fly during computation;
    // another which we expose publicly. 
    // The publicly exposed one can be viewed only in a complete 
    // state if operations on it are synchronized on this object.
    private final int[] privateData ;
    private final int[] publicData ;

    private final long[] frameTimes ;
    private int currentFrameIndex ;
    private final AtomicLong averageGenerationTime ;

    private final ReentrantLock lock ;


    private static final double TWO_PI = 2 * Math.PI;
    private static final double PI_BY_TWELVE = Math.PI / 12; // 15 degrees

    public ImageGenerator(int width, int height) {
        super();
        this.width = width;
        this.height = height;
        privateData = new int[width * height];
        publicData = new int[width * height];

        lock = new ReentrantLock();

        this.frameTimes = new long[100];
        this.averageGenerationTime = new AtomicLong();
    }

    public void generateImage(double angle) {

        // compute in private data copy:

        int minDim = Math.min(width, height);
        int minR2 = minDim * minDim / 4;
        for (int x = 0; x < width; x++) {
            int xOff = x - width / 2;
            int xOff2 = xOff * xOff;
            for (int y = 0; y < height; y++) {

                int index = x + y * width;

                int yOff = y - height / 2;
                int yOff2 = yOff * yOff;
                int r2 = xOff2 + yOff2;
                if (r2 > minR2) {
                    privateData[index] = 0xffffffff; // white
                } else {
                    double theta = Math.atan2(yOff, xOff);
                    double delta = Math.abs(theta - angle);
                    if (delta > TWO_PI - PI_BY_TWELVE) {
                        delta = TWO_PI - delta;
                    }
                    if (delta < PI_BY_TWELVE) {
                        int green = (int) (255 * (1 - delta / PI_BY_TWELVE));
                        privateData[index] = (0xff << 24) | (green << 8); // green, fading away from center
                    } else {
                        privateData[index] = 0xff << 24; // black
                    }
                }
            }
        }

        // copy computed data to public data copy:
        lock.lock(); 
        try {
            System.arraycopy(privateData, 0, publicData, 0, privateData.length);
        } finally {
            lock.unlock();
        }

        frameTimes[currentFrameIndex] = System.nanoTime() ;
        int nextIndex = (currentFrameIndex + 1) % frameTimes.length ;
        if (frameTimes[nextIndex] > 0) {
            averageGenerationTime.set((frameTimes[currentFrameIndex] - frameTimes[nextIndex]) / frameTimes.length);
        }
        currentFrameIndex = nextIndex ;
    }


    public void consumeData(Consumer<int[]> consumer) {
        lock.lock();
        try {
            consumer.accept(publicData);
        } finally {
            lock.unlock();
        }
    }

    public long getAverageGenerationTime() {
        return averageGenerationTime.get() ;
    }

}

这是用户界面:

import java.nio.IntBuffer;

import javafx.animation.AnimationTimer;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.scene.control.Label;
import javafx.scene.image.ImageView;
import javafx.scene.image.PixelFormat;
import javafx.scene.image.PixelWriter;
import javafx.scene.image.WritableImage;
import javafx.scene.layout.BorderPane;
import javafx.stage.Stage;

public class AnimationApp extends Application {


    private final int size = 400 ;
    private IntBuffer buffer ;

    @Override
    public void start(Stage primaryStage) throws Exception {

        // background image data generation:

        ImageGenerator generator = new ImageGenerator(size, size);

        // Generate new image data as fast as possible:
        Thread thread = new Thread(() ->  {
            while( true ) {
                long now = System.currentTimeMillis() ;
                double angle = 2 * Math.PI * (now % 10000) / 10000  - Math.PI;
                generator.generateImage(angle);
            }
        });
        thread.setDaemon(true);
        thread.start();


        generator.consumeData(data -> buffer = IntBuffer.wrap(data));
        PixelFormat<IntBuffer> format = PixelFormat.getIntArgbPreInstance() ;
        PixelBuffer<IntBuffer> pixelBuffer = new PixelBuffer<>(size, size, buffer, format);
        WritableImage image = new WritableImage(pixelBuffer);

        BorderPane root = new BorderPane(new ImageView(image));

        Label fps = new Label("FPS: ");
        root.setTop(fps);

        Scene scene = new Scene(root);
        primaryStage.setScene(scene);
        primaryStage.setTitle("Give me a ping, Vasili. ");
        primaryStage.show();

        AnimationTimer animation = new AnimationTimer() {

            @Override
            public void handle(long now) { 
                // Update image, ensuring we only see the underlying
                // data in a consistent state:
                generator.consumeData(data ->  {
                    pixelBuffer.updateBuffer(pb -> null); 
                });
                long aveGenTime = generator.getAverageGenerationTime() ;
                if (aveGenTime > 0) {
                    double aveFPS = 1.0 / (aveGenTime / 1_000_000_000.0);
                    fps.setText(String.format("FPS: %.2f", aveFPS));
                }
            }

        };

        animation.start();

    }



    public static void main(String[] args) {
        Application.launch(args);
    }
}

对于不依赖于 JavaFX 13 PixelBuffer 的版本，您可以修改此类以使用 PixelWriter(AIUI 这不会像高效，但在这个例子中工作同样顺利):

//      generator.consumeData(data -> buffer = IntBuffer.wrap(data));
        PixelFormat<IntBuffer> format = PixelFormat.getIntArgbPreInstance() ;
//      PixelBuffer<IntBuffer> pixelBuffer = new PixelBuffer<>(size, size, buffer, format);
//      WritableImage image = new WritableImage(pixelBuffer);

        WritableImage image = new WritableImage(size, size);
        PixelWriter pixelWriter = image.getPixelWriter() ;

和

        AnimationTimer animation = new AnimationTimer() {

            @Override
            public void handle(long now) { 
                // Update image, ensuring we only see the underlying
                // data in a consistent state:
                generator.consumeData(data ->  {
//                  pixelBuffer.updateBuffer(pb -> null); 
                    pixelWriter.setPixels(0, 0, size, size, format, data, 0, size);
                });
                long aveGenTime = generator.getAverageGenerationTime() ;
                if (aveGenTime > 0) {
                    double aveFPS = 1.0 / (aveGenTime / 1_000_000_000.0);
                    fps.setText(String.format("FPS: %.2f", aveFPS));
                }
            }

        };