redis - 在 Redis 中寻找高内存使用率的扩展

Question

我们有大量数据存储在 Redis 中。实际上，我们在 Redis 中存储了大量的键以及与每个键关联的极小数据。键的长度为 8 个字节，数据的长度为 8 个字节(长值)。有 10 亿 个 key (是的，十亿)。

鉴于 Redis 存储的结构，据我所知(https://redislabs.com/blog/redis-ram-ramifications-part-i/ 和 https://github.com/antirez/sds/blob/master/README.md)给定 8 个字节的键， header 的开销为 8 个字节，末尾的 null 为 1 个字节 key 。那是17个字节。假设这四舍五入到至少 24 个字节，加上 8 个字节的 long 值得到 32 个字节。

十亿个 key 将是 32GB。实测使用量为158GB。当然有开销，但 5:1 的比例似乎很大。任何人都可以解释这一点或指出减少内存使用的方法。

我已经包含了我基于 Jedis 的测试程序。

import java.security.SecureRandom;
import java.text.DecimalFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.pool2.impl.GenericObjectPoolConfig;

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.exceptions.JedisClusterMaxRedirectionsException;

public class Test8byteKeys {
    protected static JedisCluster cluster = null;
    protected static final ExecutorService executor;

    protected static volatile boolean shuttingDown = false;

    private static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors();

    static {
        final int cores = Math.max(4, (AVAILABLE_PROCESSORS * 3) / 4);
        executor = new ThreadPoolExecutor(cores, cores, //
                15, TimeUnit.SECONDS, //
                new LinkedBlockingQueue<>(cores), //
                new ThreadPoolExecutor.CallerRunsPolicy());

        System.out.println("Running with " + cores + " threads");
    }

    static private GenericObjectPoolConfig getPoolConfiguration() {
        GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();

        poolConfig.setLifo(true);
        poolConfig.setTestOnBorrow(true);
        poolConfig.setTestOnReturn(false);
        poolConfig.setBlockWhenExhausted(true);
        poolConfig.setMinIdle(1);
        poolConfig.setMaxTotal(101);
        poolConfig.setTestWhileIdle(false);
        poolConfig.setSoftMinEvictableIdleTimeMillis(3000L);
        poolConfig.setNumTestsPerEvictionRun(5);
        poolConfig.setTimeBetweenEvictionRunsMillis(5000L);
        poolConfig.setJmxEnabled(true);

        return poolConfig;
    }

    private static void connectToCluster() {
        try {
            Set<HostAndPort> nodes = new HashSet<>();
            String hap /* host and port */ = System.getProperty("hap", null);
            if (hap == null) {
                System.err.println("You must supply the host and port of a master in the cluster on the command line");
                System.err.println("java -Dhap=<host:port> -jar <jar> ");
                System.exit(1);
            }

            String[] parts = hap.split(":"); // assume ipv4 address
            nodes.add(new HostAndPort(parts[0].trim(), Integer.valueOf(parts[1].trim())));

            System.out.println("Connecting to " + hap);
            cluster = new JedisCluster(nodes, getPoolConfiguration());
        }
        catch (Exception e) {
            System.err.println("Could not connect to redis -- " + e.getMessage());
            System.exit(1);
        }
    }

    private static final Thread shutdown = new Thread(new Runnable() {
        // Clean up at exit
        @Override
        public void run() {
            shuttingDown = true;

            System.out.println((new Date()).toString() + "\t" + "Executor shutdown in progress");

            try {
                executor.shutdown();
                executor.awaitTermination(10L, TimeUnit.SECONDS);
            }
            catch (Exception e) {
                // ignore
            }
            finally {
                try {
                    if (!executor.isShutdown()) {
                        executor.shutdownNow();
                    }
                }
                catch (Exception e) {
                    //ignore
                }
            }

            try {
                cluster.close();
            }
            catch (Exception e) {
                System.err.println("cluster disconnection failure: " + e);
            }
            finally {
                //
            }

            System.out.println((new Date()).toString() + "\t" + "shutdown complete.");
        }
    });

    final static char[] CHARACTERS = { //
            '0', '1', '2', '3', '4', '5', //
            '6', '7', '8', '9', 'a', 'b', //
            'c', 'd', 'e', 'f', 'g', 'h', //
            'i', 'j', 'k', 'l', 'm', 'n', //
            'o', 'p', 'q', 'r', 's', 't', //
            'u', 'v', 'w', 'x', 'y', 'z', //
            'A', 'B', 'C', 'D', 'E', 'F', //
            'G', 'H', 'I', 'J', 'K', 'L', //
            'M', 'N', 'O', 'P', 'Q', 'R', //
            'S', 'T', 'U', 'V', 'W', 'X', //
            'Y', 'Z', '#', '@' //
    };

    protected final static byte[] KEY_EXISTS_MARKER = { '1' };

    static class Runner implements Runnable {
        private byte[] key = null;

        public Runner(byte[] key) {
            this.key = key;
        }

        @Override
        public void run() {
            if (!shuttingDown) {
                try {
                    cluster.set(key, KEY_EXISTS_MARKER);
                    cluster.expire(key, 60 * 60 * 48); // for test purposes, only keep around for 2 days
                }
                catch (JedisClusterMaxRedirectionsException e) {
                    System.err.println(
                            (new Date()).toString() + "\tIGNORING\t" + e + "\t" + "For key " + new String(key));
                }
                catch (Exception e) {
                    System.err.println((new Date()).toString() + "\t" + e + "\t" + "For key " + new String(key));
                    e.printStackTrace();
                    System.exit(1);
                }
            }
        }
    }

    public static void main(String[] args) {
        SecureRandom random = new SecureRandom();
        DecimalFormat decimal = new DecimalFormat("#,##0");
        final byte[] randomBytes = new byte[8];

        connectToCluster();

        Runtime.getRuntime().addShutdownHook(shutdown);

        System.out.println((new Date()) + " Starting test");

        for (int i = 0; i < 1000000000; i++) {
            random.nextBytes(randomBytes);
            final byte[] key = new byte[8];
            for (int j = 0; j < randomBytes.length; j++)
                key[j] = (byte) (CHARACTERS[((randomBytes[j] & 0xFF)) % CHARACTERS.length] & 0xFF);

            try {
                if (shuttingDown) {
                    System.err.println((new Date()).toString() + "\t" + "Main loop terminating due to shutdown");
                    break;
                }

                if (i % 1000000 == 0)
                    System.out.println((new Date()).toString() + "\t" + decimal.format(i));

                try {
                    executor.submit(new Runner(key));
                }
                catch (Exception e) {
                    System.err.println((new Date()).toString() + "\t" + e);
                }
            }
            catch (Exception e) {
                System.err.println("Failed to set key " + new String(key) + " -- " + e);
            }
        }

        if (!shuttingDown) {
            System.out.println((new Date()) + " Done");
            System.exit(0);
        }
    }
}

Answer 1

几乎每个内存管理器都会对您分配的每个对象产生内部开销，只是为了跟踪对象。例如:当您调用 free() 时，内存管理器可能需要有关该对象的一些信息来确定它属于哪个内存池/页面。小对象可能会落入一个池中，并使用与大对象不同的分配机制。

与 Redis sds.c/sds.h 的工作方式非常相似，堆管理器通常也会将它自己的结构添加到每个 malloc() 的对象中。

如果您的堆中每个对象有 16 字节的开销，那么将其添加到每个 10KB malloc() 将是难以察觉的开销。但是，如果您在 Redis 中谈论 8 字节键，那么为每个 8 字节键添加 16 字节的开销将超出键本身的内存。

你可以在这里找到更多关于 malloc block 和 fastbins 的信息: http://iarchsys.com/?p=764

对这种开销进行快速而粗略的检查是将 key 从 8 个字节增加到 16 个字节。虽然您将 key 使用的内存大小加倍，但您可能不会看到 key 消耗的内存增加一倍Redis进程。