java - 如何在 amazon s3 中列出 key ？

Question

我在 Amazon S3 中有一个桶，它的结构如下:

bucket-name/prefix1/prefix2/prefix3/prefix4/file1.txt
bucket-name/prefix1/prefix2/prefix3/prefix4/prefix5/file2.txt
bucket-name/prefix1/prefix2/prefix3/file3.txt

我只想列出这样的前缀名

prefix1
prefix1/prefix2
prefix1/prefix2/prefix3
prefix1/prefix2/prefix3/prefix4
prefix1/prefix2/prefix3/prefix4/prefix5

我怎样才能列出这些前缀？

我正在检索列表作为

 List<S3ObjectSummary> keyList = new ArrayList<S3ObjectSummary>();
    ObjectListing objects = s3Client.listObjects(bucketName);
    keyList.addAll(objects.getObjectSummaries());
    while (objects.isTruncated()) {
        objects = s3Client.listNextBatchOfObjects(objects);
        keyList.addAll(objects.getObjectSummaries());
    }
    for (String prefix : prefixes) {
        System.out.println(prefix);
    }

它给出的列表是:-

elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0006/container_1475040945730_0006_01_000001/stderr.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0006/container_1475040945730_0006_01_000002/stderr.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0006/container_1475040945730_0006_01_000002/stdout.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0006/container_1475040945730_0006_01_000003/stderr.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0006/container_1475040945730_0006_01_000003/stdout.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0007/container_1475040945730_0007_01_000001/stderr.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0007/container_1475040945730_0007_01_000002/stderr.gz
elasticmapreduce/j-1T09V6RUT8JO/containers/application_1475040945730_0007/container_1475040945730_0007_01_000002/stdout.gz
...

但我想要上面提到的列表。

Answer 1

我得到了我的问题的解决方案。我在这里分享我对上述问题的解决方案:

package com.elegant.amazon;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;

/**
 * A utility class for Amazon S3 Service. It has common methods init. which is
 * used for all common purpose. Amazon S3 client is initialized when class is
 * deployed on the server. for the connection I have used BasicAWSCredentials
 * Class. It sets all the necessary information for accessing Amazon S3 Service
 * like Access Key and Secret Key. It will load all buckets information when
 * it's deployed on the server.
 * 
 * @author Vipin Suman
 * 
 * @since May 5th,2017
 */

@SuppressWarnings("deprecation")
public class AmazonS3Util {

    public static AmazonS3 s3Client = null;

    public static List<Bucket> bucketList = null;

    static {

        try {
            /** Setting Up AWS Credentials For accessing Amazon S3 */
            BasicAWSCredentials awsCredentials = new BasicAWSCredentials(AWSCommonConstants.ACCESS_KEY,
                    AWSCommonConstants.SECRET_KEY);

            /** initializing AmazonS3 Client */
            s3Client = new AmazonS3Client(awsCredentials);

            /** initializing all buckets stored inside Amazon S3 */
            bucketList = s3Client.listBuckets();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Returns List<Bucket> object. It has informations of all buckets. It
     * stores information of all bucket when class is deployed on the server.
     */
    public static List<Bucket> getAllBuckets() {
        return bucketList;
    }

    /**
     * Returns HashMap object for which has prefixList and dataList init. in
     * this method we are getting all common prefixes list and data list for the
     * given bucket.
     * 
     * This method is called when we are giving only bucketName as Parameter.
     * 
     * @param bucketName
     * 
     */
    public static HashMap<String, List<String>> getPrefix(String bucketName) {
        Long startTime = System.currentTimeMillis();
        ListObjectsRequest listObjectsRequest = null;
        ObjectListing objectListing = null;
        HashMap<String, List<String>> prefixData = null;
        try {
            listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter(AWSCommonConstants.FILE_DELIMITER);
            prefixData = prepareDataMap(listObjectsRequest, objectListing);
        } catch (AmazonS3Exception e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println("Total Time Taken : " + (System.currentTimeMillis() - startTime));
        return prefixData;
    }

    /**
     * Returns HashMap object for which has prefixList and dataList init. in
     * this method we are getting all common prefixes list and data list for the
     * given bucket.
     * 
     * This method is called when we give only bucketName and prefix as
     * Parameter.
     * 
     * @param bucketName
     * 
     * @param prefix
     */
    public static HashMap<String, List<String>> getPrefix(String bucketName, String prefix) {
        Long startTime = System.currentTimeMillis();
        ListObjectsRequest listObjectsRequest = null;
        ObjectListing objectListing = null;
        HashMap<String, List<String>> prefixData = null;
        try {
            if (!prefix.endsWith(AWSCommonConstants.FILE_DELIMITER)) {
                prefix += AWSCommonConstants.FILE_DELIMITER;
            }
            listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix).withDelimiter(AWSCommonConstants.FILE_DELIMITER);
            prefixData = prepareDataMap(listObjectsRequest, objectListing);
        } catch (AmazonS3Exception e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println("Total Time Taken : " + (System.currentTimeMillis() - startTime));
        return prefixData;
    }

    /**
     * Returns HashMap object. It prepares the HashMap object for the above
     * methods. It's common methods for getPrefix() methods.
     * 
     * @param ListObjectsRequest
     * 
     * @param ObjectListing
     */

    private static HashMap<String, List<String>> prepareDataMap(ListObjectsRequest listObjectsRequest,
            ObjectListing objectListing) {
        HashMap<String, List<String>> prefixData = null;
        try {
            prefixData = new HashMap<String, List<String>>();
            objectListing = s3Client.listObjects(listObjectsRequest);
            List<String> commonPrefixes = objectListing.getCommonPrefixes();
            if (commonPrefixes != null && commonPrefixes.size() != 0) {
                prefixData.put(AWSCommonConstants.GET_FOLDER, commonPrefixes);
            }
            List<S3ObjectSummary> s3KeyList = new ArrayList<S3ObjectSummary>();
            s3KeyList.addAll(objectListing.getObjectSummaries());
            while (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
                s3KeyList.addAll(objectListing.getObjectSummaries());
            }
            if (s3KeyList != null && s3KeyList.size() != 0) {
                List<String> objectData = new ArrayList<String>();
                for (S3ObjectSummary s3ObjectSummary : s3KeyList) {
                    objectData.add(s3ObjectSummary.getKey());
                }
                prefixData.put(AWSCommonConstants.GET_DATA, objectData);
            }
        } catch (AmazonS3Exception e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return prefixData;
    }
}

为此测试类:

package com.elegant.amazon;

import java.util.HashMap;
import java.util.List;
import java.util.Scanner;

import com.amazonaws.services.s3.model.Bucket;

public class AmazonS3Test {

    @SuppressWarnings("resource")
    public static void main(String[] args) throws InterruptedException {
        boolean yes = true;
        do {
            System.out.println("\n\n");
            System.out.println("----------AMAZON-----------------------S3-------------------------TEST------------");
            System.out.println("                              1 Get Buckets List");
            System.out.println("                              2 Get First Prefix Of Bucket");
            System.out.println("                              3 Get Next Prefix");
            System.out.println("                              4 Exit");
            System.out.println("----------AMAZON-----------------------S3-------------------------TEST------------");
            System.out.println("\n\n");
            int choice = (new Scanner(System.in)).nextInt();
            switch (choice) {
            case 1:
                List<Bucket> buckets = AmazonS3Util.bucketList;
                for (Bucket bucket : buckets) {
                    System.out.println(bucket.getName());
                }
                break;

            case 2:
                System.out.println("Enter Bucket Name");
                String bucketName = (new Scanner(System.in)).nextLine();
                HashMap<String, List<String>> prefixName = AmazonS3Util.getPrefix(bucketName);
                if (prefixName.containsKey(AWSCommonConstants.GET_FOLDER)) {
                    List<String> prefixList = prefixName.get(AWSCommonConstants.GET_FOLDER);
                    for (String prefix : prefixList) {
                        System.out.println(prefix);
                    }
                }
                if (prefixName.containsKey(AWSCommonConstants.GET_DATA)) {
                    List<String> dataList = prefixName.get(AWSCommonConstants.GET_DATA);
                    for (String data : dataList) {
                        System.out.println(data);
                    }
                }
                break;

            case 3:
                System.out.println("Enter Bucket Name");
                String bucketName1 = (new Scanner(System.in)).nextLine();
                System.out.println("Enter Prefix Name");
                String prefix1 = (new Scanner(System.in)).nextLine();
                HashMap<String, List<String>> prefixName1 = AmazonS3Util.getPrefix(bucketName1, prefix1);
                if (prefixName1.containsKey(AWSCommonConstants.GET_FOLDER)) {
                    List<String> prefixList = prefixName1.get(AWSCommonConstants.GET_FOLDER);
                    for (String prefix : prefixList) {
                        System.out.println(prefix);
                    }
                }
                if (prefixName1.containsKey(AWSCommonConstants.GET_DATA)) {
                    List<String> dataList = prefixName1.get(AWSCommonConstants.GET_DATA);
                    for (String data : dataList) {
                        System.out.println(data);
                    }
                }
                break;

            case 4:
                yes = false;
                break;

            default:
                break;
            }
        } while (yes);
    }
}

由此我们可以列出Amazon S3的目录结构