aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Util.java
static List<S3ObjectSummary> listObjectsLexicographically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {
    // Incrementally scan objects after the marker (s3Offset).
    List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
    listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
    listObjectsRequest.setMaxKeys(BATCH_SIZE);
    if (s3Offset.getKey() != null) {
      listObjectsRequest.setMarker(s3Offset.getKey());
    }

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    while (true) {
      for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
        String fullPrefix = s.getKey();
        String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
        if (!remainingPrefix.isEmpty()) {
          if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
            list.add(s);
          }
          // We've got enough objects.
          if (list.size() == fetchSize) {
            return list;
          }
        }
      }
      // Listing is complete. No more objects to be listed.
      if (!objectListing.isTruncated()) {
        break;
      }
      objectListing = s3Client.listNextBatchOfObjects(objectListing);
    }

    return list;
  }
static List<S3ObjectSummary> listObjectsLexicographically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {
    // Incrementally scan objects after the marker (s3Offset).
    List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
    listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
    listObjectsRequest.setMaxKeys(BATCH_SIZE);
    if (s3Offset.getKey() != null) {
      listObjectsRequest.setMarker(s3Offset.getKey());
    }

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    while (true) {
      for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
        String fullPrefix = s.getKey();
        String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
        if (!remainingPrefix.isEmpty()) {
          if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
            list.add(s);
          }
          // We've got enough objects.
          if (list.size() == fetchSize) {
            return list;
          }
        }
      }
      // Listing is complete. No more objects to be listed.
      if (!objectListing.isTruncated()) {
        break;
      }
      objectListing = s3Client.listNextBatchOfObjects(objectListing);
    }

    return list;
  }
static List < S3ObjectSummary > listObjectsLexicographically ( AmazonS3 s3Client , S3ConfigBean s3ConfigBean , AntPathMatcher pathMatcher , S3Offset s3Offset , int fetchSize ) { // Incrementally scan objects after the marker (s3Offset). List < S3ObjectSummary > list = new ArrayList <> ( fetchSize ) ; ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) ; listObjectsRequest . setBucketName ( s3ConfigBean . s3Config . bucket ) ; listObjectsRequest . setPrefix ( s3ConfigBean . s3Config . commonPrefix ) ; listObjectsRequest . setMaxKeys ( BATCH_SIZE ) ; if ( s3Offset . getKey ( ) != null ) { listObjectsRequest . setMarker ( s3Offset . getKey ( ) ) ; } ObjectListing objectListing = s3Client . listObjects ( listObjectsRequest ) ; while ( true ) { for ( S3ObjectSummary s : objectListing . getObjectSummaries ( ) ) { String fullPrefix = s . getKey ( ) ; String remainingPrefix = fullPrefix . substring ( s3ConfigBean . s3Config . commonPrefix . length ( ) , fullPrefix . length ( ) ) ; if ( ! remainingPrefix . isEmpty ( ) ) { if ( pathMatcher . match ( s3ConfigBean . s3FileConfig . prefixPattern , remainingPrefix ) ) { list . add ( s ) ; } // We've got enough objects. if ( list . size ( ) == fetchSize ) { return list ; } } } // Listing is complete. No more objects to be listed. if ( ! objectListing . isTruncated ( ) ) { break ; } objectListing = s3Client . listNextBatchOfObjects ( objectListing ) ; } return list ; }
Lists objects from AmazonS3 in lexicographical order

@param s3Client
@param s3ConfigBean
@param pathMatcher glob patterns to match file name against
@param s3Offset current offset which provides the key name of the previous object
@param fetchSize number of objects to fetch in one go
@return
@throws AmazonClientException
Lists objects from AmazonS3 in lexicographical order