aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Util.java
static List<S3ObjectSummary> listObjectsChronologically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {

    //Algorithm:
    // - Full scan all objects that match the file name pattern and which are later than the file in the offset
    // - Select the oldest "fetchSize" number of files and return them.
    TreeSet<S3ObjectSummary> treeSet = new TreeSet<>((o1, o2) -> {
      int result = o1.getLastModified().compareTo(o2.getLastModified());
      if(result != 0) {
        //same modified time. Use name to sort
        return result;
      }
      return o1.getKey().compareTo(o2.getKey());
    });

    S3Objects s3ObjectSummaries = S3Objects
      .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.commonPrefix);

    // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call
    for (S3ObjectSummary s : s3ObjectSummaries) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        // remainingPrefix can be empty.
        // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
        // then the first objects returned here are:
        // myFolder/mySubFolder
        // myFolder/mySubFolder/myObject
        //
        // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
        // has no data
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix) && isEligible(s, s3Offset)) {
          treeSet.add(s);
        }
        if (treeSet.size() > fetchSize) {
          treeSet.pollLast();
        }
      }
    }

    return new ArrayList<>(treeSet);
  }
static List<S3ObjectSummary> listObjectsChronologically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {

    //Algorithm:
    // - Full scan all objects that match the file name pattern and which are later than the file in the offset
    // - Select the oldest "fetchSize" number of files and return them.
    TreeSet<S3ObjectSummary> treeSet = new TreeSet<>((o1, o2) -> {
      int result = o1.getLastModified().compareTo(o2.getLastModified());
      if(result != 0) {
        //same modified time. Use name to sort
        return result;
      }
      return o1.getKey().compareTo(o2.getKey());
    });

    S3Objects s3ObjectSummaries = S3Objects
      .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.commonPrefix);

    // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call
    for (S3ObjectSummary s : s3ObjectSummaries) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        // remainingPrefix can be empty.
        // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
        // then the first objects returned here are:
        // myFolder/mySubFolder
        // myFolder/mySubFolder/myObject
        //
        // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
        // has no data
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix) && isEligible(s, s3Offset)) {
          treeSet.add(s);
        }
        if (treeSet.size() > fetchSize) {
          treeSet.pollLast();
        }
      }
    }

    return new ArrayList<>(treeSet);
  }
static List < S3ObjectSummary > listObjectsChronologically ( AmazonS3 s3Client , S3ConfigBean s3ConfigBean , AntPathMatcher pathMatcher , S3Offset s3Offset , int fetchSize ) { //Algorithm: // - Full scan all objects that match the file name pattern and which are later than the file in the offset // - Select the oldest "fetchSize" number of files and return them. TreeSet < S3ObjectSummary > treeSet = new TreeSet <> ( ( o1 , o2 ) -> { int result = o1 . getLastModified ( ) . compareTo ( o2 . getLastModified ( ) ) ; if ( result != 0 ) { //same modified time. Use name to sort return result ; } return o1 . getKey ( ) . compareTo ( o2 . getKey ( ) ) ; } ) ; S3Objects s3ObjectSummaries = S3Objects . withPrefix ( s3Client , s3ConfigBean . s3Config . bucket , s3ConfigBean . s3Config . commonPrefix ) ; // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call for ( S3ObjectSummary s : s3ObjectSummaries ) { String fullPrefix = s . getKey ( ) ; String remainingPrefix = fullPrefix . substring ( s3ConfigBean . s3Config . commonPrefix . length ( ) , fullPrefix . length ( ) ) ; if ( ! remainingPrefix . isEmpty ( ) ) { // remainingPrefix can be empty. // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject", // then the first objects returned here are: // myFolder/mySubFolder // myFolder/mySubFolder/myObject // // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it // has no data if ( pathMatcher . match ( s3ConfigBean . s3FileConfig . prefixPattern , remainingPrefix ) && isEligible ( s , s3Offset ) ) { treeSet . add ( s ) ; } if ( treeSet . size ( ) > fetchSize ) { treeSet . pollLast ( ) ; } } } return new ArrayList <> ( treeSet ) ; }
Lists objects from AmazonS3 in chronological order [lexicographical order if 2 files have same timestamp] which are
later than or equal to the timestamp of the previous offset object

@param s3Client
@param s3ConfigBean
@param pathMatcher glob patterns to match file name against
@param s3Offset current offset which provides the timestamp of the previous object
@param fetchSize number of objects to fetch in one go
@return
@throws AmazonClientException
Lists objects from AmazonS3 in chronological order [ lexicographical order if 2 files have same timestamp ] which are later than or equal to the timestamp of the previous offset object