aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Runnable.java
private void handleWholeFileDataFormat(S3ObjectSummary s3ObjectSummary, String recordId) throws StageException {
    S3Object partialS3ObjectForMetadata;
    //partialObject with fetchSize 1 byte.
    //This is mostly used for extracting metadata and such.
    partialS3ObjectForMetadata = AmazonS3Util.getObjectRange(s3Client,
        s3ConfigBean.s3Config.bucket,
        s3ObjectSummary.getKey(),
        1,
        s3ConfigBean.sseConfig.useCustomerSSEKey,
        s3ConfigBean.sseConfig.customerKey,
        s3ConfigBean.sseConfig.customerKeyMd5
    );

    S3FileRef.Builder s3FileRefBuilder = new S3FileRef.Builder().s3Client(s3Client)
                                                                .s3ObjectSummary(s3ObjectSummary)
                                                                .useSSE(s3ConfigBean.sseConfig.useCustomerSSEKey)
                                                                .customerKey(s3ConfigBean.sseConfig.customerKey)
                                                                .customerKeyMd5(s3ConfigBean.sseConfig.customerKeyMd5)
                                                                .bufferSize((int) dataParser.suggestedWholeFileBufferSize())
                                                                .createMetrics(true)
                                                                .totalSizeInBytes(s3ObjectSummary.getSize())
                                                                .rateLimit(dataParser.wholeFileRateLimit());

    if (dataParser.isWholeFileChecksumRequired()) {
      s3FileRefBuilder.verifyChecksum(true).checksumAlgorithm(HashingUtil.HashType.MD5)
                      //128 bit hex encoded md5 checksum.
                      .checksum(partialS3ObjectForMetadata.getObjectMetadata().getETag());
    }
    Map<String, Object> metadata = AmazonS3Util.getMetaData(partialS3ObjectForMetadata);
    metadata.put(S3Constants.BUCKET, s3ObjectSummary.getBucketName());
    metadata.put(S3Constants.OBJECT_KEY, s3ObjectSummary.getKey());
    metadata.put(S3Constants.OWNER, s3ObjectSummary.getOwner());
    metadata.put(S3Constants.SIZE, s3ObjectSummary.getSize());
    metadata.put(HeaderAttributeConstants.FILE_NAME, s3ObjectSummary.getKey());

    metadata.remove(S3Constants.CONTENT_LENGTH);
    parser = dataParser.getParser(recordId, metadata, s3FileRefBuilder.build());
    //Object is assigned so that setHeaders() function can use this to get metadata
    //information about the object
    object = partialS3ObjectForMetadata;
  }
private void handleWholeFileDataFormat(S3ObjectSummary s3ObjectSummary, String recordId) throws StageException {
    S3Object partialS3ObjectForMetadata;
    //partialObject with fetchSize 1 byte.
    //This is mostly used for extracting metadata and such.
    partialS3ObjectForMetadata = AmazonS3Util.getObjectRange(s3Client,
        s3ConfigBean.s3Config.bucket,
        s3ObjectSummary.getKey(),
        1,
        s3ConfigBean.sseConfig.useCustomerSSEKey,
        s3ConfigBean.sseConfig.customerKey,
        s3ConfigBean.sseConfig.customerKeyMd5
    );

    S3FileRef.Builder s3FileRefBuilder = new S3FileRef.Builder().s3Client(s3Client)
                                                                .s3ObjectSummary(s3ObjectSummary)
                                                                .useSSE(s3ConfigBean.sseConfig.useCustomerSSEKey)
                                                                .customerKey(s3ConfigBean.sseConfig.customerKey)
                                                                .customerKeyMd5(s3ConfigBean.sseConfig.customerKeyMd5)
                                                                .bufferSize((int) dataParser.suggestedWholeFileBufferSize())
                                                                .createMetrics(true)
                                                                .totalSizeInBytes(s3ObjectSummary.getSize())
                                                                .rateLimit(dataParser.wholeFileRateLimit());

    if (dataParser.isWholeFileChecksumRequired()) {
      s3FileRefBuilder.verifyChecksum(true).checksumAlgorithm(HashingUtil.HashType.MD5)
                      //128 bit hex encoded md5 checksum.
                      .checksum(partialS3ObjectForMetadata.getObjectMetadata().getETag());
    }
    Map<String, Object> metadata = AmazonS3Util.getMetaData(partialS3ObjectForMetadata);
    metadata.put(S3Constants.BUCKET, s3ObjectSummary.getBucketName());
    metadata.put(S3Constants.OBJECT_KEY, s3ObjectSummary.getKey());
    metadata.put(S3Constants.OWNER, s3ObjectSummary.getOwner());
    metadata.put(S3Constants.SIZE, s3ObjectSummary.getSize());
    metadata.put(HeaderAttributeConstants.FILE_NAME, s3ObjectSummary.getKey());

    metadata.remove(S3Constants.CONTENT_LENGTH);
    parser = dataParser.getParser(recordId, metadata, s3FileRefBuilder.build());
    //Object is assigned so that setHeaders() function can use this to get metadata
    //information about the object
    object = partialS3ObjectForMetadata;
  }
private void handleWholeFileDataFormat ( S3ObjectSummary s3ObjectSummary , String recordId ) throws StageException { S3Object partialS3ObjectForMetadata ; //partialObject with fetchSize 1 byte. //This is mostly used for extracting metadata and such. partialS3ObjectForMetadata = AmazonS3Util . getObjectRange ( s3Client , s3ConfigBean . s3Config . bucket , s3ObjectSummary . getKey ( ) , 1 , s3ConfigBean . sseConfig . useCustomerSSEKey , s3ConfigBean . sseConfig . customerKey , s3ConfigBean . sseConfig . customerKeyMd5 ) ; S3FileRef . Builder s3FileRefBuilder = new S3FileRef . Builder ( ) . s3Client ( s3Client ) . s3ObjectSummary ( s3ObjectSummary ) . useSSE ( s3ConfigBean . sseConfig . useCustomerSSEKey ) . customerKey ( s3ConfigBean . sseConfig . customerKey ) . customerKeyMd5 ( s3ConfigBean . sseConfig . customerKeyMd5 ) . bufferSize ( ( int ) dataParser . suggestedWholeFileBufferSize ( ) ) . createMetrics ( true ) . totalSizeInBytes ( s3ObjectSummary . getSize ( ) ) . rateLimit ( dataParser . wholeFileRateLimit ( ) ) ; if ( dataParser . isWholeFileChecksumRequired ( ) ) { s3FileRefBuilder . verifyChecksum ( true ) . checksumAlgorithm ( HashingUtil . HashType . MD5 ) //128 bit hex encoded md5 checksum. . checksum ( partialS3ObjectForMetadata . getObjectMetadata ( ) . getETag ( ) ) ; } Map < String , Object > metadata = AmazonS3Util . getMetaData ( partialS3ObjectForMetadata ) ; metadata . put ( S3Constants . BUCKET , s3ObjectSummary . getBucketName ( ) ) ; metadata . put ( S3Constants . OBJECT_KEY , s3ObjectSummary . getKey ( ) ) ; metadata . put ( S3Constants . OWNER , s3ObjectSummary . getOwner ( ) ) ; metadata . put ( S3Constants . SIZE , s3ObjectSummary . getSize ( ) ) ; metadata . put ( HeaderAttributeConstants . FILE_NAME , s3ObjectSummary . getKey ( ) ) ; metadata . remove ( S3Constants . CONTENT_LENGTH ) ; parser = dataParser . getParser ( recordId , metadata , s3FileRefBuilder . build ( ) ) ; //Object is assigned so that setHeaders() function can use this to get metadata //information about the object object = partialS3ObjectForMetadata ; }
as the record is just the metadata along with file ref.
as the record is just the metadata along with file ref .