container/src/main/java/com/streamsets/datacollector/bundles/SupportBundleManager.java
public void uploadNewBundleFromInstances(List<BundleContentGenerator> generators, BundleType bundleType) throws IOException {
    // Generate bundle
    SupportBundle bundle = generateNewBundleFromInstances(generators, bundleType);

    boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED);
    String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS);
    String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET);
    String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET);
    int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE);

    if(!enabled) {
      throw new IOException("Uploading support bundles was disabled by administrator.");
    }

    AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey));
    AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration());
    s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
    s3Client.setRegion(Region.getRegion(Regions.US_WEST_2));

    // Object Metadata
    ObjectMetadata s3Metadata = new ObjectMetadata();
    for(Map.Entry<Object, Object> entry: getMetadata(bundleType).entrySet()) {
      s3Metadata.addUserMetadata((String)entry.getKey(), (String)entry.getValue());
    }

    List<PartETag> partETags;
    InitiateMultipartUploadResult initResponse = null;
    try {
      // Uploading part by part
      LOG.info("Initiating multi-part support bundle upload");
      partETags = new ArrayList<>();
      InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, bundle.getBundleKey());
      initRequest.setObjectMetadata(s3Metadata);
      initResponse = s3Client.initiateMultipartUpload(initRequest);
    } catch (AmazonClientException e) {
      LOG.error("Support bundle upload failed: ", e);
      throw new IOException("Support bundle upload failed", e);
    }

    try {
      byte[] buffer = new byte[bufferSize];
      int partId = 1;
      int size = -1;
      while ((size = readFully(bundle.getInputStream(), buffer)) != -1) {
        LOG.debug("Uploading part {} of size {}", partId, size);
        UploadPartRequest uploadRequest = new UploadPartRequest()
          .withBucketName(bucket)
          .withKey(bundle.getBundleKey())
          .withUploadId(initResponse.getUploadId())
          .withPartNumber(partId++)
          .withInputStream(new ByteArrayInputStream(buffer))
          .withPartSize(size);

        partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
      }

      CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
        bucket,
        bundle.getBundleKey(),
        initResponse.getUploadId(),
        partETags
      );

      s3Client.completeMultipartUpload(compRequest);
      LOG.info("Support bundle upload finished");
    } catch (Exception e) {
      LOG.error("Support bundle upload failed", e);
      s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(
        bucket,
        bundle.getBundleKey(),
        initResponse.getUploadId())
      );

      throw new IOException("Can't upload support bundle", e);
    } finally {
      // Close the client
      s3Client.shutdown();
    }
  }
public void uploadNewBundleFromInstances(List<BundleContentGenerator> generators, BundleType bundleType) throws IOException {
    // Generate bundle
    SupportBundle bundle = generateNewBundleFromInstances(generators, bundleType);

    boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED);
    String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS);
    String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET);
    String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET);
    int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE);

    if(!enabled) {
      throw new IOException("Uploading support bundles was disabled by administrator.");
    }

    AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey));
    AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration());
    s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
    s3Client.setRegion(Region.getRegion(Regions.US_WEST_2));

    // Object Metadata
    ObjectMetadata s3Metadata = new ObjectMetadata();
    for(Map.Entry<Object, Object> entry: getMetadata(bundleType).entrySet()) {
      s3Metadata.addUserMetadata((String)entry.getKey(), (String)entry.getValue());
    }

    List<PartETag> partETags;
    InitiateMultipartUploadResult initResponse = null;
    try {
      // Uploading part by part
      LOG.info("Initiating multi-part support bundle upload");
      partETags = new ArrayList<>();
      InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, bundle.getBundleKey());
      initRequest.setObjectMetadata(s3Metadata);
      initResponse = s3Client.initiateMultipartUpload(initRequest);
    } catch (AmazonClientException e) {
      LOG.error("Support bundle upload failed: ", e);
      throw new IOException("Support bundle upload failed", e);
    }

    try {
      byte[] buffer = new byte[bufferSize];
      int partId = 1;
      int size = -1;
      while ((size = readFully(bundle.getInputStream(), buffer)) != -1) {
        LOG.debug("Uploading part {} of size {}", partId, size);
        UploadPartRequest uploadRequest = new UploadPartRequest()
          .withBucketName(bucket)
          .withKey(bundle.getBundleKey())
          .withUploadId(initResponse.getUploadId())
          .withPartNumber(partId++)
          .withInputStream(new ByteArrayInputStream(buffer))
          .withPartSize(size);

        partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
      }

      CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
        bucket,
        bundle.getBundleKey(),
        initResponse.getUploadId(),
        partETags
      );

      s3Client.completeMultipartUpload(compRequest);
      LOG.info("Support bundle upload finished");
    } catch (Exception e) {
      LOG.error("Support bundle upload failed", e);
      s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(
        bucket,
        bundle.getBundleKey(),
        initResponse.getUploadId())
      );

      throw new IOException("Can't upload support bundle", e);
    } finally {
      // Close the client
      s3Client.shutdown();
    }
  }
public void uploadNewBundleFromInstances ( List < BundleContentGenerator > generators , BundleType bundleType ) throws IOException { // Generate bundle SupportBundle bundle = generateNewBundleFromInstances ( generators , bundleType ) ; boolean enabled = configuration . get ( Constants . UPLOAD_ENABLED , Constants . DEFAULT_UPLOAD_ENABLED ) ; String accessKey = configuration . get ( Constants . UPLOAD_ACCESS , Constants . DEFAULT_UPLOAD_ACCESS ) ; String secretKey = configuration . get ( Constants . UPLOAD_SECRET , Constants . DEFAULT_UPLOAD_SECRET ) ; String bucket = configuration . get ( Constants . UPLOAD_BUCKET , Constants . DEFAULT_UPLOAD_BUCKET ) ; int bufferSize = configuration . get ( Constants . UPLOAD_BUFFER_SIZE , Constants . DEFAULT_UPLOAD_BUFFER_SIZE ) ; if ( ! enabled ) { throw new IOException ( "Uploading support bundles was disabled by administrator." ) ; } AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider ( new BasicAWSCredentials ( accessKey , secretKey ) ) ; AmazonS3Client s3Client = new AmazonS3Client ( credentialsProvider , new ClientConfiguration ( ) ) ; s3Client . setS3ClientOptions ( new S3ClientOptions ( ) . withPathStyleAccess ( true ) ) ; s3Client . setRegion ( Region . getRegion ( Regions . US_WEST_2 ) ) ; // Object Metadata ObjectMetadata s3Metadata = new ObjectMetadata ( ) ; for ( Map . Entry < Object , Object > entry : getMetadata ( bundleType ) . entrySet ( ) ) { s3Metadata . addUserMetadata ( ( String ) entry . getKey ( ) , ( String ) entry . getValue ( ) ) ; } List < PartETag > partETags ; InitiateMultipartUploadResult initResponse = null ; try { // Uploading part by part LOG . info ( "Initiating multi-part support bundle upload" ) ; partETags = new ArrayList <> ( ) ; InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) ) ; initRequest . setObjectMetadata ( s3Metadata ) ; initResponse = s3Client . initiateMultipartUpload ( initRequest ) ; } catch ( AmazonClientException e ) { LOG . error ( "Support bundle upload failed: " , e ) ; throw new IOException ( "Support bundle upload failed" , e ) ; } try { byte [ ] buffer = new byte [ bufferSize ] ; int partId = 1 ; int size = - 1 ; while ( ( size = readFully ( bundle . getInputStream ( ) , buffer ) ) != - 1 ) { LOG . debug ( "Uploading part {} of size {}" , partId , size ) ; UploadPartRequest uploadRequest = new UploadPartRequest ( ) . withBucketName ( bucket ) . withKey ( bundle . getBundleKey ( ) ) . withUploadId ( initResponse . getUploadId ( ) ) . withPartNumber ( partId ++ ) . withInputStream ( new ByteArrayInputStream ( buffer ) ) . withPartSize ( size ) ; partETags . add ( s3Client . uploadPart ( uploadRequest ) . getPartETag ( ) ) ; } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) , initResponse . getUploadId ( ) , partETags ) ; s3Client . completeMultipartUpload ( compRequest ) ; LOG . info ( "Support bundle upload finished" ) ; } catch ( Exception e ) { LOG . error ( "Support bundle upload failed" , e ) ; s3Client . abortMultipartUpload ( new AbortMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) , initResponse . getUploadId ( ) ) ) ; throw new IOException ( "Can't upload support bundle" , e ) ; } finally { // Close the client s3Client . shutdown ( ) ; } }
Instead of providing support bundle directly to user, upload it to StreamSets backend services.
Instead of providing support bundle directly to user upload it to StreamSets backend services .