grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java
private static boolean updateFilePartition(final boolean isGrib1, final FeatureCollectionConfig config,
                                             final CollectionUpdateType updateType, boolean isTop,
                                             final Logger logger, Path dirPath) throws IOException {
    long start = System.currentTimeMillis();
    final Formatter errlog = new Formatter();
    CollectionSpecParser specp = config.getCollectionSpecParser(errlog);

    try (FilePartition partition = new FilePartition(config.collectionName, dirPath, isTop, config.olderThan, logger)) {
      partition.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
      if (specp.getFilter() != null)
        partition.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));

      logger.debug("GribCdmIndex.updateFilePartition %s %s%n", partition.getCollectionName(), updateType);
      if (!isUpdateNeeded(partition.getIndexFilename(NCX_SUFFIX), updateType, (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger))
        return false;

      final AtomicBoolean anyChange = new AtomicBoolean(false); // just need a mutable boolean we can declare final

      // redo the children here
      if (updateType != CollectionUpdateType.testIndexOnly) {   // skip children on testIndexOnly
        partition.iterateOverMFileCollection(mfile -> {
          MCollection part = new CollectionSingleFile(mfile, logger);
          part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);

          try {
            boolean changed = updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.file, logger, errlog);
            if (changed) anyChange.set(true);

          } catch (IllegalStateException t) {
            logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage());
            partition.removePartition(part); // keep on truckin; can happen if directory is empty

          } catch (Throwable t) {
            logger.error("Error making partition " + part.getRoot(), t);
            partition.removePartition(part);
          }
        });
      }

      // LOOK what if theres only one file?

      try {
        // redo partition index if needed, will detect if children have changed
        boolean recreated = updatePartition(isGrib1, partition, updateType, logger, errlog);

        long took = System.currentTimeMillis() - start;
        if (recreated) logger.info("RewriteFilePartition {} took {} msecs", partition.getCollectionName(), took);
        return recreated;

      } catch (IllegalStateException t) {
        logger.warn("Error making partition {} '{}'", partition.getRoot(), t.getMessage());
        return false;

      } catch (Throwable t) {
        logger.error("Error making partition " + partition.getRoot(), t);
        return false;
      }
    }
  }
private static boolean updateFilePartition(final boolean isGrib1, final FeatureCollectionConfig config,
                                             final CollectionUpdateType updateType, boolean isTop,
                                             final Logger logger, Path dirPath) throws IOException {
    long start = System.currentTimeMillis();
    final Formatter errlog = new Formatter();
    CollectionSpecParser specp = config.getCollectionSpecParser(errlog);

    try (FilePartition partition = new FilePartition(config.collectionName, dirPath, isTop, config.olderThan, logger)) {
      partition.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
      if (specp.getFilter() != null)
        partition.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));

      logger.debug("GribCdmIndex.updateFilePartition %s %s%n", partition.getCollectionName(), updateType);
      if (!isUpdateNeeded(partition.getIndexFilename(NCX_SUFFIX), updateType, (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger))
        return false;

      final AtomicBoolean anyChange = new AtomicBoolean(false); // just need a mutable boolean we can declare final

      // redo the children here
      if (updateType != CollectionUpdateType.testIndexOnly) {   // skip children on testIndexOnly
        partition.iterateOverMFileCollection(mfile -> {
          MCollection part = new CollectionSingleFile(mfile, logger);
          part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);

          try {
            boolean changed = updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.file, logger, errlog);
            if (changed) anyChange.set(true);

          } catch (IllegalStateException t) {
            logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage());
            partition.removePartition(part); // keep on truckin; can happen if directory is empty

          } catch (Throwable t) {
            logger.error("Error making partition " + part.getRoot(), t);
            partition.removePartition(part);
          }
        });
      }

      // LOOK what if theres only one file?

      try {
        // redo partition index if needed, will detect if children have changed
        boolean recreated = updatePartition(isGrib1, partition, updateType, logger, errlog);

        long took = System.currentTimeMillis() - start;
        if (recreated) logger.info("RewriteFilePartition {} took {} msecs", partition.getCollectionName(), took);
        return recreated;

      } catch (IllegalStateException t) {
        logger.warn("Error making partition {} '{}'", partition.getRoot(), t.getMessage());
        return false;

      } catch (Throwable t) {
        logger.error("Error making partition " + partition.getRoot(), t);
        return false;
      }
    }
  }
private static boolean updateFilePartition ( final boolean isGrib1 , final FeatureCollectionConfig config , final CollectionUpdateType updateType , boolean isTop , final Logger logger , Path dirPath ) throws IOException { long start = System . currentTimeMillis ( ) ; final Formatter errlog = new Formatter ( ) ; CollectionSpecParser specp = config . getCollectionSpecParser ( errlog ) ; try ( FilePartition partition = new FilePartition ( config . collectionName , dirPath , isTop , config . olderThan , logger ) ) { partition . putAuxInfo ( FeatureCollectionConfig . AUX_CONFIG , config ) ; if ( specp . getFilter ( ) != null ) partition . setStreamFilter ( new StreamFilter ( specp . getFilter ( ) , specp . getFilterOnName ( ) ) ) ; logger . debug ( "GribCdmIndex.updateFilePartition %s %s%n" , partition . getCollectionName ( ) , updateType ) ; if ( ! isUpdateNeeded ( partition . getIndexFilename ( NCX_SUFFIX ) , updateType , ( isGrib1 ? GribCollectionType . Partition1 : GribCollectionType . Partition2 ) , logger ) ) return false ; final AtomicBoolean anyChange = new AtomicBoolean ( false ) ; // just need a mutable boolean we can declare final // redo the children here if ( updateType != CollectionUpdateType . testIndexOnly ) { // skip children on testIndexOnly partition . iterateOverMFileCollection ( mfile -> { MCollection part = new CollectionSingleFile ( mfile , logger ) ; part . putAuxInfo ( FeatureCollectionConfig . AUX_CONFIG , config ) ; try { boolean changed = updateGribCollection ( isGrib1 , part , updateType , FeatureCollectionConfig . PartitionType . file , logger , errlog ) ; if ( changed ) anyChange . set ( true ) ; } catch ( IllegalStateException t ) { logger . warn ( "Error making partition {} '{}'" , part . getRoot ( ) , t . getMessage ( ) ) ; partition . removePartition ( part ) ; // keep on truckin; can happen if directory is empty } catch ( Throwable t ) { logger . error ( "Error making partition " + part . getRoot ( ) , t ) ; partition . removePartition ( part ) ; } } ) ; } // LOOK what if theres only one file? try { // redo partition index if needed, will detect if children have changed boolean recreated = updatePartition ( isGrib1 , partition , updateType , logger , errlog ) ; long took = System . currentTimeMillis ( ) - start ; if ( recreated ) logger . info ( "RewriteFilePartition {} took {} msecs" , partition . getCollectionName ( ) , took ) ; return recreated ; } catch ( IllegalStateException t ) { logger . warn ( "Error making partition {} '{}'" , partition . getRoot ( ) , t . getMessage ( ) ) ; return false ; } catch ( Throwable t ) { logger . error ( "Error making partition " + partition . getRoot ( ) , t ) ; return false ; } } }
File Partition: each File is a collection of Grib records, and the collection of all files in the directory is a PartitionCollection.
Rewrite the PartitionCollection and optionally its children

@param config     FeatureCollectionConfig
@param updateType always, test, nocheck, never
@return true if partition was rewritten, exception on failure
File Partition : each File is a collection of Grib records and the collection of all files in the directory is a PartitionCollection . Rewrite the PartitionCollection and optionally its children