grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java
boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, List<Group> groups, List<MFile> files,
                            GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
    Grib2Record first = null; // take global metadata from here
    boolean deleteOnClose = false;

    if (idxFile.exists()) {
      RandomAccessFile.eject(idxFile.getPath());
      if (!idxFile.delete()) {
        logger.error("gc2 cant delete index file {}", idxFile.getPath());
      }
    }
    logger.debug(" createIndex for {}", idxFile.getPath());

    try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
      //// header message
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.write(MAGIC_START.getBytes(CDM.utf8Charset));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;

      Set<Integer> allFileSet = new HashSet<>();
      for (Group g : groups) {
        g.fileSet = new HashSet<>();
        for (Grib2CollectionBuilder.VariableBag vb : g.gribVars) {
          if (first == null) first = vb.first;
          GribCollectionProto.SparseArray vr = writeSparseArray(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.coordND.getSparseArray().countNotMissing();
        }
        allFileSet.addAll(g.fileSet);
      }

      if (logger.isDebugEnabled()) {
        long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
        logger.debug("  write RecordMaps: bytes = {} record = {} bytesPerRecord={}", countBytes, countRecords, bytesPerRecord);
      }

      if (first == null) {
        deleteOnClose = true;
        throw new IOException("GribCollection " + name + " has no records");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      /*
      message GribCollection {
        string name = 1;         // must be unique - index filename is name.ncx
        string topDir = 2;       // MFile, Partition filenames are reletive to this
        repeated MFile mfiles = 3;        // list of grib MFiles
        repeated Dataset dataset = 4;
        repeated Gds gds = 5;             // unique Gds, shared amongst datasets
        Coord masterRuntime = 6;  // list of runtimes in this GC

        int32 center = 7;      // these 4 fields are to get a GribCustomizer
        int32 subcenter = 8;
        int32 master = 9;
        int32 local = 10;       // grib1 table Version

        int32 genProcessType = 11;
        int32 genProcessId = 12;
        int32 backProcessId = 13;
        int32 version = 14;     // >= 3 for proto3 (5.0+)

        // repeated Parameter params = 20;      // not used
        FcConfig config = 21;

        // extensions
        repeated Partition partitions = 100;
        bool isPartitionOfPartitions = 101;
        repeated uint32 run2part = 102 [packed=true];  // masterRuntime index to partition index
      }
       */
      GribCollectionProto.GribCollection.Builder indexBuilder = GribCollectionProto.GribCollection.newBuilder();
      indexBuilder.setName(name);
      indexBuilder.setTopDir(dcm.getRoot());
      indexBuilder.setVersion(currentVersion);

      // directory and mfile list
      File directory = new File(dcm.getRoot());
      List<GcMFile> gcmfiles = GcMFile.makeFiles(directory, files, allFileSet);
      for (GcMFile gcmfile : gcmfiles) {
        GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder();
        b.setFilename(gcmfile.getName());
        b.setLastModified(gcmfile.getLastModified());
        b.setLength(gcmfile.getLength());
        b.setIndex(gcmfile.index);
        indexBuilder.addMfiles(b.build());
      }

      indexBuilder.setMasterRuntime(writeCoordProto(masterRuntime));

       //gds
      for (Object go : groups) {
        Group g = (Group) go;
        indexBuilder.addGds(writeGdsProto(g.gdss.getRawBytes(), -1));
      }

      // the GC dataset
      indexBuilder.addDataset(writeDatasetProto(type, groups));

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      indexBuilder.setStartTime(dateRange.getStart().getMillis());
      indexBuilder.setEndTime(dateRange.getEnd().getMillis());

      GribCollectionProto.GribCollection index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp
      logger.debug("  write GribCollectionIndex= {} bytes", b.length);

    } finally {
      // remove it on failure
      if (deleteOnClose && !idxFile.delete())
        logger.error(" gc2 cant deleteOnClose index file {}", idxFile.getPath());
    }

    return true;
  }
boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, List<Group> groups, List<MFile> files,
                            GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
    Grib2Record first = null; // take global metadata from here
    boolean deleteOnClose = false;

    if (idxFile.exists()) {
      RandomAccessFile.eject(idxFile.getPath());
      if (!idxFile.delete()) {
        logger.error("gc2 cant delete index file {}", idxFile.getPath());
      }
    }
    logger.debug(" createIndex for {}", idxFile.getPath());

    try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
      //// header message
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.write(MAGIC_START.getBytes(CDM.utf8Charset));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;

      Set<Integer> allFileSet = new HashSet<>();
      for (Group g : groups) {
        g.fileSet = new HashSet<>();
        for (Grib2CollectionBuilder.VariableBag vb : g.gribVars) {
          if (first == null) first = vb.first;
          GribCollectionProto.SparseArray vr = writeSparseArray(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.coordND.getSparseArray().countNotMissing();
        }
        allFileSet.addAll(g.fileSet);
      }

      if (logger.isDebugEnabled()) {
        long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
        logger.debug("  write RecordMaps: bytes = {} record = {} bytesPerRecord={}", countBytes, countRecords, bytesPerRecord);
      }

      if (first == null) {
        deleteOnClose = true;
        throw new IOException("GribCollection " + name + " has no records");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      /*
      message GribCollection {
        string name = 1;         // must be unique - index filename is name.ncx
        string topDir = 2;       // MFile, Partition filenames are reletive to this
        repeated MFile mfiles = 3;        // list of grib MFiles
        repeated Dataset dataset = 4;
        repeated Gds gds = 5;             // unique Gds, shared amongst datasets
        Coord masterRuntime = 6;  // list of runtimes in this GC

        int32 center = 7;      // these 4 fields are to get a GribCustomizer
        int32 subcenter = 8;
        int32 master = 9;
        int32 local = 10;       // grib1 table Version

        int32 genProcessType = 11;
        int32 genProcessId = 12;
        int32 backProcessId = 13;
        int32 version = 14;     // >= 3 for proto3 (5.0+)

        // repeated Parameter params = 20;      // not used
        FcConfig config = 21;

        // extensions
        repeated Partition partitions = 100;
        bool isPartitionOfPartitions = 101;
        repeated uint32 run2part = 102 [packed=true];  // masterRuntime index to partition index
      }
       */
      GribCollectionProto.GribCollection.Builder indexBuilder = GribCollectionProto.GribCollection.newBuilder();
      indexBuilder.setName(name);
      indexBuilder.setTopDir(dcm.getRoot());
      indexBuilder.setVersion(currentVersion);

      // directory and mfile list
      File directory = new File(dcm.getRoot());
      List<GcMFile> gcmfiles = GcMFile.makeFiles(directory, files, allFileSet);
      for (GcMFile gcmfile : gcmfiles) {
        GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder();
        b.setFilename(gcmfile.getName());
        b.setLastModified(gcmfile.getLastModified());
        b.setLength(gcmfile.getLength());
        b.setIndex(gcmfile.index);
        indexBuilder.addMfiles(b.build());
      }

      indexBuilder.setMasterRuntime(writeCoordProto(masterRuntime));

       //gds
      for (Object go : groups) {
        Group g = (Group) go;
        indexBuilder.addGds(writeGdsProto(g.gdss.getRawBytes(), -1));
      }

      // the GC dataset
      indexBuilder.addDataset(writeDatasetProto(type, groups));

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      indexBuilder.setStartTime(dateRange.getStart().getMillis());
      indexBuilder.setEndTime(dateRange.getEnd().getMillis());

      GribCollectionProto.GribCollection index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp
      logger.debug("  write GribCollectionIndex= {} bytes", b.length);

    } finally {
      // remove it on failure
      if (deleteOnClose && !idxFile.delete())
        logger.error(" gc2 cant deleteOnClose index file {}", idxFile.getPath());
    }

    return true;
  }
boolean writeIndex ( String name , File idxFile , CoordinateRuntime masterRuntime , List < Group > groups , List < MFile > files , GribCollectionImmutable . Type type , CalendarDateRange dateRange ) throws IOException { Grib2Record first = null ; // take global metadata from here boolean deleteOnClose = false ; if ( idxFile . exists ( ) ) { RandomAccessFile . eject ( idxFile . getPath ( ) ) ; if ( ! idxFile . delete ( ) ) { logger . error ( "gc2 cant delete index file {}" , idxFile . getPath ( ) ) ; } } logger . debug ( " createIndex for {}" , idxFile . getPath ( ) ) ; try ( RandomAccessFile raf = new RandomAccessFile ( idxFile . getPath ( ) , "rw" ) ) { //// header message raf . order ( RandomAccessFile . BIG_ENDIAN ) ; raf . write ( MAGIC_START . getBytes ( CDM . utf8Charset ) ) ; raf . writeInt ( version ) ; long lenPos = raf . getFilePointer ( ) ; raf . writeLong ( 0 ) ; // save space to write the length of the record section long countBytes = 0 ; int countRecords = 0 ; Set < Integer > allFileSet = new HashSet <> ( ) ; for ( Group g : groups ) { g . fileSet = new HashSet <> ( ) ; for ( Grib2CollectionBuilder . VariableBag vb : g . gribVars ) { if ( first == null ) first = vb . first ; GribCollectionProto . SparseArray vr = writeSparseArray ( vb , g . fileSet ) ; byte [ ] b = vr . toByteArray ( ) ; vb . pos = raf . getFilePointer ( ) ; vb . length = b . length ; raf . write ( b ) ; countBytes += b . length ; countRecords += vb . coordND . getSparseArray ( ) . countNotMissing ( ) ; } allFileSet . addAll ( g . fileSet ) ; } if ( logger . isDebugEnabled ( ) ) { long bytesPerRecord = countBytes / ( ( countRecords == 0 ) ? 1 : countRecords ) ; logger . debug ( "  write RecordMaps: bytes = {} record = {} bytesPerRecord={}" , countBytes , countRecords , bytesPerRecord ) ; } if ( first == null ) { deleteOnClose = true ; throw new IOException ( "GribCollection " + name + " has no records" ) ; } long pos = raf . getFilePointer ( ) ; raf . seek ( lenPos ) ; raf . writeLong ( countBytes ) ; raf . seek ( pos ) ; // back to the output. /*
      message GribCollection {
        string name = 1;         // must be unique - index filename is name.ncx
        string topDir = 2;       // MFile, Partition filenames are reletive to this
        repeated MFile mfiles = 3;        // list of grib MFiles
        repeated Dataset dataset = 4;
        repeated Gds gds = 5;             // unique Gds, shared amongst datasets
        Coord masterRuntime = 6;  // list of runtimes in this GC

        int32 center = 7;      // these 4 fields are to get a GribCustomizer
        int32 subcenter = 8;
        int32 master = 9;
        int32 local = 10;       // grib1 table Version

        int32 genProcessType = 11;
        int32 genProcessId = 12;
        int32 backProcessId = 13;
        int32 version = 14;     // >= 3 for proto3 (5.0+)

        // repeated Parameter params = 20;      // not used
        FcConfig config = 21;

        // extensions
        repeated Partition partitions = 100;
        bool isPartitionOfPartitions = 101;
        repeated uint32 run2part = 102 [packed=true];  // masterRuntime index to partition index
      }
       */ GribCollectionProto . GribCollection . Builder indexBuilder = GribCollectionProto . GribCollection . newBuilder ( ) ; indexBuilder . setName ( name ) ; indexBuilder . setTopDir ( dcm . getRoot ( ) ) ; indexBuilder . setVersion ( currentVersion ) ; // directory and mfile list File directory = new File ( dcm . getRoot ( ) ) ; List < GcMFile > gcmfiles = GcMFile . makeFiles ( directory , files , allFileSet ) ; for ( GcMFile gcmfile : gcmfiles ) { GribCollectionProto . MFile . Builder b = GribCollectionProto . MFile . newBuilder ( ) ; b . setFilename ( gcmfile . getName ( ) ) ; b . setLastModified ( gcmfile . getLastModified ( ) ) ; b . setLength ( gcmfile . getLength ( ) ) ; b . setIndex ( gcmfile . index ) ; indexBuilder . addMfiles ( b . build ( ) ) ; } indexBuilder . setMasterRuntime ( writeCoordProto ( masterRuntime ) ) ; //gds for ( Object go : groups ) { Group g = ( Group ) go ; indexBuilder . addGds ( writeGdsProto ( g . gdss . getRawBytes ( ) , - 1 ) ) ; } // the GC dataset indexBuilder . addDataset ( writeDatasetProto ( type , groups ) ) ; // what about just storing first ?? Grib2SectionIdentification ids = first . getId ( ) ; indexBuilder . setCenter ( ids . getCenter_id ( ) ) ; indexBuilder . setSubcenter ( ids . getSubcenter_id ( ) ) ; indexBuilder . setMaster ( ids . getMaster_table_version ( ) ) ; indexBuilder . setLocal ( ids . getLocal_table_version ( ) ) ; Grib2Pds pds = first . getPDS ( ) ; indexBuilder . setGenProcessType ( pds . getGenProcessType ( ) ) ; indexBuilder . setGenProcessId ( pds . getGenProcessId ( ) ) ; indexBuilder . setBackProcessId ( pds . getBackProcessId ( ) ) ; indexBuilder . setStartTime ( dateRange . getStart ( ) . getMillis ( ) ) ; indexBuilder . setEndTime ( dateRange . getEnd ( ) . getMillis ( ) ) ; GribCollectionProto . GribCollection index = indexBuilder . build ( ) ; byte [ ] b = index . toByteArray ( ) ; NcStream . writeVInt ( raf , b . length ) ; // message size raf . write ( b ) ; // message  - all in one gulp logger . debug ( "  write GribCollectionIndex= {} bytes" , b . length ) ; } finally { // remove it on failure if ( deleteOnClose && ! idxFile . delete ( ) ) logger . error ( " gc2 cant deleteOnClose index file {}" , idxFile . getPath ( ) ) ; } return true ; }
/*
MAGIC_START
version
sizeRecords
SparseArray's (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
/ * MAGIC_START version sizeRecords SparseArray s ( sizeRecords bytes ) sizeIndex GribCollectionIndex ( sizeIndex bytes )