basic-lib/src/main/java/com/streamsets/pipeline/stage/processor/schemagen/generators/AvroSchemaGenerator.java
private Schema simpleSchemaForType(String fieldPath, Record record, Field field) throws OnRecordErrorException {
    switch (field.getType()) {
      // Primitive types
      case BOOLEAN:
        return Schema.create(Schema.Type.BOOLEAN);
      case INTEGER:
        return Schema.create(Schema.Type.INT);
      case LONG:
        return Schema.create(Schema.Type.LONG);
      case FLOAT:
        return Schema.create(Schema.Type.FLOAT);
      case DOUBLE:
        return Schema.create(Schema.Type.DOUBLE);
      case STRING:
        return Schema.create(Schema.Type.STRING);
      case BYTE_ARRAY:
        return Schema.create(Schema.Type.BYTES);

      // Logical types
      case DECIMAL:
        int precision = getDecimalScaleOrPrecision(record, field, getConfig().precisionAttribute, getConfig().defaultPrecision, 1);
        int scale = getDecimalScaleOrPrecision(record, field, getConfig().scaleAttribute, getConfig().defaultScale, 0);
        Schema decimalSchema = Schema.create(Schema.Type.BYTES);
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, new IntNode(precision));
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, new IntNode(scale));
        return decimalSchema;
      case DATE:
        Schema dateSchema = Schema.create(Schema.Type.INT);
        dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
        return dateSchema;
      case TIME:
        Schema timeSchema = Schema.create(Schema.Type.INT);
        timeSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MILLIS);
        return timeSchema;
      case DATETIME:
        Schema dateTimeSchema = Schema.create(Schema.Type.LONG);
        dateTimeSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
        return dateTimeSchema;

      // Complex types
      case LIST:
        // In avro list must be of the same type - which is not true with our records

        // We can't generate the list type from empty list
        if(field.getValueAsList().isEmpty()) {
          throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0006, fieldPath);
        }

        // And all items in the list must have the same schema
        Schema itemSchema = null;
        int index = 0;
        for(Field listItem : field.getValueAsList()) {
          Schema currentListItemSchema = complexSchemaForType(fieldPath + "[" + index + "]", record, listItem);
          if(itemSchema == null) {
            itemSchema = currentListItemSchema;
          } else if (!itemSchema.equals(currentListItemSchema)) {
            throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0005, fieldPath, itemSchema, currentListItemSchema);
          }
          index++;
        }
        return Schema.createArray(itemSchema);
      case MAP:
      case LIST_MAP:
        // In avro maps must have key of string (same as us) and value must be the same for all items - which
        // is different then our records.

        // We can't generate the map value type from null or empty map
        if(field.getValueAsMap() == null || field.getValueAsMap().isEmpty()) {
          throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0008, fieldPath);
        }

        // And all values in the map must be the same
        Schema mapSchema = null;
        for(Map.Entry<String, Field> item : field.getValueAsMap().entrySet()) {
          Schema currentListItemSchema = complexSchemaForType(fieldPath + "/" + item.getKey(), record, item.getValue());
          if(mapSchema == null) {
            mapSchema = currentListItemSchema;
          } else if (!mapSchema.equals(currentListItemSchema)) {
            throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0007, fieldPath, mapSchema, currentListItemSchema);
          }
        }
        return Schema.createMap(mapSchema);

      // Types that does not have direct equivalent in Avro
      case SHORT:
        if(getConfig().avroExpandTypes) {
          return Schema.create(Schema.Type.INT);
        }
        // fall through
      case CHAR:
        if(getConfig().avroExpandTypes) {
          return Schema.create(Schema.Type.STRING);
        }
        // fall through

      // Not supported data types:
      case BYTE:
      case FILE_REF:
      default:
        throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0002, field.getType());
    }
  }
private Schema simpleSchemaForType(String fieldPath, Record record, Field field) throws OnRecordErrorException {
    switch (field.getType()) {
      // Primitive types
      case BOOLEAN:
        return Schema.create(Schema.Type.BOOLEAN);
      case INTEGER:
        return Schema.create(Schema.Type.INT);
      case LONG:
        return Schema.create(Schema.Type.LONG);
      case FLOAT:
        return Schema.create(Schema.Type.FLOAT);
      case DOUBLE:
        return Schema.create(Schema.Type.DOUBLE);
      case STRING:
        return Schema.create(Schema.Type.STRING);
      case BYTE_ARRAY:
        return Schema.create(Schema.Type.BYTES);

      // Logical types
      case DECIMAL:
        int precision = getDecimalScaleOrPrecision(record, field, getConfig().precisionAttribute, getConfig().defaultPrecision, 1);
        int scale = getDecimalScaleOrPrecision(record, field, getConfig().scaleAttribute, getConfig().defaultScale, 0);
        Schema decimalSchema = Schema.create(Schema.Type.BYTES);
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, new IntNode(precision));
        decimalSchema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, new IntNode(scale));
        return decimalSchema;
      case DATE:
        Schema dateSchema = Schema.create(Schema.Type.INT);
        dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
        return dateSchema;
      case TIME:
        Schema timeSchema = Schema.create(Schema.Type.INT);
        timeSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MILLIS);
        return timeSchema;
      case DATETIME:
        Schema dateTimeSchema = Schema.create(Schema.Type.LONG);
        dateTimeSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
        return dateTimeSchema;

      // Complex types
      case LIST:
        // In avro list must be of the same type - which is not true with our records

        // We can't generate the list type from empty list
        if(field.getValueAsList().isEmpty()) {
          throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0006, fieldPath);
        }

        // And all items in the list must have the same schema
        Schema itemSchema = null;
        int index = 0;
        for(Field listItem : field.getValueAsList()) {
          Schema currentListItemSchema = complexSchemaForType(fieldPath + "[" + index + "]", record, listItem);
          if(itemSchema == null) {
            itemSchema = currentListItemSchema;
          } else if (!itemSchema.equals(currentListItemSchema)) {
            throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0005, fieldPath, itemSchema, currentListItemSchema);
          }
          index++;
        }
        return Schema.createArray(itemSchema);
      case MAP:
      case LIST_MAP:
        // In avro maps must have key of string (same as us) and value must be the same for all items - which
        // is different then our records.

        // We can't generate the map value type from null or empty map
        if(field.getValueAsMap() == null || field.getValueAsMap().isEmpty()) {
          throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0008, fieldPath);
        }

        // And all values in the map must be the same
        Schema mapSchema = null;
        for(Map.Entry<String, Field> item : field.getValueAsMap().entrySet()) {
          Schema currentListItemSchema = complexSchemaForType(fieldPath + "/" + item.getKey(), record, item.getValue());
          if(mapSchema == null) {
            mapSchema = currentListItemSchema;
          } else if (!mapSchema.equals(currentListItemSchema)) {
            throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0007, fieldPath, mapSchema, currentListItemSchema);
          }
        }
        return Schema.createMap(mapSchema);

      // Types that does not have direct equivalent in Avro
      case SHORT:
        if(getConfig().avroExpandTypes) {
          return Schema.create(Schema.Type.INT);
        }
        // fall through
      case CHAR:
        if(getConfig().avroExpandTypes) {
          return Schema.create(Schema.Type.STRING);
        }
        // fall through

      // Not supported data types:
      case BYTE:
      case FILE_REF:
      default:
        throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0002, field.getType());
    }
  }
private Schema simpleSchemaForType ( String fieldPath , Record record , Field field ) throws OnRecordErrorException { switch ( field . getType ( ) ) { // Primitive types case BOOLEAN : return Schema . create ( Schema . Type . BOOLEAN ) ; case INTEGER : return Schema . create ( Schema . Type . INT ) ; case LONG : return Schema . create ( Schema . Type . LONG ) ; case FLOAT : return Schema . create ( Schema . Type . FLOAT ) ; case DOUBLE : return Schema . create ( Schema . Type . DOUBLE ) ; case STRING : return Schema . create ( Schema . Type . STRING ) ; case BYTE_ARRAY : return Schema . create ( Schema . Type . BYTES ) ; // Logical types case DECIMAL : int precision = getDecimalScaleOrPrecision ( record , field , getConfig ( ) . precisionAttribute , getConfig ( ) . defaultPrecision , 1 ) ; int scale = getDecimalScaleOrPrecision ( record , field , getConfig ( ) . scaleAttribute , getConfig ( ) . defaultScale , 0 ) ; Schema decimalSchema = Schema . create ( Schema . Type . BYTES ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_DECIMAL ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE_ATTR_PRECISION , new IntNode ( precision ) ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE_ATTR_SCALE , new IntNode ( scale ) ) ; return decimalSchema ; case DATE : Schema dateSchema = Schema . create ( Schema . Type . INT ) ; dateSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_DATE ) ; return dateSchema ; case TIME : Schema timeSchema = Schema . create ( Schema . Type . INT ) ; timeSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_TIME_MILLIS ) ; return timeSchema ; case DATETIME : Schema dateTimeSchema = Schema . create ( Schema . Type . LONG ) ; dateTimeSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_TIMESTAMP_MILLIS ) ; return dateTimeSchema ; // Complex types case LIST : // In avro list must be of the same type - which is not true with our records // We can't generate the list type from empty list if ( field . getValueAsList ( ) . isEmpty ( ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0006 , fieldPath ) ; } // And all items in the list must have the same schema Schema itemSchema = null ; int index = 0 ; for ( Field listItem : field . getValueAsList ( ) ) { Schema currentListItemSchema = complexSchemaForType ( fieldPath + "[" + index + "]" , record , listItem ) ; if ( itemSchema == null ) { itemSchema = currentListItemSchema ; } else if ( ! itemSchema . equals ( currentListItemSchema ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0005 , fieldPath , itemSchema , currentListItemSchema ) ; } index ++ ; } return Schema . createArray ( itemSchema ) ; case MAP : case LIST_MAP : // In avro maps must have key of string (same as us) and value must be the same for all items - which // is different then our records. // We can't generate the map value type from null or empty map if ( field . getValueAsMap ( ) == null || field . getValueAsMap ( ) . isEmpty ( ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0008 , fieldPath ) ; } // And all values in the map must be the same Schema mapSchema = null ; for ( Map . Entry < String , Field > item : field . getValueAsMap ( ) . entrySet ( ) ) { Schema currentListItemSchema = complexSchemaForType ( fieldPath + "/" + item . getKey ( ) , record , item . getValue ( ) ) ; if ( mapSchema == null ) { mapSchema = currentListItemSchema ; } else if ( ! mapSchema . equals ( currentListItemSchema ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0007 , fieldPath , mapSchema , currentListItemSchema ) ; } } return Schema . createMap ( mapSchema ) ; // Types that does not have direct equivalent in Avro case SHORT : if ( getConfig ( ) . avroExpandTypes ) { return Schema . create ( Schema . Type . INT ) ; } // fall through case CHAR : if ( getConfig ( ) . avroExpandTypes ) { return Schema . create ( Schema . Type . STRING ) ; } // fall through // Not supported data types: case BYTE : case FILE_REF : default : throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0002 , field . getType ( ) ) ; } }
Generate simple schema for given field - it will never contain union nor a default value. Particularly useful
for generating Schema.Field as this will not convert simple "string" to "{type:string, defaultValue:something}"
which is hard to undo.
Generate simple schema for given field - it will never contain union nor a default value . Particularly useful for generating Schema . Field as this will not convert simple string to { type : string defaultValue : something } which is hard to undo .