[Zouxxyy]

Purpose

Subtask of #4471

Add in Variant

Object variantGet(String path, DataType dataType, VariantCastArgs castArgs);

Add in PaimonShreddingUtils

/**
 * Represent a single field in a variant struct, that is a single requested field that the scan
 * should produce by extracting from the variant column.
 */
public static class FieldToExtract {

    private final SchemaPathSegment[] path;
    private final BaseVariantReader reader;
}

/** Assemble a variant struct, in which each field is extracted from the variant value. */
public static InternalRow assembleVariantStruct(
        InternalRow inputRow, VariantSchema schema, FieldToExtract[] fields)
）

Usage

VariantCastArgs castArgs = new VariantCastArgs(true, ZoneOffset.UTC);

// shreddedType: ROW<a INT, b STRING>
RowType shreddedType =
        RowType.of(
                new DataType[] {DataTypes.INT(), DataTypes.STRING()},
                new String[] {"a", "b"});
RowType shreddingSchema = variantShreddingSchema(shreddedType);
VariantSchema variantSchema = buildVariantSchema(shreddingSchema);

// fieldsToExtract: $.a : int, $.b : string
FieldToExtract f1 = buildFieldsToExtract(DataTypes.INT(), "$.a", castArgs, variantSchema);
FieldToExtract f2 = buildFieldsToExtract(DataTypes.STRING(), "$.b", castArgs, variantSchema);
FieldToExtract[] fields = new FieldToExtract[] {f1, f2};

GenericVariant v = GenericVariant.fromJson("{\"a\": 1, \"b\": \"hello\"}");
assertThat(assembleVariantStruct(castShredded(v, variantSchema), variantSchema, fields))
        .isEqualTo(GenericRow.of(1, BinaryString.fromString("hello")));

Tests

API and Format

Documentation

[JingsongLi]

+1