From 4cf67feb722666b4161f8d529134a727a9977d57 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Wed, 10 Jun 2026 11:57:07 +0200 Subject: [PATCH 1/3] feat(core): add Substrait dialect support Add a typed model in io.substrait.dialect for creating and consuming Substrait dialect YAML files, faithful to substrait/text/dialect_schema.yaml (introduced in substrait v0.76.0). The model mirrors the SimpleExtension pattern: a @Value.Enclosing Dialect holder with nested Immutables types and Jackson (de)serialization. The three polymorphic unions (supported_types/relations/expressions) use an enum-tag class per category with typed config fields, and custom (de)serializers that collapse config-free entries to bare enum strings and expand configured ones to objects. Config sub-enums are dialect-local to keep the dialect vocabulary decoupled from the algebra model. Schema validation is test-scope only (networknt json-schema-validator), so the published core jar gains no new runtime dependency. Tests cover a schema-validated round-trip, bare-string collapse, parsing the published spark_dialect.yaml, and the per-section dialect fixtures from the spec repo. --- core/build.gradle.kts | 10 + .../java/io/substrait/dialect/Dialect.java | 552 ++++++++++++++++++ .../substrait/dialect/DialectJsonSupport.java | 81 +++ .../SupportedExpressionDeserializer.java | 46 ++ .../SupportedExpressionSerializer.java | 45 ++ .../SupportedRelationDeserializer.java | 56 ++ .../dialect/SupportedRelationSerializer.java | 54 ++ .../dialect/SupportedTypeDeserializer.java | 51 ++ .../dialect/SupportedTypeSerializer.java | 42 ++ .../DialectBareStringCollapseTest.java | 66 +++ .../dialect/DialectRoundTripTest.java | 167 ++++++ .../io/substrait/dialect/SchemaValidator.java | 42 ++ .../dialect/SparkDialectParseTest.java | 66 +++ .../dialect/SpecDialectFixturesTest.java | 66 +++ 14 files changed, 1344 insertions(+) create mode 100644 core/src/main/java/io/substrait/dialect/Dialect.java create mode 100644 core/src/main/java/io/substrait/dialect/DialectJsonSupport.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java create mode 100644 core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java create mode 100644 core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java create mode 100644 core/src/test/java/io/substrait/dialect/SchemaValidator.java create mode 100644 core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java create mode 100644 core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 5e84490ae..377f6dd98 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -90,6 +90,7 @@ dependencies { testImplementation(libs.guava) testImplementation(libs.bundles.jackson) testImplementation(libs.classgraph) + testImplementation(libs.json.schema.validator) testImplementation(libs.junit.jupiter) testRuntimeOnly(libs.junit.platform.launcher) @@ -243,6 +244,15 @@ tasks.named("processResources") { from("../substrait/extensions") { into("substrait/extensions") } } +tasks.named("processTestResources") { + // Dialect schema, used to validate dialects produced by the Dialect model in tests. + from("../substrait/text") { into("substrait/text") } + // A real-world dialect to exercise parsing against. + from("../spark/spark_dialect.yaml") { into("dialect") } + // Per-section dialect fixtures published by the substrait spec. + from("../substrait/dialects/tests") { into("dialect/tests") } +} + project.configure { module { resourceDirs.addAll( diff --git a/core/src/main/java/io/substrait/dialect/Dialect.java b/core/src/main/java/io/substrait/dialect/Dialect.java new file mode 100644 index 000000000..d773f50c1 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/Dialect.java @@ -0,0 +1,552 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Scanner; +import java.util.regex.Pattern; +import org.immutables.value.Value; + +/** + * Classes used to create and consume Substrait dialect YAML files. + * + *

A dialect describes what a target execution system supports: which types, relations, + * expressions and functions, along with per-feature configuration (join types, set operations, cast + * failure behavior, ...). The model maps the schema published at {@code + * substrait/text/dialect_schema.yaml}. + * + *

Build a dialect with the nested builders and serialize it with {@link + * #toYaml(DialectDocument)}; parse one with {@link #load(String)} and friends. + */ +@Value.Enclosing +public class Dialect { + + // `\A` means beginning of input. Using it as a delimiter in a scanner reads in the whole file. + private static final Pattern READ_WHOLE_FILE = Pattern.compile("\\A"); + + private Dialect() {} + + private static ObjectMapper objectMapper() { + return new ObjectMapper(new YAMLFactory()) + .registerModule(new Jdk8Module()) + .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + // Omit absent Optionals and empty collections so that unset sections are not emitted. + // The custom (de)serializers for the polymorphic unions write their fields explicitly and + // are unaffected by this inclusion setting. + .setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + } + + /** Parse a dialect from YAML content. */ + public static DialectDocument load(String content) { + try { + return objectMapper().readValue(content, DialectDocument.class); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + /** Parse a dialect from a YAML stream. */ + public static DialectDocument load(InputStream stream) { + try (Scanner scanner = new Scanner(stream, StandardCharsets.UTF_8.name())) { + scanner.useDelimiter(READ_WHOLE_FILE); + String content = scanner.hasNext() ? scanner.next() : ""; + return load(content); + } + } + + /** Parse a dialect from a YAML file on disk. */ + public static DialectDocument loadFromFile(Path path) { + try { + return load(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Parse a dialect from a classpath resource. */ + public static DialectDocument loadResource(String resourcePath) { + try (InputStream stream = Dialect.class.getResourceAsStream(resourcePath)) { + if (stream == null) { + throw new IllegalArgumentException("Dialect resource not found: " + resourcePath); + } + return load(stream); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Serialize a dialect to YAML. */ + public static String toYaml(DialectDocument dialect) { + try { + return objectMapper().writeValueAsString(dialect); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + // --------------------------------------------------------------------------- + // Root document + // --------------------------------------------------------------------------- + + @JsonDeserialize(as = ImmutableDialect.DialectDocument.class) + @JsonSerialize(as = ImmutableDialect.DialectDocument.class) + @Value.Immutable + public abstract static class DialectDocument { + public abstract Optional name(); + + public abstract Optional> metadata(); + + public abstract Map dependencies(); + + @JsonProperty("supported_types") + public abstract List supportedTypes(); + + @JsonProperty("supported_relations") + public abstract List supportedRelations(); + + @JsonProperty("supported_expressions") + public abstract List supportedExpressions(); + + @JsonProperty("supported_scalar_functions") + public abstract List supportedScalarFunctions(); + + @JsonProperty("supported_aggregate_functions") + public abstract List supportedAggregateFunctions(); + + @JsonProperty("supported_window_functions") + public abstract List supportedWindowFunctions(); + + @JsonProperty("supported_execution_behavior") + public abstract Optional supportedExecutionBehavior(); + + public static ImmutableDialect.DialectDocument.Builder builder() { + return ImmutableDialect.DialectDocument.builder(); + } + } + + // --------------------------------------------------------------------------- + // Functions + // --------------------------------------------------------------------------- + + @JsonDeserialize(as = ImmutableDialect.DialectFunction.class) + @JsonSerialize(as = ImmutableDialect.DialectFunction.class) + @Value.Immutable + public abstract static class DialectFunction { + /** Dependency (alias) in which the function is declared. */ + public abstract String source(); + + /** The name of the function as declared in the extension it is defined in. */ + public abstract String name(); + + public abstract Optional> metadata(); + + @JsonProperty("system_metadata") + public abstract Optional systemMetadata(); + + @JsonProperty("required_options") + public abstract Optional> requiredOptions(); + + /** + * One or more implementations supported by this function, identified by argument signatures. + */ + @JsonProperty("supported_impls") + public abstract List supportedImpls(); + + public abstract Optional variadic(); + + public static ImmutableDialect.DialectFunction.Builder builder() { + return ImmutableDialect.DialectFunction.builder(); + } + } + + @JsonDeserialize(as = ImmutableDialect.SystemFunctionMetadata.class) + @JsonSerialize(as = ImmutableDialect.SystemFunctionMetadata.class) + @Value.Immutable + public abstract static class SystemFunctionMetadata { + public abstract Optional name(); + + @Value.Default + public Notation notation() { + return Notation.FUNCTION; + } + + public static ImmutableDialect.SystemFunctionMetadata.Builder builder() { + return ImmutableDialect.SystemFunctionMetadata.builder(); + } + } + + @JsonDeserialize(as = ImmutableDialect.Variadic.class) + @JsonSerialize(as = ImmutableDialect.Variadic.class) + @Value.Immutable + public abstract static class Variadic { + public abstract OptionalInt min(); + + public abstract OptionalInt max(); + + public static ImmutableDialect.Variadic.Builder builder() { + return ImmutableDialect.Variadic.builder(); + } + } + + public enum Notation { + INFIX, + POSTFIX, + PREFIX, + FUNCTION + } + + // --------------------------------------------------------------------------- + // Types + // --------------------------------------------------------------------------- + + @JsonDeserialize(using = SupportedTypeDeserializer.class) + @JsonSerialize(using = SupportedTypeSerializer.class) + @Value.Immutable + public abstract static class SupportedType { + public abstract TypeKind type(); + + public abstract Optional> metadata(); + + public abstract Optional systemMetadata(); + + public abstract Optional maxPrecision(); + + /** Dependency (alias) where a {@code USER_DEFINED} type is declared. */ + public abstract Optional source(); + + /** The name of a {@code USER_DEFINED} type as declared in the extension it is defined in. */ + public abstract Optional name(); + + /** Whether this entry can be written as a bare enum string (no extra configuration). */ + public boolean isBare() { + return type() != TypeKind.USER_DEFINED + && !metadata().isPresent() + && !systemMetadata().isPresent() + && !maxPrecision().isPresent() + && !source().isPresent() + && !name().isPresent(); + } + + public static SupportedType of(TypeKind type) { + return builder().type(type).build(); + } + + public static ImmutableDialect.SupportedType.Builder builder() { + return ImmutableDialect.SupportedType.builder(); + } + } + + @JsonDeserialize(as = ImmutableDialect.SystemTypeMetadata.class) + @JsonSerialize(as = ImmutableDialect.SystemTypeMetadata.class) + @Value.Immutable + public abstract static class SystemTypeMetadata { + public abstract Optional name(); + + @JsonProperty("supported_as_column") + public abstract Optional supportedAsColumn(); + + public static ImmutableDialect.SystemTypeMetadata.Builder builder() { + return ImmutableDialect.SystemTypeMetadata.builder(); + } + } + + public enum TypeKind { + BOOL, + I8, + I16, + I32, + I64, + FP32, + FP64, + BINARY, + FIXED_BINARY, + STRING, + VARCHAR, + FIXED_CHAR, + PRECISION_TIME, + PRECISION_TIMESTAMP, + PRECISION_TIMESTAMP_TZ, + DATE, + TIME, + INTERVAL_COMPOUND, + INTERVAL_DAY, + INTERVAL_YEAR, + UUID, + DECIMAL, + STRUCT, + LIST, + MAP, + USER_DEFINED + } + + // --------------------------------------------------------------------------- + // Relations + // --------------------------------------------------------------------------- + + @JsonDeserialize(using = SupportedRelationDeserializer.class) + @JsonSerialize(using = SupportedRelationSerializer.class) + @Value.Immutable + public abstract static class SupportedRelation { + public abstract RelationKind relation(); + + public abstract Optional> metadata(); + + /** + * Join types for {@code JOIN}, {@code HASH_JOIN}, {@code MERGE_JOIN}, {@code NESTED_LOOP_JOIN}. + */ + public abstract List joinTypes(); + + /** Read types for {@code READ}. */ + public abstract List readTypes(); + + /** Set operations for {@code SET}. */ + public abstract List operations(); + + /** Write types for {@code WRITE} (serialized as {@code write_types}). */ + public abstract List writeTypes(); + + /** Operable object types for {@code DDL} (also serialized as {@code write_types}). */ + public abstract List ddlWriteTypes(); + + /** Exchange kinds for {@code EXCHANGE}. */ + public abstract List kinds(); + + /** Field types for {@code EXPAND}. */ + public abstract List fieldTypes(); + + /** Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. */ + public abstract List messageTypes(); + + /** + * Whether this entry can be written as a bare enum string. Extension relations are never bare: + * they are absent from the schema's bare-enum list. + */ + public boolean isBare() { + switch (relation()) { + case EXTENSION_SINGLE: + case EXTENSION_MULTI: + case EXTENSION_LEAF: + return false; + default: + break; + } + return !metadata().isPresent() + && joinTypes().isEmpty() + && readTypes().isEmpty() + && operations().isEmpty() + && writeTypes().isEmpty() + && ddlWriteTypes().isEmpty() + && kinds().isEmpty() + && fieldTypes().isEmpty() + && messageTypes().isEmpty(); + } + + public static SupportedRelation of(RelationKind relation) { + return builder().relation(relation).build(); + } + + public static ImmutableDialect.SupportedRelation.Builder builder() { + return ImmutableDialect.SupportedRelation.builder(); + } + } + + public enum RelationKind { + READ, + FILTER, + FETCH, + AGGREGATE, + SORT, + JOIN, + PROJECT, + SET, + CROSS, + REFERENCE, + WRITE, + DDL, + UPDATE, + HASH_JOIN, + MERGE_JOIN, + NESTED_LOOP_JOIN, + CONSISTENT_PARTITION_WINDOW, + EXCHANGE, + EXPAND, + EXTENSION_SINGLE, + EXTENSION_MULTI, + EXTENSION_LEAF + } + + public enum JoinType { + INNER, + OUTER, + LEFT, + RIGHT, + LEFT_SEMI, + RIGHT_SEMI, + LEFT_ANTI, + RIGHT_ANTI, + LEFT_SINGLE, + RIGHT_SINGLE, + LEFT_MARK, + RIGHT_MARK + } + + public enum ReadType { + VIRTUAL_TABLE, + LOCAL_FILES, + NAMED_TABLE, + EXTENSION_TABLE, + ICEBERG_TABLE + } + + public enum SetOperation { + MINUS_PRIMARY, + MINUS_PRIMARY_ALL, + MINUS_MULTISET, + INTERSECTION_PRIMARY, + INTERSECTION_MULTISET, + INTERSECTION_MULTISET_ALL, + UNION_DISTINCT, + UNION_ALL + } + + public enum WriteType { + NAMED_TABLE, + EXTENSION_TABLE + } + + public enum DdlWriteType { + NAMED_OBJECT, + EXTENSION_OBJECT + } + + public enum ExchangeKind { + SCATTER_BY_FIELDS, + SINGLE_TARGET, + MULTI_TARGET, + ROUND_ROBIN, + BROADCAST + } + + public enum ExpandFieldType { + SWITCHING_FIELD, + CONSTANT_FIELD + } + + // --------------------------------------------------------------------------- + // Expressions + // --------------------------------------------------------------------------- + + @JsonDeserialize(using = SupportedExpressionDeserializer.class) + @JsonSerialize(using = SupportedExpressionSerializer.class) + @Value.Immutable + public abstract static class SupportedExpression { + public abstract ExpressionKind expression(); + + public abstract Optional> metadata(); + + /** Permissible failure options for {@code CAST}. */ + public abstract List failureOptions(); + + /** Subquery types for {@code SUBQUERY}. */ + public abstract List subqueryTypes(); + + /** Nested types for {@code NESTED}. */ + public abstract List nestedTypes(); + + /** Variable types for {@code EXECUTION_CONTEXT_VARIABLE}. */ + public abstract List variableTypes(); + + /** Whether this entry can be written as a bare enum string (no extra configuration). */ + public boolean isBare() { + return !metadata().isPresent() + && failureOptions().isEmpty() + && subqueryTypes().isEmpty() + && nestedTypes().isEmpty() + && variableTypes().isEmpty(); + } + + public static SupportedExpression of(ExpressionKind expression) { + return builder().expression(expression).build(); + } + + public static ImmutableDialect.SupportedExpression.Builder builder() { + return ImmutableDialect.SupportedExpression.builder(); + } + } + + public enum ExpressionKind { + LITERAL, + SELECTION, + SCALAR_FUNCTION, + WINDOW_FUNCTION, + IF_THEN, + SWITCH, + SINGULAR_OR_LIST, + MULTI_OR_LIST, + CAST, + SUBQUERY, + NESTED, + DYNAMIC_PARAMETER, + EXECUTION_CONTEXT_VARIABLE + } + + public enum CastFailureOption { + RETURN_NULL, + THROW_EXCEPTION + } + + public enum SubqueryType { + SCALAR, + IN_PREDICATE, + SET_PREDICATE, + SET_COMPARISON + } + + public enum NestedType { + STRUCT, + LIST, + MAP + } + + public enum VariableType { + CURRENT_TIMESTAMP, + CURRENT_TIMEZONE, + CURRENT_DATE + } + + // --------------------------------------------------------------------------- + // Execution behavior + // --------------------------------------------------------------------------- + + @JsonDeserialize(as = ImmutableDialect.ExecutionBehavior.class) + @JsonSerialize(as = ImmutableDialect.ExecutionBehavior.class) + @Value.Immutable + public abstract static class ExecutionBehavior { + @JsonProperty("supported_variable_evaluation_mode") + public abstract List supportedVariableEvaluationMode(); + + public static ImmutableDialect.ExecutionBehavior.Builder builder() { + return ImmutableDialect.ExecutionBehavior.builder(); + } + } + + public enum VariableEvaluationMode { + PER_PLAN, + PER_RECORD + } +} diff --git a/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java b/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java new file mode 100644 index 000000000..74c013d3f --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java @@ -0,0 +1,81 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** Shared helpers for the dialect union (de)serializers. */ +final class DialectJsonSupport { + + private static final TypeReference> MAP_TYPE = + new TypeReference>() {}; + + private DialectJsonSupport() {} + + /** Read the {@code metadata} object on a node into a map, or {@code null} when absent. */ + static Map readMetadata(JsonParser p, JsonNode node) { + JsonNode metadata = node.get("metadata"); + if (metadata == null || metadata.isNull()) { + return null; + } + return ((ObjectMapper) p.getCodec()).convertValue(metadata, MAP_TYPE); + } + + /** + * Read a YAML/JSON array node into a list of enum constants. Missing nodes yield an empty list. + */ + static > List readEnums(JsonNode node, Class type) { + List result = new ArrayList<>(); + if (node != null && node.isArray()) { + for (JsonNode element : node) { + result.add(Enum.valueOf(type, element.asText())); + } + } + return result; + } + + /** Read a node into a list of strings. Missing nodes yield an empty list. */ + static List readStrings(JsonNode node) { + List result = new ArrayList<>(); + if (node != null && node.isArray()) { + for (JsonNode element : node) { + result.add(element.asText()); + } + } + return result; + } + + /** Write a list of enum constants as a JSON array field, using each constant's name. */ + static void writeEnumArray(JsonGenerator gen, String fieldName, List> values) + throws IOException { + gen.writeArrayFieldStart(fieldName); + for (Enum value : values) { + gen.writeString(value.name()); + } + gen.writeEndArray(); + } + + /** Write a list of strings as a JSON array field. */ + static void writeStringArray(JsonGenerator gen, String fieldName, List values) + throws IOException { + gen.writeArrayFieldStart(fieldName); + for (String value : values) { + gen.writeString(value); + } + gen.writeEndArray(); + } + + /** Write an optional metadata map as a {@code metadata} field if present. */ + static void writeMetadata( + JsonGenerator gen, SerializerProvider provider, Map metadata) + throws IOException { + provider.defaultSerializeField("metadata", metadata, gen); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java new file mode 100644 index 000000000..3621faf89 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java @@ -0,0 +1,46 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import io.substrait.dialect.Dialect.ExpressionKind; +import io.substrait.dialect.Dialect.SupportedExpression; +import java.io.IOException; +import java.util.Map; + +/** + * Deserializes a {@code supported_expressions} entry, which is either a bare enum string (e.g. + * {@code LITERAL}) or a configuration object (e.g. {@code {expression: CAST, failure_options: + * [RETURN_NULL]}}). + */ +public class SupportedExpressionDeserializer extends JsonDeserializer { + + @Override + public SupportedExpression deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException { + JsonNode node = p.getCodec().readTree(p); + if (node.isTextual()) { + return SupportedExpression.of(ExpressionKind.valueOf(node.asText())); + } + + ExpressionKind kind = ExpressionKind.valueOf(node.get("expression").asText()); + ImmutableDialect.SupportedExpression.Builder builder = + SupportedExpression.builder().expression(kind); + + Map metadata = DialectJsonSupport.readMetadata(p, node); + if (metadata != null) { + builder.metadata(metadata); + } + builder.addAllFailureOptions( + DialectJsonSupport.readEnums(node.get("failure_options"), Dialect.CastFailureOption.class)); + builder.addAllSubqueryTypes( + DialectJsonSupport.readEnums(node.get("subquery_types"), Dialect.SubqueryType.class)); + builder.addAllNestedTypes( + DialectJsonSupport.readEnums(node.get("nested_types"), Dialect.NestedType.class)); + builder.addAllVariableTypes( + DialectJsonSupport.readEnums(node.get("variable_types"), Dialect.VariableType.class)); + + return builder.build(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java new file mode 100644 index 000000000..f6c80d0e5 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java @@ -0,0 +1,45 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.substrait.dialect.Dialect.ExpressionKind; +import io.substrait.dialect.Dialect.SupportedExpression; +import java.io.IOException; + +/** + * Serializes a {@code supported_expressions} entry as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +public class SupportedExpressionSerializer extends JsonSerializer { + + @Override + public void serialize(SupportedExpression value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (value.isBare()) { + gen.writeString(value.expression().name()); + return; + } + + gen.writeStartObject(); + gen.writeStringField("expression", value.expression().name()); + if (!value.failureOptions().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "failure_options", value.failureOptions()); + } + if (!value.subqueryTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "subquery_types", value.subqueryTypes()); + } + if (!value.nestedTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "nested_types", value.nestedTypes()); + } + if (!value.variableTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "variable_types", value.variableTypes()); + } + // The schema forbids `metadata` on EXECUTION_CONTEXT_VARIABLE, so only emit it for others. + if (value.metadata().isPresent() + && value.expression() != ExpressionKind.EXECUTION_CONTEXT_VARIABLE) { + DialectJsonSupport.writeMetadata(gen, provider, value.metadata().get()); + } + gen.writeEndObject(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java new file mode 100644 index 000000000..9c5529f47 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java @@ -0,0 +1,56 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import io.substrait.dialect.Dialect.RelationKind; +import io.substrait.dialect.Dialect.SupportedRelation; +import java.io.IOException; +import java.util.Map; + +/** + * Deserializes a {@code supported_relations} entry, which is either a bare enum string (e.g. {@code + * FILTER}) or a configuration object (e.g. {@code {relation: JOIN, join_types: [INNER]}}). + */ +public class SupportedRelationDeserializer extends JsonDeserializer { + + @Override + public SupportedRelation deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException { + JsonNode node = p.getCodec().readTree(p); + if (node.isTextual()) { + return SupportedRelation.of(RelationKind.valueOf(node.asText())); + } + + RelationKind kind = RelationKind.valueOf(node.get("relation").asText()); + ImmutableDialect.SupportedRelation.Builder builder = SupportedRelation.builder().relation(kind); + + Map metadata = DialectJsonSupport.readMetadata(p, node); + if (metadata != null) { + builder.metadata(metadata); + } + builder.addAllJoinTypes( + DialectJsonSupport.readEnums(node.get("join_types"), Dialect.JoinType.class)); + builder.addAllReadTypes( + DialectJsonSupport.readEnums(node.get("read_types"), Dialect.ReadType.class)); + builder.addAllOperations( + DialectJsonSupport.readEnums(node.get("operations"), Dialect.SetOperation.class)); + builder.addAllKinds( + DialectJsonSupport.readEnums(node.get("kinds"), Dialect.ExchangeKind.class)); + builder.addAllFieldTypes( + DialectJsonSupport.readEnums(node.get("field_types"), Dialect.ExpandFieldType.class)); + builder.addAllMessageTypes(DialectJsonSupport.readStrings(node.get("message_types"))); + + // `write_types` is shared between WRITE and DDL but uses a different enum for each. + if (kind == RelationKind.DDL) { + builder.addAllDdlWriteTypes( + DialectJsonSupport.readEnums(node.get("write_types"), Dialect.DdlWriteType.class)); + } else { + builder.addAllWriteTypes( + DialectJsonSupport.readEnums(node.get("write_types"), Dialect.WriteType.class)); + } + + return builder.build(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java new file mode 100644 index 000000000..6c5b3e35a --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java @@ -0,0 +1,54 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.substrait.dialect.Dialect.SupportedRelation; +import java.io.IOException; + +/** + * Serializes a {@code supported_relations} entry as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +public class SupportedRelationSerializer extends JsonSerializer { + + @Override + public void serialize(SupportedRelation value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (value.isBare()) { + gen.writeString(value.relation().name()); + return; + } + + gen.writeStartObject(); + gen.writeStringField("relation", value.relation().name()); + if (!value.joinTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "join_types", value.joinTypes()); + } + if (!value.readTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "read_types", value.readTypes()); + } + if (!value.operations().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "operations", value.operations()); + } + if (!value.writeTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "write_types", value.writeTypes()); + } + if (!value.ddlWriteTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "write_types", value.ddlWriteTypes()); + } + if (!value.kinds().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "kinds", value.kinds()); + } + if (!value.fieldTypes().isEmpty()) { + DialectJsonSupport.writeEnumArray(gen, "field_types", value.fieldTypes()); + } + if (!value.messageTypes().isEmpty()) { + DialectJsonSupport.writeStringArray(gen, "message_types", value.messageTypes()); + } + if (value.metadata().isPresent()) { + DialectJsonSupport.writeMetadata(gen, provider, value.metadata().get()); + } + gen.writeEndObject(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java new file mode 100644 index 000000000..c94666bb3 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java @@ -0,0 +1,51 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.substrait.dialect.Dialect.SupportedType; +import io.substrait.dialect.Dialect.SystemTypeMetadata; +import io.substrait.dialect.Dialect.TypeKind; +import java.io.IOException; +import java.util.Map; + +/** + * Deserializes a {@code supported_types} entry, which is either a bare enum string (e.g. {@code + * BOOL}) or a configuration object (e.g. {@code {type: PRECISION_TIMESTAMP, max_precision: 9}}). + */ +public class SupportedTypeDeserializer extends JsonDeserializer { + + @Override + public SupportedType deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + JsonNode node = p.getCodec().readTree(p); + if (node.isTextual()) { + return SupportedType.of(TypeKind.valueOf(node.asText())); + } + + ObjectMapper mapper = (ObjectMapper) p.getCodec(); + ImmutableDialect.SupportedType.Builder builder = + SupportedType.builder().type(TypeKind.valueOf(node.get("type").asText())); + + Map metadata = DialectJsonSupport.readMetadata(p, node); + if (metadata != null) { + builder.metadata(metadata); + } + if (node.hasNonNull("system_metadata")) { + builder.systemMetadata( + mapper.convertValue(node.get("system_metadata"), SystemTypeMetadata.class)); + } + if (node.hasNonNull("max_precision")) { + builder.maxPrecision(node.get("max_precision").asInt()); + } + if (node.hasNonNull("source")) { + builder.source(node.get("source").asText()); + } + if (node.hasNonNull("name")) { + builder.name(node.get("name").asText()); + } + + return builder.build(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java new file mode 100644 index 000000000..03e9a0b16 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java @@ -0,0 +1,42 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.substrait.dialect.Dialect.SupportedType; +import java.io.IOException; + +/** + * Serializes a {@code supported_types} entry as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +public class SupportedTypeSerializer extends JsonSerializer { + + @Override + public void serialize(SupportedType value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (value.isBare()) { + gen.writeString(value.type().name()); + return; + } + + gen.writeStartObject(); + gen.writeStringField("type", value.type().name()); + if (value.source().isPresent()) { + gen.writeStringField("source", value.source().get()); + } + if (value.name().isPresent()) { + gen.writeStringField("name", value.name().get()); + } + if (value.maxPrecision().isPresent()) { + gen.writeNumberField("max_precision", value.maxPrecision().get()); + } + if (value.systemMetadata().isPresent()) { + provider.defaultSerializeField("system_metadata", value.systemMetadata().get(), gen); + } + if (value.metadata().isPresent()) { + DialectJsonSupport.writeMetadata(gen, provider, value.metadata().get()); + } + gen.writeEndObject(); + } +} diff --git a/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java new file mode 100644 index 000000000..14f4047d2 --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java @@ -0,0 +1,66 @@ +package io.substrait.dialect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.substrait.dialect.Dialect.DialectDocument; +import io.substrait.dialect.Dialect.JoinType; +import io.substrait.dialect.Dialect.RelationKind; +import io.substrait.dialect.Dialect.SupportedRelation; +import org.junit.jupiter.api.Test; + +/** + * Verifies that configuration-free union entries collapse to a bare enum string in YAML, while + * configured entries serialize as mappings, and both parse back to equal objects. + */ +class DialectBareStringCollapseTest { + + @Test + void configFreeRelationSerializesAsBareString() { + DialectDocument dialect = + DialectDocument.builder() + .addSupportedRelations(SupportedRelation.of(RelationKind.FILTER)) + .build(); + + String yaml = Dialect.toYaml(dialect); + + // The list item is the scalar "FILTER", not a "- relation: FILTER" mapping. + assertTrue(yaml.contains("- \"FILTER\"") || yaml.contains("- FILTER"), yaml); + assertFalse(yaml.contains("relation:"), yaml); + + assertEquals(dialect, Dialect.load(yaml)); + } + + @Test + void configuredRelationSerializesAsObject() { + DialectDocument dialect = + DialectDocument.builder() + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.JOIN) + .addJoinTypes(JoinType.INNER) + .build()) + .build(); + + String yaml = Dialect.toYaml(dialect); + + assertTrue(yaml.contains("relation: \"JOIN\"") || yaml.contains("relation: JOIN"), yaml); + assertTrue(yaml.contains("join_types"), yaml); + + assertEquals(dialect, Dialect.load(yaml)); + } + + @Test + void extensionRelationIsNeverBare() { + SupportedRelation extension = + SupportedRelation.builder().relation(RelationKind.EXTENSION_LEAF).build(); + assertFalse(extension.isBare()); + + String yaml = + Dialect.toYaml(DialectDocument.builder().addSupportedRelations(extension).build()); + assertTrue( + yaml.contains("relation: \"EXTENSION_LEAF\"") || yaml.contains("relation: EXTENSION_LEAF"), + yaml); + } +} diff --git a/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java new file mode 100644 index 000000000..fab4ed2fa --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java @@ -0,0 +1,167 @@ +package io.substrait.dialect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.networknt.schema.Error; +import io.substrait.dialect.Dialect.CastFailureOption; +import io.substrait.dialect.Dialect.DdlWriteType; +import io.substrait.dialect.Dialect.DialectDocument; +import io.substrait.dialect.Dialect.DialectFunction; +import io.substrait.dialect.Dialect.ExchangeKind; +import io.substrait.dialect.Dialect.ExecutionBehavior; +import io.substrait.dialect.Dialect.ExpandFieldType; +import io.substrait.dialect.Dialect.ExpressionKind; +import io.substrait.dialect.Dialect.JoinType; +import io.substrait.dialect.Dialect.NestedType; +import io.substrait.dialect.Dialect.Notation; +import io.substrait.dialect.Dialect.ReadType; +import io.substrait.dialect.Dialect.RelationKind; +import io.substrait.dialect.Dialect.SetOperation; +import io.substrait.dialect.Dialect.SubqueryType; +import io.substrait.dialect.Dialect.SupportedExpression; +import io.substrait.dialect.Dialect.SupportedRelation; +import io.substrait.dialect.Dialect.SupportedType; +import io.substrait.dialect.Dialect.SystemFunctionMetadata; +import io.substrait.dialect.Dialect.SystemTypeMetadata; +import io.substrait.dialect.Dialect.TypeKind; +import io.substrait.dialect.Dialect.VariableEvaluationMode; +import io.substrait.dialect.Dialect.VariableType; +import io.substrait.dialect.Dialect.WriteType; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Builds a dialect in Java exercising every union and configuration, serializes it, validates the + * YAML against the published schema, and parses it back to assert a lossless round-trip. + */ +class DialectRoundTripTest { + + private static DialectDocument sampleDialect() { + return DialectDocument.builder() + .name("Round Trip Dialect") + .putDependencies("arithmetic", "extension:io.substrait:functions_arithmetic") + .putDependencies("spark", "extension:substrait:spark") + // Types: bare, precision (max_precision), system_metadata, and user-defined. + .addSupportedTypes(SupportedType.of(TypeKind.BOOL)) + .addSupportedTypes( + SupportedType.builder() + .type(TypeKind.PRECISION_TIMESTAMP) + .maxPrecision(9) + .systemMetadata( + SystemTypeMetadata.builder() + .name("TimestampNTZType") + .supportedAsColumn(true) + .build()) + .build()) + .addSupportedTypes( + SupportedType.builder() + .type(TypeKind.USER_DEFINED) + .source("spark") + .name("interval") + .build()) + // Relations: bare, join, read, set, write, ddl, exchange, expand, extension. + .addSupportedRelations(SupportedRelation.of(RelationKind.FILTER)) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.JOIN) + .addJoinTypes(JoinType.INNER, JoinType.LEFT) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.READ) + .addReadTypes(ReadType.NAMED_TABLE, ReadType.VIRTUAL_TABLE) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.SET) + .addOperations(SetOperation.UNION_ALL) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.WRITE) + .addWriteTypes(WriteType.NAMED_TABLE) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.DDL) + .addDdlWriteTypes(DdlWriteType.NAMED_OBJECT) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.EXCHANGE) + .addKinds(ExchangeKind.ROUND_ROBIN, ExchangeKind.BROADCAST) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.EXPAND) + .addFieldTypes(ExpandFieldType.SWITCHING_FIELD) + .build()) + .addSupportedRelations( + SupportedRelation.builder() + .relation(RelationKind.EXTENSION_SINGLE) + .addMessageTypes("type.googleapis.com/google.profile.Person") + .build()) + // Expressions: bare, cast, subquery, nested, execution-context-variable. + .addSupportedExpressions(SupportedExpression.of(ExpressionKind.LITERAL)) + .addSupportedExpressions( + SupportedExpression.builder() + .expression(ExpressionKind.CAST) + .addFailureOptions(CastFailureOption.RETURN_NULL) + .build()) + .addSupportedExpressions( + SupportedExpression.builder() + .expression(ExpressionKind.SUBQUERY) + .addSubqueryTypes(SubqueryType.SCALAR, SubqueryType.IN_PREDICATE) + .build()) + .addSupportedExpressions( + SupportedExpression.builder() + .expression(ExpressionKind.NESTED) + .addNestedTypes(NestedType.STRUCT, NestedType.LIST) + .build()) + .addSupportedExpressions( + SupportedExpression.builder() + .expression(ExpressionKind.EXECUTION_CONTEXT_VARIABLE) + .addVariableTypes(VariableType.CURRENT_DATE, VariableType.CURRENT_TIMESTAMP) + .build()) + // Functions across the three categories. + .addSupportedScalarFunctions( + DialectFunction.builder() + .source("arithmetic") + .name("add") + .systemMetadata( + SystemFunctionMetadata.builder().name("+").notation(Notation.INFIX).build()) + .addSupportedImpls("i32_i32", "i64_i64") + .build()) + .addSupportedAggregateFunctions( + DialectFunction.builder() + .source("arithmetic") + .name("sum") + .addSupportedImpls("i64") + .build()) + .addSupportedWindowFunctions( + DialectFunction.builder() + .source("arithmetic") + .name("row_number") + .addSupportedImpls("") + .build()) + .supportedExecutionBehavior( + ExecutionBehavior.builder() + .addSupportedVariableEvaluationMode(VariableEvaluationMode.PER_PLAN) + .build()) + .build(); + } + + @Test + void roundTripThroughYamlAndSchema() { + DialectDocument original = sampleDialect(); + + String yaml = Dialect.toYaml(original); + + List errors = SchemaValidator.validate(yaml); + assertTrue(errors.isEmpty(), () -> "Generated dialect failed schema validation: " + errors); + + DialectDocument parsed = Dialect.load(yaml); + assertEquals(original, parsed); + } +} diff --git a/core/src/test/java/io/substrait/dialect/SchemaValidator.java b/core/src/test/java/io/substrait/dialect/SchemaValidator.java new file mode 100644 index 000000000..c381275a0 --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/SchemaValidator.java @@ -0,0 +1,42 @@ +package io.substrait.dialect; + +import com.networknt.schema.Error; +import com.networknt.schema.InputFormat; +import com.networknt.schema.Schema; +import com.networknt.schema.SchemaRegistry; +import com.networknt.schema.SpecificationVersion; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.List; + +/** + * Validates dialect YAML against the published {@code dialect_schema.yaml}, which is placed on the + * test classpath by the {@code processTestResources} task in {@code core/build.gradle.kts}. + */ +final class SchemaValidator { + + private static final String SCHEMA_RESOURCE = "/substrait/text/dialect_schema.yaml"; + + private static final Schema SCHEMA = loadSchema(); + + private SchemaValidator() {} + + private static Schema loadSchema() { + try (InputStream stream = SchemaValidator.class.getResourceAsStream(SCHEMA_RESOURCE)) { + if (stream == null) { + throw new IllegalStateException( + "Dialect schema not found on classpath: " + SCHEMA_RESOURCE); + } + return SchemaRegistry.withDefaultDialect(SpecificationVersion.DRAFT_2020_12) + .getSchema(stream, InputFormat.YAML); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Returns the validation errors produced by checking {@code yaml} against the schema. */ + static List validate(String yaml) { + return SCHEMA.validate(yaml, InputFormat.YAML); + } +} diff --git a/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java b/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java new file mode 100644 index 000000000..c5681c871 --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java @@ -0,0 +1,66 @@ +package io.substrait.dialect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.networknt.schema.Error; +import io.substrait.dialect.Dialect.DialectDocument; +import io.substrait.dialect.Dialect.ExpressionKind; +import io.substrait.dialect.Dialect.RelationKind; +import io.substrait.dialect.Dialect.SubqueryType; +import io.substrait.dialect.Dialect.SupportedExpression; +import io.substrait.dialect.Dialect.SupportedRelation; +import io.substrait.dialect.Dialect.SupportedType; +import io.substrait.dialect.Dialect.TypeKind; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Parses the real-world {@code spark_dialect.yaml} (copied onto the test classpath by the build) to + * confirm the model consumes a production dialect, and re-validates it against the schema. + */ +class SparkDialectParseTest { + + private static final DialectDocument SPARK = Dialect.loadResource("/dialect/spark_dialect.yaml"); + + @Test + void parsesTopLevelFields() { + assertEquals("Spark Dialect", SPARK.name().orElse(null)); + assertTrue(SPARK.dependencies().containsKey("spark")); + assertTrue(SPARK.supportedScalarFunctions().size() > 0); + } + + @Test + void parsesConfiguredRelationsAndExpressions() { + SupportedRelation join = + SPARK.supportedRelations().stream() + .filter(r -> r.relation() == RelationKind.JOIN) + .findFirst() + .orElseThrow(); + assertTrue(join.joinTypes().contains(Dialect.JoinType.INNER)); + + SupportedExpression subquery = + SPARK.supportedExpressions().stream() + .filter(e -> e.expression() == ExpressionKind.SUBQUERY) + .findFirst() + .orElseThrow(); + assertEquals(List.of(SubqueryType.SCALAR, SubqueryType.IN_PREDICATE), subquery.subqueryTypes()); + } + + @Test + void parsesPrecisionTypes() { + SupportedType precisionTimestamp = + SPARK.supportedTypes().stream() + .filter(t -> t.type() == TypeKind.PRECISION_TIMESTAMP) + .findFirst() + .orElseThrow(); + assertEquals(9, precisionTimestamp.maxPrecision().orElse(-1)); + } + + @Test + void reserializesToSchemaValidYaml() { + String yaml = Dialect.toYaml(SPARK); + List errors = SchemaValidator.validate(yaml); + assertTrue(errors.isEmpty(), () -> "Re-serialized Spark dialect failed validation: " + errors); + } +} diff --git a/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java b/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java new file mode 100644 index 000000000..16f52a330 --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java @@ -0,0 +1,66 @@ +package io.substrait.dialect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.networknt.schema.Error; +import io.substrait.dialect.Dialect.DialectDocument; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Scanner; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Exercises the per-section dialect fixtures published by the substrait spec (copied onto the test + * classpath by the build). For each fixture we confirm it is schema-valid, parses, and survives a + * lossless serialize/parse round-trip whose output is itself schema-valid. + */ +class SpecDialectFixturesTest { + + private static String readResource(String resourcePath) { + try (InputStream stream = SpecDialectFixturesTest.class.getResourceAsStream(resourcePath)) { + if (stream == null) { + throw new IllegalStateException("Fixture not found on classpath: " + resourcePath); + } + try (Scanner scanner = new Scanner(stream, StandardCharsets.UTF_8.name())) { + scanner.useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : ""; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "types_test.yaml", + "relations_test.yaml", + "expressions_test.yaml", + "functions_test.yaml", + "execution_behavior_test.yaml" + }) + void roundTrips(String fixture) { + String resourcePath = "/dialect/tests/" + fixture; + String original = readResource(resourcePath); + + // The published fixture is itself schema-valid. + List fixtureErrors = SchemaValidator.validate(original); + assertTrue(fixtureErrors.isEmpty(), () -> fixture + " is not schema-valid: " + fixtureErrors); + + // Parse, re-serialize, and confirm the output is still schema-valid. + DialectDocument parsed = Dialect.loadResource(resourcePath); + String reserialized = Dialect.toYaml(parsed); + List roundTripErrors = SchemaValidator.validate(reserialized); + assertTrue( + roundTripErrors.isEmpty(), + () -> "Re-serialized " + fixture + " is not schema-valid: " + roundTripErrors); + + // Re-parsing the serialized form yields an equal model (lossless round-trip). + assertEquals(parsed, Dialect.load(reserialized)); + } +} From fb74b41d2db2b75517927985c09ea7522665e923 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Mon, 29 Jun 2026 08:36:51 +0200 Subject: [PATCH 2/3] refactor(core): flatten dialect model and address review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promote the nested dialect types out of the @Value.Enclosing Dialect holder into top-level classes in io.substrait.dialect, and rename DialectDocument to Dialect (now carrying the builder/load/toYaml factory methods). This removes the io.substrait.dialect.Dialect.DialectDocument repetition. Also addresses the remaining review comments: - share a single package-scoped ObjectMapper instead of creating one per call, and have the union (de)serializers use it directly rather than casting JsonParser#getCodec() - read directly from InputStream/File instead of slurping into a String; the InputStream overload no longer closes a caller-owned stream - accept a single scalar as a one-element list in readEnums/readStrings, consistent with ACCEPT_SINGLE_VALUE_AS_ARRAY - reject metadata on EXECUTION_CONTEXT_VARIABLE and forbid setting both writeTypes and ddlWriteTypes (they share the write_types field), so entries always round-trip - make the (de)serializer classes package-private - use InputStream.readAllBytes() in the spec-fixtures test 🤖 Generated with AI --- .../substrait/dialect/CastFailureOption.java | 7 + .../io/substrait/dialect/DdlWriteType.java | 7 + .../java/io/substrait/dialect/Dialect.java | 550 ++---------------- .../io/substrait/dialect/DialectFunction.java | 39 ++ .../substrait/dialect/DialectJsonSupport.java | 48 +- .../io/substrait/dialect/ExchangeKind.java | 10 + .../substrait/dialect/ExecutionBehavior.java | 20 + .../io/substrait/dialect/ExpandFieldType.java | 7 + .../io/substrait/dialect/ExpressionKind.java | 18 + .../java/io/substrait/dialect/JoinType.java | 17 + .../java/io/substrait/dialect/NestedType.java | 8 + .../java/io/substrait/dialect/Notation.java | 9 + .../java/io/substrait/dialect/ReadType.java | 10 + .../io/substrait/dialect/RelationKind.java | 27 + .../io/substrait/dialect/SetOperation.java | 13 + .../io/substrait/dialect/SubqueryType.java | 9 + .../dialect/SupportedExpression.java | 62 ++ .../SupportedExpressionDeserializer.java | 17 +- .../SupportedExpressionSerializer.java | 9 +- .../substrait/dialect/SupportedRelation.java | 93 +++ .../SupportedRelationDeserializer.java | 25 +- .../dialect/SupportedRelationSerializer.java | 4 +- .../io/substrait/dialect/SupportedType.java | 48 ++ .../dialect/SupportedTypeDeserializer.java | 14 +- .../dialect/SupportedTypeSerializer.java | 3 +- .../dialect/SystemFunctionMetadata.java | 23 + .../substrait/dialect/SystemTypeMetadata.java | 22 + .../java/io/substrait/dialect/TypeKind.java | 31 + .../dialect/VariableEvaluationMode.java | 7 + .../io/substrait/dialect/VariableType.java | 8 + .../java/io/substrait/dialect/Variadic.java | 20 + .../java/io/substrait/dialect/WriteType.java | 7 + .../DialectBareStringCollapseTest.java | 17 +- .../dialect/DialectRoundTripTest.java | 32 +- .../dialect/DialectValidationTest.java | 33 ++ .../dialect/SparkDialectParseTest.java | 12 +- .../dialect/SpecDialectFixturesTest.java | 9 +- 37 files changed, 692 insertions(+), 603 deletions(-) create mode 100644 core/src/main/java/io/substrait/dialect/CastFailureOption.java create mode 100644 core/src/main/java/io/substrait/dialect/DdlWriteType.java create mode 100644 core/src/main/java/io/substrait/dialect/DialectFunction.java create mode 100644 core/src/main/java/io/substrait/dialect/ExchangeKind.java create mode 100644 core/src/main/java/io/substrait/dialect/ExecutionBehavior.java create mode 100644 core/src/main/java/io/substrait/dialect/ExpandFieldType.java create mode 100644 core/src/main/java/io/substrait/dialect/ExpressionKind.java create mode 100644 core/src/main/java/io/substrait/dialect/JoinType.java create mode 100644 core/src/main/java/io/substrait/dialect/NestedType.java create mode 100644 core/src/main/java/io/substrait/dialect/Notation.java create mode 100644 core/src/main/java/io/substrait/dialect/ReadType.java create mode 100644 core/src/main/java/io/substrait/dialect/RelationKind.java create mode 100644 core/src/main/java/io/substrait/dialect/SetOperation.java create mode 100644 core/src/main/java/io/substrait/dialect/SubqueryType.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedExpression.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedRelation.java create mode 100644 core/src/main/java/io/substrait/dialect/SupportedType.java create mode 100644 core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java create mode 100644 core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java create mode 100644 core/src/main/java/io/substrait/dialect/TypeKind.java create mode 100644 core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java create mode 100644 core/src/main/java/io/substrait/dialect/VariableType.java create mode 100644 core/src/main/java/io/substrait/dialect/Variadic.java create mode 100644 core/src/main/java/io/substrait/dialect/WriteType.java create mode 100644 core/src/test/java/io/substrait/dialect/DialectValidationTest.java diff --git a/core/src/main/java/io/substrait/dialect/CastFailureOption.java b/core/src/main/java/io/substrait/dialect/CastFailureOption.java new file mode 100644 index 000000000..a1beba2ac --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/CastFailureOption.java @@ -0,0 +1,7 @@ +package io.substrait.dialect; + +/** Permissible failure options for {@code CAST} expressions. */ +public enum CastFailureOption { + RETURN_NULL, + THROW_EXCEPTION +} diff --git a/core/src/main/java/io/substrait/dialect/DdlWriteType.java b/core/src/main/java/io/substrait/dialect/DdlWriteType.java new file mode 100644 index 000000000..be5a182d4 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/DdlWriteType.java @@ -0,0 +1,7 @@ +package io.substrait.dialect; + +/** Operable object types for {@code DDL} relations. */ +public enum DdlWriteType { + NAMED_OBJECT, + EXTENSION_OBJECT +} diff --git a/core/src/main/java/io/substrait/dialect/Dialect.java b/core/src/main/java/io/substrait/dialect/Dialect.java index d773f50c1..75bae97be 100644 --- a/core/src/main/java/io/substrait/dialect/Dialect.java +++ b/core/src/main/java/io/substrait/dialect/Dialect.java @@ -1,85 +1,98 @@ package io.substrait.dialect; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; -import java.util.Scanner; -import java.util.regex.Pattern; import org.immutables.value.Value; /** - * Classes used to create and consume Substrait dialect YAML files. - * - *

A dialect describes what a target execution system supports: which types, relations, - * expressions and functions, along with per-feature configuration (join types, set operations, cast - * failure behavior, ...). The model maps the schema published at {@code + * A Substrait dialect: a description of what a target execution system supports — which types, + * relations, expressions and functions, along with per-feature configuration (join types, set + * operations, cast failure behavior, ...). The model maps the schema published at {@code * substrait/text/dialect_schema.yaml}. * - *

Build a dialect with the nested builders and serialize it with {@link - * #toYaml(DialectDocument)}; parse one with {@link #load(String)} and friends. + *

Build a dialect with {@link #builder()} and the sibling entry types and serialize it with + * {@link #toYaml(Dialect)}; parse one with {@link #load(String)} and friends. */ -@Value.Enclosing -public class Dialect { +@JsonDeserialize(as = ImmutableDialect.class) +@JsonSerialize(as = ImmutableDialect.class) +@Value.Immutable +public abstract class Dialect { + + public abstract Optional name(); + + public abstract Optional> metadata(); + + public abstract Map dependencies(); + + @JsonProperty("supported_types") + public abstract List supportedTypes(); + + @JsonProperty("supported_relations") + public abstract List supportedRelations(); + + @JsonProperty("supported_expressions") + public abstract List supportedExpressions(); + + @JsonProperty("supported_scalar_functions") + public abstract List supportedScalarFunctions(); + + @JsonProperty("supported_aggregate_functions") + public abstract List supportedAggregateFunctions(); - // `\A` means beginning of input. Using it as a delimiter in a scanner reads in the whole file. - private static final Pattern READ_WHOLE_FILE = Pattern.compile("\\A"); + @JsonProperty("supported_window_functions") + public abstract List supportedWindowFunctions(); - private Dialect() {} + @JsonProperty("supported_execution_behavior") + public abstract Optional supportedExecutionBehavior(); - private static ObjectMapper objectMapper() { - return new ObjectMapper(new YAMLFactory()) - .registerModule(new Jdk8Module()) - .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) - // Omit absent Optionals and empty collections so that unset sections are not emitted. - // The custom (de)serializers for the polymorphic unions write their fields explicitly and - // are unaffected by this inclusion setting. - .setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + public static ImmutableDialect.Builder builder() { + return ImmutableDialect.builder(); } /** Parse a dialect from YAML content. */ - public static DialectDocument load(String content) { + public static Dialect load(String content) { try { - return objectMapper().readValue(content, DialectDocument.class); + return DialectJsonSupport.MAPPER.readValue(content, Dialect.class); } catch (IOException e) { throw new IllegalStateException(e); } } - /** Parse a dialect from a YAML stream. */ - public static DialectDocument load(InputStream stream) { - try (Scanner scanner = new Scanner(stream, StandardCharsets.UTF_8.name())) { - scanner.useDelimiter(READ_WHOLE_FILE); - String content = scanner.hasNext() ? scanner.next() : ""; - return load(content); + /** + * Parse a dialect from a YAML stream. The caller retains ownership of the stream; it is not + * closed by this method. + */ + public static Dialect load(InputStream stream) { + try { + return DialectJsonSupport.MAPPER + .readerFor(Dialect.class) + .without(JsonParser.Feature.AUTO_CLOSE_SOURCE) + .readValue(stream); + } catch (IOException e) { + throw new IllegalStateException(e); } } /** Parse a dialect from a YAML file on disk. */ - public static DialectDocument loadFromFile(Path path) { + public static Dialect loadFromFile(Path path) { try { - return load(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + return DialectJsonSupport.MAPPER.readValue(path.toFile(), Dialect.class); } catch (IOException e) { throw new UncheckedIOException(e); } } /** Parse a dialect from a classpath resource. */ - public static DialectDocument loadResource(String resourcePath) { + public static Dialect loadResource(String resourcePath) { try (InputStream stream = Dialect.class.getResourceAsStream(resourcePath)) { if (stream == null) { throw new IllegalArgumentException("Dialect resource not found: " + resourcePath); @@ -91,462 +104,11 @@ public static DialectDocument loadResource(String resourcePath) { } /** Serialize a dialect to YAML. */ - public static String toYaml(DialectDocument dialect) { + public static String toYaml(Dialect dialect) { try { - return objectMapper().writeValueAsString(dialect); + return DialectJsonSupport.MAPPER.writeValueAsString(dialect); } catch (IOException e) { throw new IllegalStateException(e); } } - - // --------------------------------------------------------------------------- - // Root document - // --------------------------------------------------------------------------- - - @JsonDeserialize(as = ImmutableDialect.DialectDocument.class) - @JsonSerialize(as = ImmutableDialect.DialectDocument.class) - @Value.Immutable - public abstract static class DialectDocument { - public abstract Optional name(); - - public abstract Optional> metadata(); - - public abstract Map dependencies(); - - @JsonProperty("supported_types") - public abstract List supportedTypes(); - - @JsonProperty("supported_relations") - public abstract List supportedRelations(); - - @JsonProperty("supported_expressions") - public abstract List supportedExpressions(); - - @JsonProperty("supported_scalar_functions") - public abstract List supportedScalarFunctions(); - - @JsonProperty("supported_aggregate_functions") - public abstract List supportedAggregateFunctions(); - - @JsonProperty("supported_window_functions") - public abstract List supportedWindowFunctions(); - - @JsonProperty("supported_execution_behavior") - public abstract Optional supportedExecutionBehavior(); - - public static ImmutableDialect.DialectDocument.Builder builder() { - return ImmutableDialect.DialectDocument.builder(); - } - } - - // --------------------------------------------------------------------------- - // Functions - // --------------------------------------------------------------------------- - - @JsonDeserialize(as = ImmutableDialect.DialectFunction.class) - @JsonSerialize(as = ImmutableDialect.DialectFunction.class) - @Value.Immutable - public abstract static class DialectFunction { - /** Dependency (alias) in which the function is declared. */ - public abstract String source(); - - /** The name of the function as declared in the extension it is defined in. */ - public abstract String name(); - - public abstract Optional> metadata(); - - @JsonProperty("system_metadata") - public abstract Optional systemMetadata(); - - @JsonProperty("required_options") - public abstract Optional> requiredOptions(); - - /** - * One or more implementations supported by this function, identified by argument signatures. - */ - @JsonProperty("supported_impls") - public abstract List supportedImpls(); - - public abstract Optional variadic(); - - public static ImmutableDialect.DialectFunction.Builder builder() { - return ImmutableDialect.DialectFunction.builder(); - } - } - - @JsonDeserialize(as = ImmutableDialect.SystemFunctionMetadata.class) - @JsonSerialize(as = ImmutableDialect.SystemFunctionMetadata.class) - @Value.Immutable - public abstract static class SystemFunctionMetadata { - public abstract Optional name(); - - @Value.Default - public Notation notation() { - return Notation.FUNCTION; - } - - public static ImmutableDialect.SystemFunctionMetadata.Builder builder() { - return ImmutableDialect.SystemFunctionMetadata.builder(); - } - } - - @JsonDeserialize(as = ImmutableDialect.Variadic.class) - @JsonSerialize(as = ImmutableDialect.Variadic.class) - @Value.Immutable - public abstract static class Variadic { - public abstract OptionalInt min(); - - public abstract OptionalInt max(); - - public static ImmutableDialect.Variadic.Builder builder() { - return ImmutableDialect.Variadic.builder(); - } - } - - public enum Notation { - INFIX, - POSTFIX, - PREFIX, - FUNCTION - } - - // --------------------------------------------------------------------------- - // Types - // --------------------------------------------------------------------------- - - @JsonDeserialize(using = SupportedTypeDeserializer.class) - @JsonSerialize(using = SupportedTypeSerializer.class) - @Value.Immutable - public abstract static class SupportedType { - public abstract TypeKind type(); - - public abstract Optional> metadata(); - - public abstract Optional systemMetadata(); - - public abstract Optional maxPrecision(); - - /** Dependency (alias) where a {@code USER_DEFINED} type is declared. */ - public abstract Optional source(); - - /** The name of a {@code USER_DEFINED} type as declared in the extension it is defined in. */ - public abstract Optional name(); - - /** Whether this entry can be written as a bare enum string (no extra configuration). */ - public boolean isBare() { - return type() != TypeKind.USER_DEFINED - && !metadata().isPresent() - && !systemMetadata().isPresent() - && !maxPrecision().isPresent() - && !source().isPresent() - && !name().isPresent(); - } - - public static SupportedType of(TypeKind type) { - return builder().type(type).build(); - } - - public static ImmutableDialect.SupportedType.Builder builder() { - return ImmutableDialect.SupportedType.builder(); - } - } - - @JsonDeserialize(as = ImmutableDialect.SystemTypeMetadata.class) - @JsonSerialize(as = ImmutableDialect.SystemTypeMetadata.class) - @Value.Immutable - public abstract static class SystemTypeMetadata { - public abstract Optional name(); - - @JsonProperty("supported_as_column") - public abstract Optional supportedAsColumn(); - - public static ImmutableDialect.SystemTypeMetadata.Builder builder() { - return ImmutableDialect.SystemTypeMetadata.builder(); - } - } - - public enum TypeKind { - BOOL, - I8, - I16, - I32, - I64, - FP32, - FP64, - BINARY, - FIXED_BINARY, - STRING, - VARCHAR, - FIXED_CHAR, - PRECISION_TIME, - PRECISION_TIMESTAMP, - PRECISION_TIMESTAMP_TZ, - DATE, - TIME, - INTERVAL_COMPOUND, - INTERVAL_DAY, - INTERVAL_YEAR, - UUID, - DECIMAL, - STRUCT, - LIST, - MAP, - USER_DEFINED - } - - // --------------------------------------------------------------------------- - // Relations - // --------------------------------------------------------------------------- - - @JsonDeserialize(using = SupportedRelationDeserializer.class) - @JsonSerialize(using = SupportedRelationSerializer.class) - @Value.Immutable - public abstract static class SupportedRelation { - public abstract RelationKind relation(); - - public abstract Optional> metadata(); - - /** - * Join types for {@code JOIN}, {@code HASH_JOIN}, {@code MERGE_JOIN}, {@code NESTED_LOOP_JOIN}. - */ - public abstract List joinTypes(); - - /** Read types for {@code READ}. */ - public abstract List readTypes(); - - /** Set operations for {@code SET}. */ - public abstract List operations(); - - /** Write types for {@code WRITE} (serialized as {@code write_types}). */ - public abstract List writeTypes(); - - /** Operable object types for {@code DDL} (also serialized as {@code write_types}). */ - public abstract List ddlWriteTypes(); - - /** Exchange kinds for {@code EXCHANGE}. */ - public abstract List kinds(); - - /** Field types for {@code EXPAND}. */ - public abstract List fieldTypes(); - - /** Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. */ - public abstract List messageTypes(); - - /** - * Whether this entry can be written as a bare enum string. Extension relations are never bare: - * they are absent from the schema's bare-enum list. - */ - public boolean isBare() { - switch (relation()) { - case EXTENSION_SINGLE: - case EXTENSION_MULTI: - case EXTENSION_LEAF: - return false; - default: - break; - } - return !metadata().isPresent() - && joinTypes().isEmpty() - && readTypes().isEmpty() - && operations().isEmpty() - && writeTypes().isEmpty() - && ddlWriteTypes().isEmpty() - && kinds().isEmpty() - && fieldTypes().isEmpty() - && messageTypes().isEmpty(); - } - - public static SupportedRelation of(RelationKind relation) { - return builder().relation(relation).build(); - } - - public static ImmutableDialect.SupportedRelation.Builder builder() { - return ImmutableDialect.SupportedRelation.builder(); - } - } - - public enum RelationKind { - READ, - FILTER, - FETCH, - AGGREGATE, - SORT, - JOIN, - PROJECT, - SET, - CROSS, - REFERENCE, - WRITE, - DDL, - UPDATE, - HASH_JOIN, - MERGE_JOIN, - NESTED_LOOP_JOIN, - CONSISTENT_PARTITION_WINDOW, - EXCHANGE, - EXPAND, - EXTENSION_SINGLE, - EXTENSION_MULTI, - EXTENSION_LEAF - } - - public enum JoinType { - INNER, - OUTER, - LEFT, - RIGHT, - LEFT_SEMI, - RIGHT_SEMI, - LEFT_ANTI, - RIGHT_ANTI, - LEFT_SINGLE, - RIGHT_SINGLE, - LEFT_MARK, - RIGHT_MARK - } - - public enum ReadType { - VIRTUAL_TABLE, - LOCAL_FILES, - NAMED_TABLE, - EXTENSION_TABLE, - ICEBERG_TABLE - } - - public enum SetOperation { - MINUS_PRIMARY, - MINUS_PRIMARY_ALL, - MINUS_MULTISET, - INTERSECTION_PRIMARY, - INTERSECTION_MULTISET, - INTERSECTION_MULTISET_ALL, - UNION_DISTINCT, - UNION_ALL - } - - public enum WriteType { - NAMED_TABLE, - EXTENSION_TABLE - } - - public enum DdlWriteType { - NAMED_OBJECT, - EXTENSION_OBJECT - } - - public enum ExchangeKind { - SCATTER_BY_FIELDS, - SINGLE_TARGET, - MULTI_TARGET, - ROUND_ROBIN, - BROADCAST - } - - public enum ExpandFieldType { - SWITCHING_FIELD, - CONSTANT_FIELD - } - - // --------------------------------------------------------------------------- - // Expressions - // --------------------------------------------------------------------------- - - @JsonDeserialize(using = SupportedExpressionDeserializer.class) - @JsonSerialize(using = SupportedExpressionSerializer.class) - @Value.Immutable - public abstract static class SupportedExpression { - public abstract ExpressionKind expression(); - - public abstract Optional> metadata(); - - /** Permissible failure options for {@code CAST}. */ - public abstract List failureOptions(); - - /** Subquery types for {@code SUBQUERY}. */ - public abstract List subqueryTypes(); - - /** Nested types for {@code NESTED}. */ - public abstract List nestedTypes(); - - /** Variable types for {@code EXECUTION_CONTEXT_VARIABLE}. */ - public abstract List variableTypes(); - - /** Whether this entry can be written as a bare enum string (no extra configuration). */ - public boolean isBare() { - return !metadata().isPresent() - && failureOptions().isEmpty() - && subqueryTypes().isEmpty() - && nestedTypes().isEmpty() - && variableTypes().isEmpty(); - } - - public static SupportedExpression of(ExpressionKind expression) { - return builder().expression(expression).build(); - } - - public static ImmutableDialect.SupportedExpression.Builder builder() { - return ImmutableDialect.SupportedExpression.builder(); - } - } - - public enum ExpressionKind { - LITERAL, - SELECTION, - SCALAR_FUNCTION, - WINDOW_FUNCTION, - IF_THEN, - SWITCH, - SINGULAR_OR_LIST, - MULTI_OR_LIST, - CAST, - SUBQUERY, - NESTED, - DYNAMIC_PARAMETER, - EXECUTION_CONTEXT_VARIABLE - } - - public enum CastFailureOption { - RETURN_NULL, - THROW_EXCEPTION - } - - public enum SubqueryType { - SCALAR, - IN_PREDICATE, - SET_PREDICATE, - SET_COMPARISON - } - - public enum NestedType { - STRUCT, - LIST, - MAP - } - - public enum VariableType { - CURRENT_TIMESTAMP, - CURRENT_TIMEZONE, - CURRENT_DATE - } - - // --------------------------------------------------------------------------- - // Execution behavior - // --------------------------------------------------------------------------- - - @JsonDeserialize(as = ImmutableDialect.ExecutionBehavior.class) - @JsonSerialize(as = ImmutableDialect.ExecutionBehavior.class) - @Value.Immutable - public abstract static class ExecutionBehavior { - @JsonProperty("supported_variable_evaluation_mode") - public abstract List supportedVariableEvaluationMode(); - - public static ImmutableDialect.ExecutionBehavior.Builder builder() { - return ImmutableDialect.ExecutionBehavior.builder(); - } - } - - public enum VariableEvaluationMode { - PER_PLAN, - PER_RECORD - } } diff --git a/core/src/main/java/io/substrait/dialect/DialectFunction.java b/core/src/main/java/io/substrait/dialect/DialectFunction.java new file mode 100644 index 000000000..de784496c --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/DialectFunction.java @@ -0,0 +1,39 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** A function supported by a dialect, referencing a dependency alias. */ +@JsonDeserialize(as = ImmutableDialectFunction.class) +@JsonSerialize(as = ImmutableDialectFunction.class) +@Value.Immutable +public abstract class DialectFunction { + /** Dependency (alias) in which the function is declared. */ + public abstract String source(); + + /** The name of the function as declared in the extension it is defined in. */ + public abstract String name(); + + public abstract Optional> metadata(); + + @JsonProperty("system_metadata") + public abstract Optional systemMetadata(); + + @JsonProperty("required_options") + public abstract Optional> requiredOptions(); + + /** One or more implementations supported by this function, identified by argument signatures. */ + @JsonProperty("supported_impls") + public abstract List supportedImpls(); + + public abstract Optional variadic(); + + public static ImmutableDialectFunction.Builder builder() { + return ImmutableDialectFunction.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java b/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java index 74c013d3f..1250e0d45 100644 --- a/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java +++ b/core/src/main/java/io/substrait/dialect/DialectJsonSupport.java @@ -1,53 +1,85 @@ package io.substrait.dialect; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -/** Shared helpers for the dialect union (de)serializers. */ +/** Shared helpers and the single, shared {@link ObjectMapper} for the dialect (de)serializers. */ final class DialectJsonSupport { + /** + * A single, shared mapper. {@link ObjectMapper} is thread-safe once configured, so the dialect + * code reuses one instance for all (de)serialization. The custom union (de)serializers also read + * it directly rather than casting {@code JsonParser#getCodec()}. + */ + static final ObjectMapper MAPPER = + new ObjectMapper(new YAMLFactory()) + .registerModule(new Jdk8Module()) + .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + // Omit absent Optionals and empty collections so that unset sections are not emitted. + // The custom (de)serializers for the polymorphic unions write their fields explicitly and + // are unaffected by this inclusion setting. + .setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + private static final TypeReference> MAP_TYPE = new TypeReference>() {}; private DialectJsonSupport() {} /** Read the {@code metadata} object on a node into a map, or {@code null} when absent. */ - static Map readMetadata(JsonParser p, JsonNode node) { + static Map readMetadata(JsonNode node) { JsonNode metadata = node.get("metadata"); if (metadata == null || metadata.isNull()) { return null; } - return ((ObjectMapper) p.getCodec()).convertValue(metadata, MAP_TYPE); + return MAPPER.convertValue(metadata, MAP_TYPE); } /** - * Read a YAML/JSON array node into a list of enum constants. Missing nodes yield an empty list. + * Read a YAML/JSON node into a list of enum constants. Both array nodes and a single scalar node + * (consistent with {@code ACCEPT_SINGLE_VALUE_AS_ARRAY}) are accepted. Missing nodes yield an + * empty list. */ static > List readEnums(JsonNode node, Class type) { List result = new ArrayList<>(); - if (node != null && node.isArray()) { + if (node == null || node.isNull()) { + return result; + } + if (node.isArray()) { for (JsonNode element : node) { result.add(Enum.valueOf(type, element.asText())); } + } else { + result.add(Enum.valueOf(type, node.asText())); } return result; } - /** Read a node into a list of strings. Missing nodes yield an empty list. */ + /** + * Read a node into a list of strings. Both array nodes and a single scalar node are accepted. + * Missing nodes yield an empty list. + */ static List readStrings(JsonNode node) { List result = new ArrayList<>(); - if (node != null && node.isArray()) { + if (node == null || node.isNull()) { + return result; + } + if (node.isArray()) { for (JsonNode element : node) { result.add(element.asText()); } + } else { + result.add(node.asText()); } return result; } diff --git a/core/src/main/java/io/substrait/dialect/ExchangeKind.java b/core/src/main/java/io/substrait/dialect/ExchangeKind.java new file mode 100644 index 000000000..cd3089e7f --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/ExchangeKind.java @@ -0,0 +1,10 @@ +package io.substrait.dialect; + +/** Exchange kinds for {@code EXCHANGE} relations. */ +public enum ExchangeKind { + SCATTER_BY_FIELDS, + SINGLE_TARGET, + MULTI_TARGET, + ROUND_ROBIN, + BROADCAST +} diff --git a/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java b/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java new file mode 100644 index 000000000..8651e8df6 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java @@ -0,0 +1,20 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import org.immutables.value.Value; + +/** Execution-behavior configuration for a dialect. */ +@JsonDeserialize(as = ImmutableExecutionBehavior.class) +@JsonSerialize(as = ImmutableExecutionBehavior.class) +@Value.Immutable +public abstract class ExecutionBehavior { + @JsonProperty("supported_variable_evaluation_mode") + public abstract List supportedVariableEvaluationMode(); + + public static ImmutableExecutionBehavior.Builder builder() { + return ImmutableExecutionBehavior.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/ExpandFieldType.java b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java new file mode 100644 index 000000000..61d05a147 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java @@ -0,0 +1,7 @@ +package io.substrait.dialect; + +/** Field types for {@code EXPAND} relations. */ +public enum ExpandFieldType { + SWITCHING_FIELD, + CONSTANT_FIELD +} diff --git a/core/src/main/java/io/substrait/dialect/ExpressionKind.java b/core/src/main/java/io/substrait/dialect/ExpressionKind.java new file mode 100644 index 000000000..ba767c989 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/ExpressionKind.java @@ -0,0 +1,18 @@ +package io.substrait.dialect; + +/** The kinds of expressions a dialect can declare support for. */ +public enum ExpressionKind { + LITERAL, + SELECTION, + SCALAR_FUNCTION, + WINDOW_FUNCTION, + IF_THEN, + SWITCH, + SINGULAR_OR_LIST, + MULTI_OR_LIST, + CAST, + SUBQUERY, + NESTED, + DYNAMIC_PARAMETER, + EXECUTION_CONTEXT_VARIABLE +} diff --git a/core/src/main/java/io/substrait/dialect/JoinType.java b/core/src/main/java/io/substrait/dialect/JoinType.java new file mode 100644 index 000000000..3980addf3 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/JoinType.java @@ -0,0 +1,17 @@ +package io.substrait.dialect; + +/** Join types for join relations. */ +public enum JoinType { + INNER, + OUTER, + LEFT, + RIGHT, + LEFT_SEMI, + RIGHT_SEMI, + LEFT_ANTI, + RIGHT_ANTI, + LEFT_SINGLE, + RIGHT_SINGLE, + LEFT_MARK, + RIGHT_MARK +} diff --git a/core/src/main/java/io/substrait/dialect/NestedType.java b/core/src/main/java/io/substrait/dialect/NestedType.java new file mode 100644 index 000000000..796a31202 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/NestedType.java @@ -0,0 +1,8 @@ +package io.substrait.dialect; + +/** Nested types for {@code NESTED} expressions. */ +public enum NestedType { + STRUCT, + LIST, + MAP +} diff --git a/core/src/main/java/io/substrait/dialect/Notation.java b/core/src/main/java/io/substrait/dialect/Notation.java new file mode 100644 index 000000000..8648679e8 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/Notation.java @@ -0,0 +1,9 @@ +package io.substrait.dialect; + +/** How a system function is written in the system's own syntax. */ +public enum Notation { + INFIX, + POSTFIX, + PREFIX, + FUNCTION +} diff --git a/core/src/main/java/io/substrait/dialect/ReadType.java b/core/src/main/java/io/substrait/dialect/ReadType.java new file mode 100644 index 000000000..41843b881 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/ReadType.java @@ -0,0 +1,10 @@ +package io.substrait.dialect; + +/** Read types for {@code READ} relations. */ +public enum ReadType { + VIRTUAL_TABLE, + LOCAL_FILES, + NAMED_TABLE, + EXTENSION_TABLE, + ICEBERG_TABLE +} diff --git a/core/src/main/java/io/substrait/dialect/RelationKind.java b/core/src/main/java/io/substrait/dialect/RelationKind.java new file mode 100644 index 000000000..c823e0b76 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/RelationKind.java @@ -0,0 +1,27 @@ +package io.substrait.dialect; + +/** The kinds of relations a dialect can declare support for. */ +public enum RelationKind { + READ, + FILTER, + FETCH, + AGGREGATE, + SORT, + JOIN, + PROJECT, + SET, + CROSS, + REFERENCE, + WRITE, + DDL, + UPDATE, + HASH_JOIN, + MERGE_JOIN, + NESTED_LOOP_JOIN, + CONSISTENT_PARTITION_WINDOW, + EXCHANGE, + EXPAND, + EXTENSION_SINGLE, + EXTENSION_MULTI, + EXTENSION_LEAF +} diff --git a/core/src/main/java/io/substrait/dialect/SetOperation.java b/core/src/main/java/io/substrait/dialect/SetOperation.java new file mode 100644 index 000000000..a1112f458 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SetOperation.java @@ -0,0 +1,13 @@ +package io.substrait.dialect; + +/** Set operations for {@code SET} relations. */ +public enum SetOperation { + MINUS_PRIMARY, + MINUS_PRIMARY_ALL, + MINUS_MULTISET, + INTERSECTION_PRIMARY, + INTERSECTION_MULTISET, + INTERSECTION_MULTISET_ALL, + UNION_DISTINCT, + UNION_ALL +} diff --git a/core/src/main/java/io/substrait/dialect/SubqueryType.java b/core/src/main/java/io/substrait/dialect/SubqueryType.java new file mode 100644 index 000000000..60892f396 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SubqueryType.java @@ -0,0 +1,9 @@ +package io.substrait.dialect; + +/** Subquery types for {@code SUBQUERY} expressions. */ +public enum SubqueryType { + SCALAR, + IN_PREDICATE, + SET_PREDICATE, + SET_COMPARISON +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpression.java b/core/src/main/java/io/substrait/dialect/SupportedExpression.java new file mode 100644 index 000000000..f84db23e8 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedExpression.java @@ -0,0 +1,62 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * A {@code supported_expressions} entry. Serializes as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +@JsonDeserialize(using = SupportedExpressionDeserializer.class) +@JsonSerialize(using = SupportedExpressionSerializer.class) +@Value.Immutable +public abstract class SupportedExpression { + public abstract ExpressionKind expression(); + + public abstract Optional> metadata(); + + /** Permissible failure options for {@code CAST}. */ + public abstract List failureOptions(); + + /** Subquery types for {@code SUBQUERY}. */ + public abstract List subqueryTypes(); + + /** Nested types for {@code NESTED}. */ + public abstract List nestedTypes(); + + /** Variable types for {@code EXECUTION_CONTEXT_VARIABLE}. */ + public abstract List variableTypes(); + + /** + * The schema's {@code execution_context_variable} entry forbids a {@code metadata} field, so + * reject the combination rather than silently dropping it on serialization. + */ + @Value.Check + protected void checkMetadata() { + if (expression() == ExpressionKind.EXECUTION_CONTEXT_VARIABLE && metadata().isPresent()) { + throw new IllegalArgumentException( + "EXECUTION_CONTEXT_VARIABLE expressions cannot carry metadata."); + } + } + + /** Whether this entry can be written as a bare enum string (no extra configuration). */ + public boolean isBare() { + return !metadata().isPresent() + && failureOptions().isEmpty() + && subqueryTypes().isEmpty() + && nestedTypes().isEmpty() + && variableTypes().isEmpty(); + } + + public static SupportedExpression of(ExpressionKind expression) { + return builder().expression(expression).build(); + } + + public static ImmutableSupportedExpression.Builder builder() { + return ImmutableSupportedExpression.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java index 3621faf89..eaa8a0b72 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedExpressionDeserializer.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; -import io.substrait.dialect.Dialect.ExpressionKind; -import io.substrait.dialect.Dialect.SupportedExpression; import java.io.IOException; import java.util.Map; @@ -14,7 +12,7 @@ * {@code LITERAL}) or a configuration object (e.g. {@code {expression: CAST, failure_options: * [RETURN_NULL]}}). */ -public class SupportedExpressionDeserializer extends JsonDeserializer { +class SupportedExpressionDeserializer extends JsonDeserializer { @Override public SupportedExpression deserialize(JsonParser p, DeserializationContext ctxt) @@ -25,21 +23,20 @@ public SupportedExpression deserialize(JsonParser p, DeserializationContext ctxt } ExpressionKind kind = ExpressionKind.valueOf(node.get("expression").asText()); - ImmutableDialect.SupportedExpression.Builder builder = - SupportedExpression.builder().expression(kind); + ImmutableSupportedExpression.Builder builder = SupportedExpression.builder().expression(kind); - Map metadata = DialectJsonSupport.readMetadata(p, node); + Map metadata = DialectJsonSupport.readMetadata(node); if (metadata != null) { builder.metadata(metadata); } builder.addAllFailureOptions( - DialectJsonSupport.readEnums(node.get("failure_options"), Dialect.CastFailureOption.class)); + DialectJsonSupport.readEnums(node.get("failure_options"), CastFailureOption.class)); builder.addAllSubqueryTypes( - DialectJsonSupport.readEnums(node.get("subquery_types"), Dialect.SubqueryType.class)); + DialectJsonSupport.readEnums(node.get("subquery_types"), SubqueryType.class)); builder.addAllNestedTypes( - DialectJsonSupport.readEnums(node.get("nested_types"), Dialect.NestedType.class)); + DialectJsonSupport.readEnums(node.get("nested_types"), NestedType.class)); builder.addAllVariableTypes( - DialectJsonSupport.readEnums(node.get("variable_types"), Dialect.VariableType.class)); + DialectJsonSupport.readEnums(node.get("variable_types"), VariableType.class)); return builder.build(); } diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java index f6c80d0e5..a2a088b80 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedExpressionSerializer.java @@ -3,15 +3,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import io.substrait.dialect.Dialect.ExpressionKind; -import io.substrait.dialect.Dialect.SupportedExpression; import java.io.IOException; /** * Serializes a {@code supported_expressions} entry as a bare enum string when it carries no * configuration, or as a configuration object otherwise. */ -public class SupportedExpressionSerializer extends JsonSerializer { +class SupportedExpressionSerializer extends JsonSerializer { @Override public void serialize(SupportedExpression value, JsonGenerator gen, SerializerProvider provider) @@ -35,9 +33,8 @@ public void serialize(SupportedExpression value, JsonGenerator gen, SerializerPr if (!value.variableTypes().isEmpty()) { DialectJsonSupport.writeEnumArray(gen, "variable_types", value.variableTypes()); } - // The schema forbids `metadata` on EXECUTION_CONTEXT_VARIABLE, so only emit it for others. - if (value.metadata().isPresent() - && value.expression() != ExpressionKind.EXECUTION_CONTEXT_VARIABLE) { + // EXECUTION_CONTEXT_VARIABLE cannot carry metadata (enforced by SupportedExpression). + if (value.metadata().isPresent()) { DialectJsonSupport.writeMetadata(gen, provider, value.metadata().get()); } gen.writeEndObject(); diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelation.java b/core/src/main/java/io/substrait/dialect/SupportedRelation.java new file mode 100644 index 000000000..e2e914dd5 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedRelation.java @@ -0,0 +1,93 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * A {@code supported_relations} entry. Serializes as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +@JsonDeserialize(using = SupportedRelationDeserializer.class) +@JsonSerialize(using = SupportedRelationSerializer.class) +@Value.Immutable +public abstract class SupportedRelation { + public abstract RelationKind relation(); + + public abstract Optional> metadata(); + + /** + * Join types for {@code JOIN}, {@code HASH_JOIN}, {@code MERGE_JOIN}, {@code NESTED_LOOP_JOIN}. + */ + public abstract List joinTypes(); + + /** Read types for {@code READ}. */ + public abstract List readTypes(); + + /** Set operations for {@code SET}. */ + public abstract List operations(); + + /** Write types for {@code WRITE} (serialized as {@code write_types}). */ + public abstract List writeTypes(); + + /** Operable object types for {@code DDL} (also serialized as {@code write_types}). */ + public abstract List ddlWriteTypes(); + + /** Exchange kinds for {@code EXCHANGE}. */ + public abstract List kinds(); + + /** Field types for {@code EXPAND}. */ + public abstract List fieldTypes(); + + /** Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. */ + public abstract List messageTypes(); + + /** + * {@code write_types} is a single YAML field shared between {@code WRITE} and {@code DDL} + * relations, each carrying a different enum. Forbid populating both so serialization stays + * unambiguous. + */ + @Value.Check + protected void checkWriteTypes() { + if (!writeTypes().isEmpty() && !ddlWriteTypes().isEmpty()) { + throw new IllegalArgumentException( + "A supported relation cannot set both writeTypes and ddlWriteTypes; " + + "they share the write_types field."); + } + } + + /** + * Whether this entry can be written as a bare enum string. Extension relations are never bare: + * they are absent from the schema's bare-enum list. + */ + public boolean isBare() { + switch (relation()) { + case EXTENSION_SINGLE: + case EXTENSION_MULTI: + case EXTENSION_LEAF: + return false; + default: + break; + } + return !metadata().isPresent() + && joinTypes().isEmpty() + && readTypes().isEmpty() + && operations().isEmpty() + && writeTypes().isEmpty() + && ddlWriteTypes().isEmpty() + && kinds().isEmpty() + && fieldTypes().isEmpty() + && messageTypes().isEmpty(); + } + + public static SupportedRelation of(RelationKind relation) { + return builder().relation(relation).build(); + } + + public static ImmutableSupportedRelation.Builder builder() { + return ImmutableSupportedRelation.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java index 9c5529f47..53609b581 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedRelationDeserializer.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; -import io.substrait.dialect.Dialect.RelationKind; -import io.substrait.dialect.Dialect.SupportedRelation; import java.io.IOException; import java.util.Map; @@ -13,7 +11,7 @@ * Deserializes a {@code supported_relations} entry, which is either a bare enum string (e.g. {@code * FILTER}) or a configuration object (e.g. {@code {relation: JOIN, join_types: [INNER]}}). */ -public class SupportedRelationDeserializer extends JsonDeserializer { +class SupportedRelationDeserializer extends JsonDeserializer { @Override public SupportedRelation deserialize(JsonParser p, DeserializationContext ctxt) @@ -24,31 +22,28 @@ public SupportedRelation deserialize(JsonParser p, DeserializationContext ctxt) } RelationKind kind = RelationKind.valueOf(node.get("relation").asText()); - ImmutableDialect.SupportedRelation.Builder builder = SupportedRelation.builder().relation(kind); + ImmutableSupportedRelation.Builder builder = SupportedRelation.builder().relation(kind); - Map metadata = DialectJsonSupport.readMetadata(p, node); + Map metadata = DialectJsonSupport.readMetadata(node); if (metadata != null) { builder.metadata(metadata); } - builder.addAllJoinTypes( - DialectJsonSupport.readEnums(node.get("join_types"), Dialect.JoinType.class)); - builder.addAllReadTypes( - DialectJsonSupport.readEnums(node.get("read_types"), Dialect.ReadType.class)); + builder.addAllJoinTypes(DialectJsonSupport.readEnums(node.get("join_types"), JoinType.class)); + builder.addAllReadTypes(DialectJsonSupport.readEnums(node.get("read_types"), ReadType.class)); builder.addAllOperations( - DialectJsonSupport.readEnums(node.get("operations"), Dialect.SetOperation.class)); - builder.addAllKinds( - DialectJsonSupport.readEnums(node.get("kinds"), Dialect.ExchangeKind.class)); + DialectJsonSupport.readEnums(node.get("operations"), SetOperation.class)); + builder.addAllKinds(DialectJsonSupport.readEnums(node.get("kinds"), ExchangeKind.class)); builder.addAllFieldTypes( - DialectJsonSupport.readEnums(node.get("field_types"), Dialect.ExpandFieldType.class)); + DialectJsonSupport.readEnums(node.get("field_types"), ExpandFieldType.class)); builder.addAllMessageTypes(DialectJsonSupport.readStrings(node.get("message_types"))); // `write_types` is shared between WRITE and DDL but uses a different enum for each. if (kind == RelationKind.DDL) { builder.addAllDdlWriteTypes( - DialectJsonSupport.readEnums(node.get("write_types"), Dialect.DdlWriteType.class)); + DialectJsonSupport.readEnums(node.get("write_types"), DdlWriteType.class)); } else { builder.addAllWriteTypes( - DialectJsonSupport.readEnums(node.get("write_types"), Dialect.WriteType.class)); + DialectJsonSupport.readEnums(node.get("write_types"), WriteType.class)); } return builder.build(); diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java index 6c5b3e35a..567b7a8a6 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedRelationSerializer.java @@ -3,14 +3,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import io.substrait.dialect.Dialect.SupportedRelation; import java.io.IOException; /** * Serializes a {@code supported_relations} entry as a bare enum string when it carries no * configuration, or as a configuration object otherwise. */ -public class SupportedRelationSerializer extends JsonSerializer { +class SupportedRelationSerializer extends JsonSerializer { @Override public void serialize(SupportedRelation value, JsonGenerator gen, SerializerProvider provider) @@ -31,6 +30,7 @@ public void serialize(SupportedRelation value, JsonGenerator gen, SerializerProv if (!value.operations().isEmpty()) { DialectJsonSupport.writeEnumArray(gen, "operations", value.operations()); } + // WRITE and DDL share the `write_types` field; SupportedRelation forbids populating both. if (!value.writeTypes().isEmpty()) { DialectJsonSupport.writeEnumArray(gen, "write_types", value.writeTypes()); } diff --git a/core/src/main/java/io/substrait/dialect/SupportedType.java b/core/src/main/java/io/substrait/dialect/SupportedType.java new file mode 100644 index 000000000..b3c058729 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SupportedType.java @@ -0,0 +1,48 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * A {@code supported_types} entry. Serializes as a bare enum string when it carries no + * configuration, or as a configuration object otherwise. + */ +@JsonDeserialize(using = SupportedTypeDeserializer.class) +@JsonSerialize(using = SupportedTypeSerializer.class) +@Value.Immutable +public abstract class SupportedType { + public abstract TypeKind type(); + + public abstract Optional> metadata(); + + public abstract Optional systemMetadata(); + + public abstract Optional maxPrecision(); + + /** Dependency (alias) where a {@code USER_DEFINED} type is declared. */ + public abstract Optional source(); + + /** The name of a {@code USER_DEFINED} type as declared in the extension it is defined in. */ + public abstract Optional name(); + + /** Whether this entry can be written as a bare enum string (no extra configuration). */ + public boolean isBare() { + return type() != TypeKind.USER_DEFINED + && !metadata().isPresent() + && !systemMetadata().isPresent() + && !maxPrecision().isPresent() + && !source().isPresent() + && !name().isPresent(); + } + + public static SupportedType of(TypeKind type) { + return builder().type(type).build(); + } + + public static ImmutableSupportedType.Builder builder() { + return ImmutableSupportedType.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java b/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java index c94666bb3..e54a79fe7 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedTypeDeserializer.java @@ -4,10 +4,6 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.substrait.dialect.Dialect.SupportedType; -import io.substrait.dialect.Dialect.SystemTypeMetadata; -import io.substrait.dialect.Dialect.TypeKind; import java.io.IOException; import java.util.Map; @@ -15,7 +11,7 @@ * Deserializes a {@code supported_types} entry, which is either a bare enum string (e.g. {@code * BOOL}) or a configuration object (e.g. {@code {type: PRECISION_TIMESTAMP, max_precision: 9}}). */ -public class SupportedTypeDeserializer extends JsonDeserializer { +class SupportedTypeDeserializer extends JsonDeserializer { @Override public SupportedType deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { @@ -24,17 +20,17 @@ public SupportedType deserialize(JsonParser p, DeserializationContext ctxt) thro return SupportedType.of(TypeKind.valueOf(node.asText())); } - ObjectMapper mapper = (ObjectMapper) p.getCodec(); - ImmutableDialect.SupportedType.Builder builder = + ImmutableSupportedType.Builder builder = SupportedType.builder().type(TypeKind.valueOf(node.get("type").asText())); - Map metadata = DialectJsonSupport.readMetadata(p, node); + Map metadata = DialectJsonSupport.readMetadata(node); if (metadata != null) { builder.metadata(metadata); } if (node.hasNonNull("system_metadata")) { builder.systemMetadata( - mapper.convertValue(node.get("system_metadata"), SystemTypeMetadata.class)); + DialectJsonSupport.MAPPER.convertValue( + node.get("system_metadata"), SystemTypeMetadata.class)); } if (node.hasNonNull("max_precision")) { builder.maxPrecision(node.get("max_precision").asInt()); diff --git a/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java b/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java index 03e9a0b16..698a74be0 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java +++ b/core/src/main/java/io/substrait/dialect/SupportedTypeSerializer.java @@ -3,14 +3,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import io.substrait.dialect.Dialect.SupportedType; import java.io.IOException; /** * Serializes a {@code supported_types} entry as a bare enum string when it carries no * configuration, or as a configuration object otherwise. */ -public class SupportedTypeSerializer extends JsonSerializer { +class SupportedTypeSerializer extends JsonSerializer { @Override public void serialize(SupportedType value, JsonGenerator gen, SerializerProvider provider) diff --git a/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java b/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java new file mode 100644 index 000000000..71b226e49 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java @@ -0,0 +1,23 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.Optional; +import org.immutables.value.Value; + +/** System-specific metadata for a dialect function. */ +@JsonDeserialize(as = ImmutableSystemFunctionMetadata.class) +@JsonSerialize(as = ImmutableSystemFunctionMetadata.class) +@Value.Immutable +public abstract class SystemFunctionMetadata { + public abstract Optional name(); + + @Value.Default + public Notation notation() { + return Notation.FUNCTION; + } + + public static ImmutableSystemFunctionMetadata.Builder builder() { + return ImmutableSystemFunctionMetadata.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java b/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java new file mode 100644 index 000000000..4ae9abdf9 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java @@ -0,0 +1,22 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.Optional; +import org.immutables.value.Value; + +/** System-specific metadata for a dialect type. */ +@JsonDeserialize(as = ImmutableSystemTypeMetadata.class) +@JsonSerialize(as = ImmutableSystemTypeMetadata.class) +@Value.Immutable +public abstract class SystemTypeMetadata { + public abstract Optional name(); + + @JsonProperty("supported_as_column") + public abstract Optional supportedAsColumn(); + + public static ImmutableSystemTypeMetadata.Builder builder() { + return ImmutableSystemTypeMetadata.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/TypeKind.java b/core/src/main/java/io/substrait/dialect/TypeKind.java new file mode 100644 index 000000000..9ecda820c --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/TypeKind.java @@ -0,0 +1,31 @@ +package io.substrait.dialect; + +/** The kinds of types a dialect can declare support for. */ +public enum TypeKind { + BOOL, + I8, + I16, + I32, + I64, + FP32, + FP64, + BINARY, + FIXED_BINARY, + STRING, + VARCHAR, + FIXED_CHAR, + PRECISION_TIME, + PRECISION_TIMESTAMP, + PRECISION_TIMESTAMP_TZ, + DATE, + TIME, + INTERVAL_COMPOUND, + INTERVAL_DAY, + INTERVAL_YEAR, + UUID, + DECIMAL, + STRUCT, + LIST, + MAP, + USER_DEFINED +} diff --git a/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java b/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java new file mode 100644 index 000000000..3245fb507 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java @@ -0,0 +1,7 @@ +package io.substrait.dialect; + +/** When execution context variables are evaluated. */ +public enum VariableEvaluationMode { + PER_PLAN, + PER_RECORD +} diff --git a/core/src/main/java/io/substrait/dialect/VariableType.java b/core/src/main/java/io/substrait/dialect/VariableType.java new file mode 100644 index 000000000..25fe5ff37 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/VariableType.java @@ -0,0 +1,8 @@ +package io.substrait.dialect; + +/** Variable types for {@code EXECUTION_CONTEXT_VARIABLE} expressions. */ +public enum VariableType { + CURRENT_TIMESTAMP, + CURRENT_TIMEZONE, + CURRENT_DATE +} diff --git a/core/src/main/java/io/substrait/dialect/Variadic.java b/core/src/main/java/io/substrait/dialect/Variadic.java new file mode 100644 index 000000000..8ed2ab8ed --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/Variadic.java @@ -0,0 +1,20 @@ +package io.substrait.dialect; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.OptionalInt; +import org.immutables.value.Value; + +/** Variadic argument bounds for a dialect function. */ +@JsonDeserialize(as = ImmutableVariadic.class) +@JsonSerialize(as = ImmutableVariadic.class) +@Value.Immutable +public abstract class Variadic { + public abstract OptionalInt min(); + + public abstract OptionalInt max(); + + public static ImmutableVariadic.Builder builder() { + return ImmutableVariadic.builder(); + } +} diff --git a/core/src/main/java/io/substrait/dialect/WriteType.java b/core/src/main/java/io/substrait/dialect/WriteType.java new file mode 100644 index 000000000..1d17dd424 --- /dev/null +++ b/core/src/main/java/io/substrait/dialect/WriteType.java @@ -0,0 +1,7 @@ +package io.substrait.dialect; + +/** Write types for {@code WRITE} relations. */ +public enum WriteType { + NAMED_TABLE, + EXTENSION_TABLE +} diff --git a/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java index 14f4047d2..46806bb29 100644 --- a/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java +++ b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java @@ -4,10 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.substrait.dialect.Dialect.DialectDocument; -import io.substrait.dialect.Dialect.JoinType; -import io.substrait.dialect.Dialect.RelationKind; -import io.substrait.dialect.Dialect.SupportedRelation; import org.junit.jupiter.api.Test; /** @@ -18,10 +14,8 @@ class DialectBareStringCollapseTest { @Test void configFreeRelationSerializesAsBareString() { - DialectDocument dialect = - DialectDocument.builder() - .addSupportedRelations(SupportedRelation.of(RelationKind.FILTER)) - .build(); + Dialect dialect = + Dialect.builder().addSupportedRelations(SupportedRelation.of(RelationKind.FILTER)).build(); String yaml = Dialect.toYaml(dialect); @@ -34,8 +28,8 @@ void configFreeRelationSerializesAsBareString() { @Test void configuredRelationSerializesAsObject() { - DialectDocument dialect = - DialectDocument.builder() + Dialect dialect = + Dialect.builder() .addSupportedRelations( SupportedRelation.builder() .relation(RelationKind.JOIN) @@ -57,8 +51,7 @@ void extensionRelationIsNeverBare() { SupportedRelation.builder().relation(RelationKind.EXTENSION_LEAF).build(); assertFalse(extension.isBare()); - String yaml = - Dialect.toYaml(DialectDocument.builder().addSupportedRelations(extension).build()); + String yaml = Dialect.toYaml(Dialect.builder().addSupportedRelations(extension).build()); assertTrue( yaml.contains("relation: \"EXTENSION_LEAF\"") || yaml.contains("relation: EXTENSION_LEAF"), yaml); diff --git a/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java index fab4ed2fa..58d17c253 100644 --- a/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java +++ b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java @@ -4,30 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.networknt.schema.Error; -import io.substrait.dialect.Dialect.CastFailureOption; -import io.substrait.dialect.Dialect.DdlWriteType; -import io.substrait.dialect.Dialect.DialectDocument; -import io.substrait.dialect.Dialect.DialectFunction; -import io.substrait.dialect.Dialect.ExchangeKind; -import io.substrait.dialect.Dialect.ExecutionBehavior; -import io.substrait.dialect.Dialect.ExpandFieldType; -import io.substrait.dialect.Dialect.ExpressionKind; -import io.substrait.dialect.Dialect.JoinType; -import io.substrait.dialect.Dialect.NestedType; -import io.substrait.dialect.Dialect.Notation; -import io.substrait.dialect.Dialect.ReadType; -import io.substrait.dialect.Dialect.RelationKind; -import io.substrait.dialect.Dialect.SetOperation; -import io.substrait.dialect.Dialect.SubqueryType; -import io.substrait.dialect.Dialect.SupportedExpression; -import io.substrait.dialect.Dialect.SupportedRelation; -import io.substrait.dialect.Dialect.SupportedType; -import io.substrait.dialect.Dialect.SystemFunctionMetadata; -import io.substrait.dialect.Dialect.SystemTypeMetadata; -import io.substrait.dialect.Dialect.TypeKind; -import io.substrait.dialect.Dialect.VariableEvaluationMode; -import io.substrait.dialect.Dialect.VariableType; -import io.substrait.dialect.Dialect.WriteType; import java.util.List; import org.junit.jupiter.api.Test; @@ -37,8 +13,8 @@ */ class DialectRoundTripTest { - private static DialectDocument sampleDialect() { - return DialectDocument.builder() + private static Dialect sampleDialect() { + return Dialect.builder() .name("Round Trip Dialect") .putDependencies("arithmetic", "extension:io.substrait:functions_arithmetic") .putDependencies("spark", "extension:substrait:spark") @@ -154,14 +130,14 @@ private static DialectDocument sampleDialect() { @Test void roundTripThroughYamlAndSchema() { - DialectDocument original = sampleDialect(); + Dialect original = sampleDialect(); String yaml = Dialect.toYaml(original); List errors = SchemaValidator.validate(yaml); assertTrue(errors.isEmpty(), () -> "Generated dialect failed schema validation: " + errors); - DialectDocument parsed = Dialect.load(yaml); + Dialect parsed = Dialect.load(yaml); assertEquals(original, parsed); } } diff --git a/core/src/test/java/io/substrait/dialect/DialectValidationTest.java b/core/src/test/java/io/substrait/dialect/DialectValidationTest.java new file mode 100644 index 000000000..2ce573732 --- /dev/null +++ b/core/src/test/java/io/substrait/dialect/DialectValidationTest.java @@ -0,0 +1,33 @@ +package io.substrait.dialect; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** Verifies the build-time guards that keep dialect entries serializable and schema-valid. */ +class DialectValidationTest { + + @Test + void executionContextVariableRejectsMetadata() { + assertThrows( + IllegalArgumentException.class, + () -> + SupportedExpression.builder() + .expression(ExpressionKind.EXECUTION_CONTEXT_VARIABLE) + .metadata(Map.of("k", "v")) + .build()); + } + + @Test + void writeAndDdlWriteTypesAreMutuallyExclusive() { + assertThrows( + IllegalArgumentException.class, + () -> + SupportedRelation.builder() + .relation(RelationKind.WRITE) + .addWriteTypes(WriteType.NAMED_TABLE) + .addDdlWriteTypes(DdlWriteType.NAMED_OBJECT) + .build()); + } +} diff --git a/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java b/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java index c5681c871..c84d99e41 100644 --- a/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java +++ b/core/src/test/java/io/substrait/dialect/SparkDialectParseTest.java @@ -4,14 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.networknt.schema.Error; -import io.substrait.dialect.Dialect.DialectDocument; -import io.substrait.dialect.Dialect.ExpressionKind; -import io.substrait.dialect.Dialect.RelationKind; -import io.substrait.dialect.Dialect.SubqueryType; -import io.substrait.dialect.Dialect.SupportedExpression; -import io.substrait.dialect.Dialect.SupportedRelation; -import io.substrait.dialect.Dialect.SupportedType; -import io.substrait.dialect.Dialect.TypeKind; import java.util.List; import org.junit.jupiter.api.Test; @@ -21,7 +13,7 @@ */ class SparkDialectParseTest { - private static final DialectDocument SPARK = Dialect.loadResource("/dialect/spark_dialect.yaml"); + private static final Dialect SPARK = Dialect.loadResource("/dialect/spark_dialect.yaml"); @Test void parsesTopLevelFields() { @@ -37,7 +29,7 @@ void parsesConfiguredRelationsAndExpressions() { .filter(r -> r.relation() == RelationKind.JOIN) .findFirst() .orElseThrow(); - assertTrue(join.joinTypes().contains(Dialect.JoinType.INNER)); + assertTrue(join.joinTypes().contains(JoinType.INNER)); SupportedExpression subquery = SPARK.supportedExpressions().stream() diff --git a/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java b/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java index 16f52a330..d0409d4b2 100644 --- a/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java +++ b/core/src/test/java/io/substrait/dialect/SpecDialectFixturesTest.java @@ -4,13 +4,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.networknt.schema.Error; -import io.substrait.dialect.Dialect.DialectDocument; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Scanner; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -26,10 +24,7 @@ private static String readResource(String resourcePath) { if (stream == null) { throw new IllegalStateException("Fixture not found on classpath: " + resourcePath); } - try (Scanner scanner = new Scanner(stream, StandardCharsets.UTF_8.name())) { - scanner.useDelimiter("\\A"); - return scanner.hasNext() ? scanner.next() : ""; - } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -53,7 +48,7 @@ void roundTrips(String fixture) { assertTrue(fixtureErrors.isEmpty(), () -> fixture + " is not schema-valid: " + fixtureErrors); // Parse, re-serialize, and confirm the output is still schema-valid. - DialectDocument parsed = Dialect.loadResource(resourcePath); + Dialect parsed = Dialect.loadResource(resourcePath); String reserialized = Dialect.toYaml(parsed); List roundTripErrors = SchemaValidator.validate(reserialized); assertTrue( From 8a41635086d236fbd566d497a851b24de36477a3 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Mon, 29 Jun 2026 09:09:59 +0200 Subject: [PATCH 3/3] docs(core): add javadoc to dialect public API The upstream main now runs `core:javadoc` with `-Xdoclint:all -Xwerror` (added in #949), which fails the build on any missing-javadoc warning. Flattening the dialect model into top-level public types exposed their members to this check, so document every public enum constant and every public accessor / factory method (with @param/@return) following the existing convention (e.g. io.substrait.hint.Hint). --- .../substrait/dialect/CastFailureOption.java | 2 + .../io/substrait/dialect/DdlWriteType.java | 2 + .../java/io/substrait/dialect/Dialect.java | 86 ++++++++++++++++++- .../io/substrait/dialect/DialectFunction.java | 43 +++++++++- .../io/substrait/dialect/ExchangeKind.java | 5 ++ .../substrait/dialect/ExecutionBehavior.java | 10 +++ .../io/substrait/dialect/ExpandFieldType.java | 2 + .../io/substrait/dialect/ExpressionKind.java | 13 +++ .../java/io/substrait/dialect/JoinType.java | 12 +++ .../java/io/substrait/dialect/NestedType.java | 3 + .../java/io/substrait/dialect/Notation.java | 4 + .../java/io/substrait/dialect/ReadType.java | 5 ++ .../io/substrait/dialect/RelationKind.java | 22 +++++ .../io/substrait/dialect/SetOperation.java | 8 ++ .../io/substrait/dialect/SubqueryType.java | 4 + .../dialect/SupportedExpression.java | 51 +++++++++-- .../substrait/dialect/SupportedRelation.java | 67 +++++++++++++-- .../io/substrait/dialect/SupportedType.java | 49 ++++++++++- .../dialect/SystemFunctionMetadata.java | 15 ++++ .../substrait/dialect/SystemTypeMetadata.java | 15 ++++ .../java/io/substrait/dialect/TypeKind.java | 26 ++++++ .../dialect/VariableEvaluationMode.java | 2 + .../io/substrait/dialect/VariableType.java | 3 + .../java/io/substrait/dialect/Variadic.java | 15 ++++ .../java/io/substrait/dialect/WriteType.java | 2 + 25 files changed, 444 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/substrait/dialect/CastFailureOption.java b/core/src/main/java/io/substrait/dialect/CastFailureOption.java index a1beba2ac..f72aa9be6 100644 --- a/core/src/main/java/io/substrait/dialect/CastFailureOption.java +++ b/core/src/main/java/io/substrait/dialect/CastFailureOption.java @@ -2,6 +2,8 @@ /** Permissible failure options for {@code CAST} expressions. */ public enum CastFailureOption { + /** Return null when a cast fails. */ RETURN_NULL, + /** Throw an exception when a cast fails. */ THROW_EXCEPTION } diff --git a/core/src/main/java/io/substrait/dialect/DdlWriteType.java b/core/src/main/java/io/substrait/dialect/DdlWriteType.java index be5a182d4..126aef5c1 100644 --- a/core/src/main/java/io/substrait/dialect/DdlWriteType.java +++ b/core/src/main/java/io/substrait/dialect/DdlWriteType.java @@ -2,6 +2,8 @@ /** Operable object types for {@code DDL} relations. */ public enum DdlWriteType { + /** A named DDL object. */ NAMED_OBJECT, + /** An extension-defined DDL object. */ EXTENSION_OBJECT } diff --git a/core/src/main/java/io/substrait/dialect/Dialect.java b/core/src/main/java/io/substrait/dialect/Dialect.java index 75bae97be..845f8d76a 100644 --- a/core/src/main/java/io/substrait/dialect/Dialect.java +++ b/core/src/main/java/io/substrait/dialect/Dialect.java @@ -27,38 +27,98 @@ @Value.Immutable public abstract class Dialect { + /** + * The name of the dialect, if any. + * + * @return the optional dialect name + */ public abstract Optional name(); + /** + * Free-form metadata associated with the dialect, if any. + * + * @return the optional metadata + */ public abstract Optional> metadata(); + /** + * The dependencies referenced by the dialect, keyed by alias. + * + * @return the dependency aliases mapped to their URIs + */ public abstract Map dependencies(); + /** + * The types supported by the dialect. + * + * @return the supported types + */ @JsonProperty("supported_types") public abstract List supportedTypes(); + /** + * The relations supported by the dialect. + * + * @return the supported relations + */ @JsonProperty("supported_relations") public abstract List supportedRelations(); + /** + * The expressions supported by the dialect. + * + * @return the supported expressions + */ @JsonProperty("supported_expressions") public abstract List supportedExpressions(); + /** + * The scalar functions supported by the dialect. + * + * @return the supported scalar functions + */ @JsonProperty("supported_scalar_functions") public abstract List supportedScalarFunctions(); + /** + * The aggregate functions supported by the dialect. + * + * @return the supported aggregate functions + */ @JsonProperty("supported_aggregate_functions") public abstract List supportedAggregateFunctions(); + /** + * The window functions supported by the dialect. + * + * @return the supported window functions + */ @JsonProperty("supported_window_functions") public abstract List supportedWindowFunctions(); + /** + * The execution-behavior configuration of the dialect, if any. + * + * @return the optional execution behavior + */ @JsonProperty("supported_execution_behavior") public abstract Optional supportedExecutionBehavior(); + /** + * Creates a builder for {@link Dialect}. + * + * @return a new builder + */ public static ImmutableDialect.Builder builder() { return ImmutableDialect.builder(); } - /** Parse a dialect from YAML content. */ + /** + * Parse a dialect from YAML content. + * + * @param content the YAML content + * @return the parsed dialect + */ public static Dialect load(String content) { try { return DialectJsonSupport.MAPPER.readValue(content, Dialect.class); @@ -70,6 +130,9 @@ public static Dialect load(String content) { /** * Parse a dialect from a YAML stream. The caller retains ownership of the stream; it is not * closed by this method. + * + * @param stream the YAML input stream + * @return the parsed dialect */ public static Dialect load(InputStream stream) { try { @@ -82,7 +145,12 @@ public static Dialect load(InputStream stream) { } } - /** Parse a dialect from a YAML file on disk. */ + /** + * Parse a dialect from a YAML file on disk. + * + * @param path the path to the YAML file + * @return the parsed dialect + */ public static Dialect loadFromFile(Path path) { try { return DialectJsonSupport.MAPPER.readValue(path.toFile(), Dialect.class); @@ -91,7 +159,12 @@ public static Dialect loadFromFile(Path path) { } } - /** Parse a dialect from a classpath resource. */ + /** + * Parse a dialect from a classpath resource. + * + * @param resourcePath the classpath resource path + * @return the parsed dialect + */ public static Dialect loadResource(String resourcePath) { try (InputStream stream = Dialect.class.getResourceAsStream(resourcePath)) { if (stream == null) { @@ -103,7 +176,12 @@ public static Dialect loadResource(String resourcePath) { } } - /** Serialize a dialect to YAML. */ + /** + * Serialize a dialect to YAML. + * + * @param dialect the dialect to serialize + * @return the YAML representation + */ public static String toYaml(Dialect dialect) { try { return DialectJsonSupport.MAPPER.writeValueAsString(dialect); diff --git a/core/src/main/java/io/substrait/dialect/DialectFunction.java b/core/src/main/java/io/substrait/dialect/DialectFunction.java index de784496c..9702c896c 100644 --- a/core/src/main/java/io/substrait/dialect/DialectFunction.java +++ b/core/src/main/java/io/substrait/dialect/DialectFunction.java @@ -13,26 +13,63 @@ @JsonSerialize(as = ImmutableDialectFunction.class) @Value.Immutable public abstract class DialectFunction { - /** Dependency (alias) in which the function is declared. */ + /** + * Dependency (alias) in which the function is declared. + * + * @return the dependency alias + */ public abstract String source(); - /** The name of the function as declared in the extension it is defined in. */ + /** + * The name of the function as declared in the extension it is defined in. + * + * @return the function name + */ public abstract String name(); + /** + * Free-form metadata associated with the function, if any. + * + * @return the optional metadata + */ public abstract Optional> metadata(); + /** + * System-specific metadata for the function, if any. + * + * @return the optional system metadata + */ @JsonProperty("system_metadata") public abstract Optional systemMetadata(); + /** + * The options required when invoking the function, if any. + * + * @return the optional required options + */ @JsonProperty("required_options") public abstract Optional> requiredOptions(); - /** One or more implementations supported by this function, identified by argument signatures. */ + /** + * One or more implementations supported by this function, identified by argument signatures. + * + * @return the supported implementation signatures + */ @JsonProperty("supported_impls") public abstract List supportedImpls(); + /** + * The variadic argument bounds of the function, if any. + * + * @return the optional variadic bounds + */ public abstract Optional variadic(); + /** + * Creates a builder for {@link DialectFunction}. + * + * @return a new builder + */ public static ImmutableDialectFunction.Builder builder() { return ImmutableDialectFunction.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/ExchangeKind.java b/core/src/main/java/io/substrait/dialect/ExchangeKind.java index cd3089e7f..0bbb549a6 100644 --- a/core/src/main/java/io/substrait/dialect/ExchangeKind.java +++ b/core/src/main/java/io/substrait/dialect/ExchangeKind.java @@ -2,9 +2,14 @@ /** Exchange kinds for {@code EXCHANGE} relations. */ public enum ExchangeKind { + /** Distribute rows across targets by hashing fields. */ SCATTER_BY_FIELDS, + /** Route all rows to a single target. */ SINGLE_TARGET, + /** Route rows to multiple targets. */ MULTI_TARGET, + /** Distribute rows across targets in round-robin order. */ ROUND_ROBIN, + /** Broadcast all rows to every target. */ BROADCAST } diff --git a/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java b/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java index 8651e8df6..fea7109ca 100644 --- a/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java +++ b/core/src/main/java/io/substrait/dialect/ExecutionBehavior.java @@ -11,9 +11,19 @@ @JsonSerialize(as = ImmutableExecutionBehavior.class) @Value.Immutable public abstract class ExecutionBehavior { + /** + * The variable evaluation modes supported by the dialect. + * + * @return the supported variable evaluation modes + */ @JsonProperty("supported_variable_evaluation_mode") public abstract List supportedVariableEvaluationMode(); + /** + * Creates a builder for {@link ExecutionBehavior}. + * + * @return a new builder + */ public static ImmutableExecutionBehavior.Builder builder() { return ImmutableExecutionBehavior.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/ExpandFieldType.java b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java index 61d05a147..d6e76b34f 100644 --- a/core/src/main/java/io/substrait/dialect/ExpandFieldType.java +++ b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java @@ -2,6 +2,8 @@ /** Field types for {@code EXPAND} relations. */ public enum ExpandFieldType { + /** A field whose value switches between expressions per expansion. */ SWITCHING_FIELD, + /** A field with a constant value across expansions. */ CONSTANT_FIELD } diff --git a/core/src/main/java/io/substrait/dialect/ExpressionKind.java b/core/src/main/java/io/substrait/dialect/ExpressionKind.java index ba767c989..bb42360e1 100644 --- a/core/src/main/java/io/substrait/dialect/ExpressionKind.java +++ b/core/src/main/java/io/substrait/dialect/ExpressionKind.java @@ -2,17 +2,30 @@ /** The kinds of expressions a dialect can declare support for. */ public enum ExpressionKind { + /** A literal value expression. */ LITERAL, + /** A field selection (reference) expression. */ SELECTION, + /** A scalar function call expression. */ SCALAR_FUNCTION, + /** A window function call expression. */ WINDOW_FUNCTION, + /** An if-then conditional expression. */ IF_THEN, + /** A switch expression. */ SWITCH, + /** A singular {@code OR}-list (IN) expression. */ SINGULAR_OR_LIST, + /** A multi-valued {@code OR}-list expression. */ MULTI_OR_LIST, + /** A cast expression. */ CAST, + /** A subquery expression. */ SUBQUERY, + /** A nested (struct/list/map) constructor expression. */ NESTED, + /** A dynamic parameter expression. */ DYNAMIC_PARAMETER, + /** An execution-context variable expression. */ EXECUTION_CONTEXT_VARIABLE } diff --git a/core/src/main/java/io/substrait/dialect/JoinType.java b/core/src/main/java/io/substrait/dialect/JoinType.java index 3980addf3..0c0fe4986 100644 --- a/core/src/main/java/io/substrait/dialect/JoinType.java +++ b/core/src/main/java/io/substrait/dialect/JoinType.java @@ -2,16 +2,28 @@ /** Join types for join relations. */ public enum JoinType { + /** An inner join. */ INNER, + /** A full outer join. */ OUTER, + /** A left outer join. */ LEFT, + /** A right outer join. */ RIGHT, + /** A left semi join. */ LEFT_SEMI, + /** A right semi join. */ RIGHT_SEMI, + /** A left anti join. */ LEFT_ANTI, + /** A right anti join. */ RIGHT_ANTI, + /** A left single join. */ LEFT_SINGLE, + /** A right single join. */ RIGHT_SINGLE, + /** A left mark join. */ LEFT_MARK, + /** A right mark join. */ RIGHT_MARK } diff --git a/core/src/main/java/io/substrait/dialect/NestedType.java b/core/src/main/java/io/substrait/dialect/NestedType.java index 796a31202..869fce9ab 100644 --- a/core/src/main/java/io/substrait/dialect/NestedType.java +++ b/core/src/main/java/io/substrait/dialect/NestedType.java @@ -2,7 +2,10 @@ /** Nested types for {@code NESTED} expressions. */ public enum NestedType { + /** A struct constructor. */ STRUCT, + /** A list constructor. */ LIST, + /** A map constructor. */ MAP } diff --git a/core/src/main/java/io/substrait/dialect/Notation.java b/core/src/main/java/io/substrait/dialect/Notation.java index 8648679e8..d80443d6b 100644 --- a/core/src/main/java/io/substrait/dialect/Notation.java +++ b/core/src/main/java/io/substrait/dialect/Notation.java @@ -2,8 +2,12 @@ /** How a system function is written in the system's own syntax. */ public enum Notation { + /** The function is written between its operands (e.g. {@code a + b}). */ INFIX, + /** The function is written after its operand (e.g. {@code a!}). */ POSTFIX, + /** The function is written before its operand (e.g. {@code -a}). */ PREFIX, + /** The function is written using call syntax (e.g. {@code f(a, b)}). */ FUNCTION } diff --git a/core/src/main/java/io/substrait/dialect/ReadType.java b/core/src/main/java/io/substrait/dialect/ReadType.java index 41843b881..9695463a3 100644 --- a/core/src/main/java/io/substrait/dialect/ReadType.java +++ b/core/src/main/java/io/substrait/dialect/ReadType.java @@ -2,9 +2,14 @@ /** Read types for {@code READ} relations. */ public enum ReadType { + /** A read from an inline virtual table. */ VIRTUAL_TABLE, + /** A read from local files. */ LOCAL_FILES, + /** A read from a named table. */ NAMED_TABLE, + /** A read from an extension-defined table. */ EXTENSION_TABLE, + /** A read from an Iceberg table. */ ICEBERG_TABLE } diff --git a/core/src/main/java/io/substrait/dialect/RelationKind.java b/core/src/main/java/io/substrait/dialect/RelationKind.java index c823e0b76..65c4efee4 100644 --- a/core/src/main/java/io/substrait/dialect/RelationKind.java +++ b/core/src/main/java/io/substrait/dialect/RelationKind.java @@ -2,26 +2,48 @@ /** The kinds of relations a dialect can declare support for. */ public enum RelationKind { + /** The read relation. */ READ, + /** The filter relation. */ FILTER, + /** The fetch (limit/offset) relation. */ FETCH, + /** The aggregate relation. */ AGGREGATE, + /** The sort relation. */ SORT, + /** The join relation. */ JOIN, + /** The project relation. */ PROJECT, + /** The set-operation relation. */ SET, + /** The cross-product relation. */ CROSS, + /** The reference relation. */ REFERENCE, + /** The write relation. */ WRITE, + /** The DDL relation. */ DDL, + /** The update relation. */ UPDATE, + /** The hash join relation. */ HASH_JOIN, + /** The merge join relation. */ MERGE_JOIN, + /** The nested-loop join relation. */ NESTED_LOOP_JOIN, + /** The consistent partition window relation. */ CONSISTENT_PARTITION_WINDOW, + /** The exchange relation. */ EXCHANGE, + /** The expand relation. */ EXPAND, + /** The single-input extension relation. */ EXTENSION_SINGLE, + /** The multi-input extension relation. */ EXTENSION_MULTI, + /** The leaf extension relation. */ EXTENSION_LEAF } diff --git a/core/src/main/java/io/substrait/dialect/SetOperation.java b/core/src/main/java/io/substrait/dialect/SetOperation.java index a1112f458..72340e32b 100644 --- a/core/src/main/java/io/substrait/dialect/SetOperation.java +++ b/core/src/main/java/io/substrait/dialect/SetOperation.java @@ -2,12 +2,20 @@ /** Set operations for {@code SET} relations. */ public enum SetOperation { + /** The primary set difference (minus) operation. */ MINUS_PRIMARY, + /** The primary set difference (minus) operation retaining all rows. */ MINUS_PRIMARY_ALL, + /** The multiset difference (minus) operation. */ MINUS_MULTISET, + /** The primary set intersection operation. */ INTERSECTION_PRIMARY, + /** The multiset intersection operation. */ INTERSECTION_MULTISET, + /** The multiset intersection operation retaining all rows. */ INTERSECTION_MULTISET_ALL, + /** The distinct union operation. */ UNION_DISTINCT, + /** The union-all operation. */ UNION_ALL } diff --git a/core/src/main/java/io/substrait/dialect/SubqueryType.java b/core/src/main/java/io/substrait/dialect/SubqueryType.java index 60892f396..daf74e1aa 100644 --- a/core/src/main/java/io/substrait/dialect/SubqueryType.java +++ b/core/src/main/java/io/substrait/dialect/SubqueryType.java @@ -2,8 +2,12 @@ /** Subquery types for {@code SUBQUERY} expressions. */ public enum SubqueryType { + /** A scalar subquery. */ SCALAR, + /** An {@code IN} predicate subquery. */ IN_PREDICATE, + /** A set predicate ({@code EXISTS}/{@code UNIQUE}) subquery. */ SET_PREDICATE, + /** A set comparison ({@code ANY}/{@code ALL}) subquery. */ SET_COMPARISON } diff --git a/core/src/main/java/io/substrait/dialect/SupportedExpression.java b/core/src/main/java/io/substrait/dialect/SupportedExpression.java index f84db23e8..18483c585 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedExpression.java +++ b/core/src/main/java/io/substrait/dialect/SupportedExpression.java @@ -15,20 +15,46 @@ @JsonSerialize(using = SupportedExpressionSerializer.class) @Value.Immutable public abstract class SupportedExpression { + /** + * The kind of expression this entry describes. + * + * @return the expression kind + */ public abstract ExpressionKind expression(); + /** + * Free-form metadata associated with the expression, if any. + * + * @return the optional metadata + */ public abstract Optional> metadata(); - /** Permissible failure options for {@code CAST}. */ + /** + * Permissible failure options for {@code CAST}. + * + * @return the supported cast failure options + */ public abstract List failureOptions(); - /** Subquery types for {@code SUBQUERY}. */ + /** + * Subquery types for {@code SUBQUERY}. + * + * @return the supported subquery types + */ public abstract List subqueryTypes(); - /** Nested types for {@code NESTED}. */ + /** + * Nested types for {@code NESTED}. + * + * @return the supported nested types + */ public abstract List nestedTypes(); - /** Variable types for {@code EXECUTION_CONTEXT_VARIABLE}. */ + /** + * Variable types for {@code EXECUTION_CONTEXT_VARIABLE}. + * + * @return the supported variable types + */ public abstract List variableTypes(); /** @@ -43,7 +69,11 @@ protected void checkMetadata() { } } - /** Whether this entry can be written as a bare enum string (no extra configuration). */ + /** + * Whether this entry can be written as a bare enum string (no extra configuration). + * + * @return {@code true} if the entry carries no configuration + */ public boolean isBare() { return !metadata().isPresent() && failureOptions().isEmpty() @@ -52,10 +82,21 @@ && nestedTypes().isEmpty() && variableTypes().isEmpty(); } + /** + * Creates a configuration-free entry for the given kind. + * + * @param expression the expression kind + * @return a new {@link SupportedExpression} + */ public static SupportedExpression of(ExpressionKind expression) { return builder().expression(expression).build(); } + /** + * Creates a builder for {@link SupportedExpression}. + * + * @return a new builder + */ public static ImmutableSupportedExpression.Builder builder() { return ImmutableSupportedExpression.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelation.java b/core/src/main/java/io/substrait/dialect/SupportedRelation.java index e2e914dd5..8f58bf2c5 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedRelation.java +++ b/core/src/main/java/io/substrait/dialect/SupportedRelation.java @@ -15,34 +15,74 @@ @JsonSerialize(using = SupportedRelationSerializer.class) @Value.Immutable public abstract class SupportedRelation { + /** + * The kind of relation this entry describes. + * + * @return the relation kind + */ public abstract RelationKind relation(); + /** + * Free-form metadata associated with the relation, if any. + * + * @return the optional metadata + */ public abstract Optional> metadata(); /** * Join types for {@code JOIN}, {@code HASH_JOIN}, {@code MERGE_JOIN}, {@code NESTED_LOOP_JOIN}. + * + * @return the supported join types */ public abstract List joinTypes(); - /** Read types for {@code READ}. */ + /** + * Read types for {@code READ}. + * + * @return the supported read types + */ public abstract List readTypes(); - /** Set operations for {@code SET}. */ + /** + * Set operations for {@code SET}. + * + * @return the supported set operations + */ public abstract List operations(); - /** Write types for {@code WRITE} (serialized as {@code write_types}). */ + /** + * Write types for {@code WRITE} (serialized as {@code write_types}). + * + * @return the supported write types + */ public abstract List writeTypes(); - /** Operable object types for {@code DDL} (also serialized as {@code write_types}). */ + /** + * Operable object types for {@code DDL} (also serialized as {@code write_types}). + * + * @return the supported DDL write types + */ public abstract List ddlWriteTypes(); - /** Exchange kinds for {@code EXCHANGE}. */ + /** + * Exchange kinds for {@code EXCHANGE}. + * + * @return the supported exchange kinds + */ public abstract List kinds(); - /** Field types for {@code EXPAND}. */ + /** + * Field types for {@code EXPAND}. + * + * @return the supported expand field types + */ public abstract List fieldTypes(); - /** Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. */ + /** + * Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. + * + * @return the supported message type URIs + */ public abstract List messageTypes(); /** @@ -62,6 +102,8 @@ protected void checkWriteTypes() { /** * Whether this entry can be written as a bare enum string. Extension relations are never bare: * they are absent from the schema's bare-enum list. + * + * @return {@code true} if the entry carries no configuration */ public boolean isBare() { switch (relation()) { @@ -83,10 +125,21 @@ && fieldTypes().isEmpty() && messageTypes().isEmpty(); } + /** + * Creates a configuration-free entry for the given kind. + * + * @param relation the relation kind + * @return a new {@link SupportedRelation} + */ public static SupportedRelation of(RelationKind relation) { return builder().relation(relation).build(); } + /** + * Creates a builder for {@link SupportedRelation}. + * + * @return a new builder + */ public static ImmutableSupportedRelation.Builder builder() { return ImmutableSupportedRelation.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/SupportedType.java b/core/src/main/java/io/substrait/dialect/SupportedType.java index b3c058729..9ba23d36a 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedType.java +++ b/core/src/main/java/io/substrait/dialect/SupportedType.java @@ -14,21 +14,53 @@ @JsonSerialize(using = SupportedTypeSerializer.class) @Value.Immutable public abstract class SupportedType { + /** + * The kind of type this entry describes. + * + * @return the type kind + */ public abstract TypeKind type(); + /** + * Free-form metadata associated with the type, if any. + * + * @return the optional metadata + */ public abstract Optional> metadata(); + /** + * System-specific metadata for the type, if any. + * + * @return the optional system metadata + */ public abstract Optional systemMetadata(); + /** + * The maximum precision supported for the type, if constrained. + * + * @return the optional maximum precision + */ public abstract Optional maxPrecision(); - /** Dependency (alias) where a {@code USER_DEFINED} type is declared. */ + /** + * Dependency (alias) where a {@code USER_DEFINED} type is declared. + * + * @return the optional dependency alias + */ public abstract Optional source(); - /** The name of a {@code USER_DEFINED} type as declared in the extension it is defined in. */ + /** + * The name of a {@code USER_DEFINED} type as declared in the extension it is defined in. + * + * @return the optional type name + */ public abstract Optional name(); - /** Whether this entry can be written as a bare enum string (no extra configuration). */ + /** + * Whether this entry can be written as a bare enum string (no extra configuration). + * + * @return {@code true} if the entry carries no configuration + */ public boolean isBare() { return type() != TypeKind.USER_DEFINED && !metadata().isPresent() @@ -38,10 +70,21 @@ public boolean isBare() { && !name().isPresent(); } + /** + * Creates a configuration-free entry for the given kind. + * + * @param type the type kind + * @return a new {@link SupportedType} + */ public static SupportedType of(TypeKind type) { return builder().type(type).build(); } + /** + * Creates a builder for {@link SupportedType}. + * + * @return a new builder + */ public static ImmutableSupportedType.Builder builder() { return ImmutableSupportedType.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java b/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java index 71b226e49..0ba8dc2b2 100644 --- a/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java +++ b/core/src/main/java/io/substrait/dialect/SystemFunctionMetadata.java @@ -10,13 +10,28 @@ @JsonSerialize(as = ImmutableSystemFunctionMetadata.class) @Value.Immutable public abstract class SystemFunctionMetadata { + /** + * The name of the function in the system's own syntax, if any. + * + * @return the optional system name + */ public abstract Optional name(); + /** + * The notation in which the function is written in the system's own syntax. + * + * @return the notation + */ @Value.Default public Notation notation() { return Notation.FUNCTION; } + /** + * Creates a builder for {@link SystemFunctionMetadata}. + * + * @return a new builder + */ public static ImmutableSystemFunctionMetadata.Builder builder() { return ImmutableSystemFunctionMetadata.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java b/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java index 4ae9abdf9..526c6886c 100644 --- a/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java +++ b/core/src/main/java/io/substrait/dialect/SystemTypeMetadata.java @@ -11,11 +11,26 @@ @JsonSerialize(as = ImmutableSystemTypeMetadata.class) @Value.Immutable public abstract class SystemTypeMetadata { + /** + * The name of the type in the system's own syntax, if any. + * + * @return the optional system name + */ public abstract Optional name(); + /** + * Whether the type is supported as a column type, if specified. + * + * @return the optional column-support flag + */ @JsonProperty("supported_as_column") public abstract Optional supportedAsColumn(); + /** + * Creates a builder for {@link SystemTypeMetadata}. + * + * @return a new builder + */ public static ImmutableSystemTypeMetadata.Builder builder() { return ImmutableSystemTypeMetadata.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/TypeKind.java b/core/src/main/java/io/substrait/dialect/TypeKind.java index 9ecda820c..66a6fb2da 100644 --- a/core/src/main/java/io/substrait/dialect/TypeKind.java +++ b/core/src/main/java/io/substrait/dialect/TypeKind.java @@ -2,30 +2,56 @@ /** The kinds of types a dialect can declare support for. */ public enum TypeKind { + /** The boolean type. */ BOOL, + /** The 8-bit signed integer type. */ I8, + /** The 16-bit signed integer type. */ I16, + /** The 32-bit signed integer type. */ I32, + /** The 64-bit signed integer type. */ I64, + /** The 32-bit floating-point type. */ FP32, + /** The 64-bit floating-point type. */ FP64, + /** The variable-length binary type. */ BINARY, + /** The fixed-length binary type. */ FIXED_BINARY, + /** The variable-length string type. */ STRING, + /** The variable-length character type with a maximum length. */ VARCHAR, + /** The fixed-length character type. */ FIXED_CHAR, + /** The time-of-day type with configurable precision. */ PRECISION_TIME, + /** The timestamp type with configurable precision. */ PRECISION_TIMESTAMP, + /** The timezone-aware timestamp type with configurable precision. */ PRECISION_TIMESTAMP_TZ, + /** The calendar date type. */ DATE, + /** The time-of-day type. */ TIME, + /** The compound interval type. */ INTERVAL_COMPOUND, + /** The day-based interval type. */ INTERVAL_DAY, + /** The year-based interval type. */ INTERVAL_YEAR, + /** The UUID type. */ UUID, + /** The fixed-point decimal type. */ DECIMAL, + /** The struct (record) type. */ STRUCT, + /** The list type. */ LIST, + /** The map type. */ MAP, + /** A user-defined extension type. */ USER_DEFINED } diff --git a/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java b/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java index 3245fb507..fdd5f9d61 100644 --- a/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java +++ b/core/src/main/java/io/substrait/dialect/VariableEvaluationMode.java @@ -2,6 +2,8 @@ /** When execution context variables are evaluated. */ public enum VariableEvaluationMode { + /** The variable is evaluated once per plan. */ PER_PLAN, + /** The variable is evaluated once per record. */ PER_RECORD } diff --git a/core/src/main/java/io/substrait/dialect/VariableType.java b/core/src/main/java/io/substrait/dialect/VariableType.java index 25fe5ff37..529dd7f5f 100644 --- a/core/src/main/java/io/substrait/dialect/VariableType.java +++ b/core/src/main/java/io/substrait/dialect/VariableType.java @@ -2,7 +2,10 @@ /** Variable types for {@code EXECUTION_CONTEXT_VARIABLE} expressions. */ public enum VariableType { + /** The current timestamp variable. */ CURRENT_TIMESTAMP, + /** The current timezone variable. */ CURRENT_TIMEZONE, + /** The current date variable. */ CURRENT_DATE } diff --git a/core/src/main/java/io/substrait/dialect/Variadic.java b/core/src/main/java/io/substrait/dialect/Variadic.java index 8ed2ab8ed..5b8f2b904 100644 --- a/core/src/main/java/io/substrait/dialect/Variadic.java +++ b/core/src/main/java/io/substrait/dialect/Variadic.java @@ -10,10 +10,25 @@ @JsonSerialize(as = ImmutableVariadic.class) @Value.Immutable public abstract class Variadic { + /** + * The minimum number of variadic arguments, if bounded. + * + * @return the optional minimum + */ public abstract OptionalInt min(); + /** + * The maximum number of variadic arguments, if bounded. + * + * @return the optional maximum + */ public abstract OptionalInt max(); + /** + * Creates a builder for {@link Variadic}. + * + * @return a new builder + */ public static ImmutableVariadic.Builder builder() { return ImmutableVariadic.builder(); } diff --git a/core/src/main/java/io/substrait/dialect/WriteType.java b/core/src/main/java/io/substrait/dialect/WriteType.java index 1d17dd424..24221c8c8 100644 --- a/core/src/main/java/io/substrait/dialect/WriteType.java +++ b/core/src/main/java/io/substrait/dialect/WriteType.java @@ -2,6 +2,8 @@ /** Write types for {@code WRITE} relations. */ public enum WriteType { + /** A write to a named table. */ NAMED_TABLE, + /** A write to an extension-defined table. */ EXTENSION_TABLE }