713 lines
34 KiB
Kotlin
713 lines
34 KiB
Kotlin
package kestrel.engine
|
|
|
|
import kestrel.KestrelException
|
|
import kestrel.catalog.Catalog
|
|
import kestrel.catalog.Column
|
|
import kestrel.catalog.TableSchema
|
|
import kestrel.sql.ast.BetweenExpression
|
|
import kestrel.sql.ast.BinaryExpression
|
|
import kestrel.sql.ast.BinaryOperator
|
|
import kestrel.sql.ast.CallExpression
|
|
import kestrel.sql.ast.CaseExpression
|
|
import kestrel.sql.ast.CastExpression
|
|
import kestrel.sql.ast.CollateExpression
|
|
import kestrel.sql.ast.ColumnExpression
|
|
import kestrel.sql.ast.CommonTableExpression
|
|
import kestrel.sql.ast.CurrentValueExpression
|
|
import kestrel.sql.ast.ExistsExpression
|
|
import kestrel.sql.ast.Expression
|
|
import kestrel.sql.ast.ExpressionProjection
|
|
import kestrel.sql.ast.FrameBound
|
|
import kestrel.sql.ast.FrameBoundType
|
|
import kestrel.sql.ast.FrameExclude
|
|
import kestrel.sql.ast.FrameUnit
|
|
import kestrel.sql.ast.FromItem
|
|
import kestrel.sql.ast.FunctionExpression
|
|
import kestrel.sql.ast.InExpression
|
|
import kestrel.sql.ast.InSubqueryExpression
|
|
import kestrel.sql.ast.IsNullExpression
|
|
import kestrel.sql.ast.JoinClause
|
|
import kestrel.sql.ast.JoinType
|
|
import kestrel.sql.ast.LikeExpression
|
|
import kestrel.sql.ast.LiteralExpression
|
|
import kestrel.sql.ast.MergeActionExpression
|
|
import kestrel.sql.ast.NextValueExpression
|
|
import kestrel.sql.ast.ParameterExpression
|
|
import kestrel.sql.ast.OrderKey
|
|
import kestrel.sql.ast.QueryStatement
|
|
import kestrel.sql.ast.ScalarSubqueryExpression
|
|
import kestrel.sql.ast.SelectItem
|
|
import kestrel.sql.ast.SelectStatement
|
|
import kestrel.sql.ast.SetOperationStatement
|
|
import kestrel.sql.ast.SetOperator
|
|
import kestrel.sql.ast.StarProjection
|
|
import kestrel.sql.ast.SubquerySource
|
|
import kestrel.sql.ast.TableSource
|
|
import kestrel.sql.ast.UnaryExpression
|
|
import kestrel.sql.ast.WindowExpression
|
|
import kestrel.sql.ast.WindowFrame
|
|
import kestrel.sql.ast.WindowFunctionKind
|
|
import kestrel.sql.ast.WithQuery
|
|
import kestrel.storage.Collation
|
|
import kestrel.storage.ColumnType
|
|
import kestrel.storage.Row
|
|
import kestrel.storage.Value
|
|
import kestrel.storage.displayWith
|
|
import java.math.BigDecimal
|
|
|
|
class SelectRunner(private val catalog: Catalog) {
|
|
private val indexPlanner = IndexAccessPlanner(catalog)
|
|
private val subqueryRunner = SubqueryRunner { select, outer -> run(select, outer) }
|
|
private val expandingViews = mutableSetOf<String>()
|
|
private val activeCtes = mutableListOf<CteScope>()
|
|
|
|
fun run(statement: SelectStatement, outer: ColumnResolver? = null): SelectResult {
|
|
val plan = buildPlan(statement, outer)
|
|
val combined = plan.materialize()
|
|
return if (plan.aggregated) plan.aggregate(combined) else plan.projectRows(combined)
|
|
}
|
|
|
|
fun runQuery(query: QueryStatement, outer: ColumnResolver? = null): SelectResult = when (query) {
|
|
is SelectStatement -> run(query, outer)
|
|
is SetOperationStatement -> runSetOperation(query, outer)
|
|
is WithQuery -> runWith(query, outer)
|
|
}
|
|
|
|
fun explain(query: QueryStatement): List<String> = when (query) {
|
|
is SelectStatement -> buildPlan(query, null).describe()
|
|
is SetOperationStatement -> explainSetOperation(query)
|
|
is WithQuery -> explainWith(query)
|
|
}
|
|
|
|
/**
|
|
* Resolves a single [FromItem] to its schema and row snapshot - the same lookup a `FROM`/`JOIN`
|
|
* source uses ([resolveSource]), exposed for [Executor.executeMerge]'s `USING` clause so a MERGE
|
|
* source can be either a real table or a derived-table subquery without duplicating view/CTE/
|
|
* subquery resolution.
|
|
*/
|
|
fun resolveMergeSource(item: FromItem): Pair<TableSchema, List<Row>> = resolveSource(item)
|
|
|
|
/** The name a MERGE `ON`/`WHEN` condition would qualify a column from [item] with - its alias if given, its bare table name otherwise (a subquery source always has one). */
|
|
fun mergeSourceLabel(item: FromItem): String = sourceLabel(item)
|
|
|
|
private fun explainSetOperation(statement: SetOperationStatement): List<String> {
|
|
val header = setOperatorKeyword(statement.operator)
|
|
val left = explain(statement.left).map { " -> $it" }
|
|
val right = explain(statement.right).map { " -> $it" }
|
|
var lines = listOf(header) + left + right
|
|
statement.orderBy?.let { lines = listOf("Sort") + lines.map { line -> " $line" } }
|
|
statement.offset?.let { lines = listOf("Offset ${statement.offset}") + lines.map { line -> " $line" } }
|
|
statement.limit?.let { lines = listOf("Limit ${statement.limit}") + lines.map { line -> " $line" } }
|
|
return lines
|
|
}
|
|
|
|
private fun explainWith(query: WithQuery): List<String> {
|
|
var pushed = 0
|
|
try {
|
|
for (cte in query.ctes) {
|
|
activeCtes.add(buildCteScope(query.recursive, cte))
|
|
pushed++
|
|
}
|
|
val cteLines = query.ctes.flatMap { cte ->
|
|
listOf("CTE ${cte.name}") + explain(cte.query).map { " -> $it" }
|
|
}
|
|
return cteLines + explain(query.body)
|
|
} finally {
|
|
repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) }
|
|
}
|
|
}
|
|
|
|
private fun setOperatorKeyword(operator: SetOperator): String = when (operator) {
|
|
SetOperator.Union -> "Union"
|
|
SetOperator.UnionAll -> "Union All"
|
|
SetOperator.Intersect -> "Intersect"
|
|
SetOperator.IntersectAll -> "Intersect All"
|
|
SetOperator.Except -> "Except"
|
|
SetOperator.ExceptAll -> "Except All"
|
|
}
|
|
|
|
private fun runSetOperation(statement: SetOperationStatement, outer: ColumnResolver?): SelectResult {
|
|
val left = runQuery(statement.left, outer)
|
|
val right = runQuery(statement.right, outer)
|
|
if (left.columns.size != right.columns.size) {
|
|
throw KestrelException(
|
|
"${setOperatorKeyword(statement.operator)} queries must have the same number of columns " +
|
|
"but got ${left.columns.size} and ${right.columns.size}"
|
|
)
|
|
}
|
|
val declaredCollationOf = declaredSetOperationCollation(statement, left.columns)
|
|
val outputExpressions = left.columns.map { ColumnExpression(it) }
|
|
fun foldedKey(row: List<Value>) = collationFoldedKey(row, outputExpressions, declaredCollationOf)
|
|
var rows = when (statement.operator) {
|
|
SetOperator.UnionAll -> left.rows + right.rows
|
|
SetOperator.Union -> (left.rows + right.rows).distinctBy(::foldedKey)
|
|
SetOperator.Intersect -> {
|
|
val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey)
|
|
left.rows.filter { foldedKey(it) in rightKeys }.distinctBy(::foldedKey)
|
|
}
|
|
SetOperator.Except -> {
|
|
val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey)
|
|
left.rows.filter { foldedKey(it) !in rightKeys }.distinctBy(::foldedKey)
|
|
}
|
|
SetOperator.IntersectAll -> multisetFilter(left.rows, right.rows, keepOnMatch = true, keyOf = ::foldedKey)
|
|
SetOperator.ExceptAll -> multisetFilter(left.rows, right.rows, keepOnMatch = false, keyOf = ::foldedKey)
|
|
}
|
|
val orderKeys = statement.orderBy?.keys.orEmpty()
|
|
if (orderKeys.isNotEmpty()) {
|
|
rows = rows.sortedWith(
|
|
multiKeyComparator(
|
|
orderKeys,
|
|
evaluate = { expression, row -> evaluateOutputRow(left.columns, expression, row) },
|
|
declaredCollationOf = declaredCollationOf
|
|
)
|
|
)
|
|
}
|
|
statement.offset?.let { if (it < 0) throw KestrelException("OFFSET must not be negative") }
|
|
statement.limit?.let { if (it < 0) throw KestrelException("LIMIT must not be negative") }
|
|
if (statement.offset != null || statement.limit != null) {
|
|
rows = rows.drop(statement.offset ?: 0)
|
|
statement.limit?.let { rows = rows.take(it) }
|
|
}
|
|
return SelectResult(left.columns, rows)
|
|
}
|
|
|
|
/**
|
|
* The declared `COLLATE` of [statement]'s combined output column named by an ORDER BY key
|
|
* expression (an unqualified [ColumnExpression] - the only kind [evaluateOutputRow] resolves
|
|
* for a set operation's own ORDER BY, since it rejects any qualifier), traced back to a real
|
|
* table column in [statement]'s LEFT branch only - the branch [SelectResult.columns] itself
|
|
* always takes column identity from (`runSetOperation` returns `SelectResult(left.columns,
|
|
* ...)`, never consulting the right side's names). Resolves only when that branch is a plain
|
|
* `SELECT ... FROM realTable [JOIN realTable ...]` (no CTE, view, or derived table anywhere in
|
|
* its FROM clause) referencing the column bare - going further would mean inheriting collation
|
|
* through a view/CTE/derived-table boundary, the still-open scope-cut already left by #65/#66
|
|
* for a plain SELECT's own WHERE/ORDER BY over such a source (see kestrel-stan.md), not
|
|
* extended here.
|
|
*/
|
|
private fun declaredSetOperationCollation(statement: SetOperationStatement, columns: List<String>): (ColumnExpression) -> Collation? =
|
|
(statement.left as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null }
|
|
|
|
/**
|
|
* The same "plain `SELECT ... FROM realTable [JOIN realTable ...]`, no CTE/view/derived table"
|
|
* structural resolution [declaredSetOperationCollation] does for a set operation's LEFT branch,
|
|
* factored out so [declaredRecursiveCteCollation] can apply the identical rule to a recursive
|
|
* CTE's own anchor (base) term - the branch that owns [CteScope.schema]/column identity there,
|
|
* exactly like the LEFT branch owns [SelectResult.columns] for a plain set operation.
|
|
*/
|
|
private fun declaredCollationForSelectOutput(select: SelectStatement, columns: List<String>): (ColumnExpression) -> Collation? {
|
|
val items = listOf(select.from) + select.joins.map { it.from }
|
|
val sources = items.map { item ->
|
|
(item as? TableSource)?.let { source -> catalog.tableOrNull(source.name)?.schema?.let { (source.alias ?: source.name) to it } }
|
|
}
|
|
if (sources.any { it == null }) return { null }
|
|
val resolvedSources = sources.filterNotNull()
|
|
fun columnCollation(expression: ColumnExpression): Collation? {
|
|
val matches = resolvedSources
|
|
.filter { (alias, _) -> expression.qualifier == null || alias.equals(expression.qualifier, ignoreCase = true) }
|
|
.mapNotNull { (_, schema) -> schema.column(expression.name) }
|
|
return matches.singleOrNull()?.collation
|
|
}
|
|
return when (val projection = select.projection) {
|
|
is ExpressionProjection -> outputExpr@{ outputExpr ->
|
|
val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) }
|
|
val itemExpression = index.takeIf { it in projection.items.indices }?.let { projection.items[it].expression }
|
|
(itemExpression as? ColumnExpression)?.let(::columnCollation)
|
|
}
|
|
is StarProjection -> outputExpr@{ outputExpr ->
|
|
val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) }
|
|
if (index < 0) return@outputExpr null
|
|
var remaining = index
|
|
for ((_, schema) in resolvedSources) {
|
|
if (remaining < schema.columns.size) return@outputExpr schema.columns[remaining].collation
|
|
remaining -= schema.columns.size
|
|
}
|
|
null
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Mirrors [declaredSetOperationCollation] but resolves against a recursive CTE's anchor (`base`) term. */
|
|
private fun declaredRecursiveCteCollation(base: QueryStatement, columns: List<String>): (ColumnExpression) -> Collation? =
|
|
(base as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null }
|
|
|
|
private fun multisetFilter(
|
|
left: List<List<Value>>,
|
|
right: List<List<Value>>,
|
|
keepOnMatch: Boolean,
|
|
keyOf: (List<Value>) -> List<Value>
|
|
): List<List<Value>> {
|
|
val remainingInRight = right.groupingBy(keyOf).eachCount().toMutableMap()
|
|
return left.filter { row ->
|
|
val key = keyOf(row)
|
|
val remaining = remainingInRight[key] ?: 0
|
|
if (remaining > 0) {
|
|
remainingInRight[key] = remaining - 1
|
|
keepOnMatch
|
|
} else {
|
|
!keepOnMatch
|
|
}
|
|
}
|
|
}
|
|
|
|
private fun multiKeyComparator(
|
|
keys: List<OrderKey>,
|
|
evaluate: (Expression, List<Value>) -> Value,
|
|
declaredCollationOf: (ColumnExpression) -> Collation? = { null }
|
|
): Comparator<List<Value>> = Comparator { first, second ->
|
|
var result = 0
|
|
for (key in keys) {
|
|
val comparison = compareOrderKeyValues(
|
|
evaluate(key.expression, first), evaluate(key.expression, second), key, declaredCollationOf
|
|
)
|
|
if (comparison != 0) {
|
|
result = comparison
|
|
break
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
private fun evaluateOutputRow(columns: List<String>, expression: Expression, row: List<Value>): Value {
|
|
val resolver = kestrel.engine.ColumnResolver { column ->
|
|
if (column.qualifier != null) throw KestrelException("no such column: ${column.qualifier}.${column.name}")
|
|
val index = columns.indexOfFirst { it.equals(column.name, ignoreCase = true) }
|
|
if (index == -1) throw KestrelException("no such column: ${column.name}")
|
|
row[index]
|
|
}
|
|
return Evaluator(resolver, subqueryRunner).evaluate(expression)
|
|
}
|
|
|
|
private fun runWith(query: WithQuery, outer: ColumnResolver?): SelectResult {
|
|
var pushed = 0
|
|
try {
|
|
for (cte in query.ctes) {
|
|
activeCtes.add(buildCteScope(query.recursive, cte))
|
|
pushed++
|
|
}
|
|
return runQuery(query.body, outer)
|
|
} finally {
|
|
repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) }
|
|
}
|
|
}
|
|
|
|
private fun buildCteScope(recursive: Boolean, cte: CommonTableExpression): CteScope {
|
|
val recursivePlan = if (recursive) splitRecursiveCte(cte) else null
|
|
if (recursivePlan == null) {
|
|
val result = runQuery(cte.query, null)
|
|
return CteScope(cte.name, cteSchema(cte, result), result.rows.map { Row(it) })
|
|
}
|
|
val (base, operator, recursiveTerm) = recursivePlan
|
|
val baseResult = runQuery(base, null)
|
|
val schema = cteSchema(cte, baseResult)
|
|
val scope = CteScope(cte.name, schema, baseResult.rows.map { Row(it) })
|
|
val declaredCollationOf = declaredRecursiveCteCollation(base, baseResult.columns)
|
|
val outputExpressions = baseResult.columns.map { ColumnExpression(it) }
|
|
val seenKeys = if (operator == SetOperator.Union)
|
|
baseResult.rows.mapTo(HashSet()) { collationFoldedKey(it, outputExpressions, declaredCollationOf) }
|
|
else null
|
|
activeCtes.add(scope)
|
|
try {
|
|
val accumulated = mutableListOf<List<Value>>()
|
|
accumulated.addAll(baseResult.rows)
|
|
var working = baseResult.rows
|
|
var iterations = 0
|
|
while (working.isNotEmpty()) {
|
|
iterations++
|
|
if (iterations > MAX_RECURSION_ITERATIONS) {
|
|
throw KestrelException(
|
|
"recursive common table expression '${cte.name}' did not terminate within $MAX_RECURSION_ITERATIONS iterations"
|
|
)
|
|
}
|
|
scope.rows = working.map { Row(it) }
|
|
var newRows = runQuery(recursiveTerm, null).rows
|
|
if (operator == SetOperator.Union) {
|
|
newRows = newRows.filter { candidate -> seenKeys!!.add(collationFoldedKey(candidate, outputExpressions, declaredCollationOf)) }
|
|
}
|
|
if (newRows.isEmpty()) break
|
|
accumulated.addAll(newRows)
|
|
working = newRows
|
|
}
|
|
scope.rows = accumulated.map { Row(it) }
|
|
} finally {
|
|
activeCtes.removeAt(activeCtes.size - 1)
|
|
}
|
|
return scope
|
|
}
|
|
|
|
private data class RecursiveCtePlan(val base: QueryStatement, val operator: SetOperator, val recursiveTerm: QueryStatement)
|
|
|
|
private fun splitRecursiveCte(cte: CommonTableExpression): RecursiveCtePlan? {
|
|
val query = cte.query
|
|
if (query !is SetOperationStatement) return null
|
|
if (query.operator != SetOperator.Union && query.operator != SetOperator.UnionAll) return null
|
|
val leftRefs = referencesName(query.left, cte.name)
|
|
val rightRefs = referencesName(query.right, cte.name)
|
|
return when {
|
|
leftRefs && rightRefs ->
|
|
throw KestrelException("recursive common table expression '${cte.name}' must not reference itself on both sides of ${query.operator}")
|
|
rightRefs -> RecursiveCtePlan(query.left, query.operator, query.right)
|
|
leftRefs -> RecursiveCtePlan(query.right, query.operator, query.left)
|
|
else -> null
|
|
}
|
|
}
|
|
|
|
private fun referencesName(query: QueryStatement, name: String): Boolean = when (query) {
|
|
is SelectStatement -> referencesName(query.from, name) || query.joins.any { referencesName(it.from, name) }
|
|
is SetOperationStatement -> referencesName(query.left, name) || referencesName(query.right, name)
|
|
is WithQuery -> query.ctes.none { it.name.equals(name, ignoreCase = true) } && referencesName(query.body, name)
|
|
}
|
|
|
|
private fun referencesName(item: FromItem, name: String): Boolean = when (item) {
|
|
is TableSource -> item.name.equals(name, ignoreCase = true)
|
|
is SubquerySource -> referencesName(item.select, name)
|
|
}
|
|
|
|
private fun cteSchema(cte: CommonTableExpression, result: SelectResult): TableSchema {
|
|
val labels = cte.columns ?: result.columns
|
|
if (labels.size != result.columns.size) {
|
|
throw KestrelException(
|
|
"common table expression '${cte.name}' expects ${labels.size} columns but its query produces ${result.columns.size}"
|
|
)
|
|
}
|
|
return derivedSchema(cte.name, result, labels)
|
|
}
|
|
|
|
private fun buildPlan(statement: SelectStatement, outer: ColumnResolver?): Plan {
|
|
val sources = buildSources(statement)
|
|
val plan = Plan(statement, sources, outer)
|
|
plan.validate()
|
|
return plan
|
|
}
|
|
|
|
private fun buildSources(statement: SelectStatement): List<Source> {
|
|
val sources = mutableListOf<Source>()
|
|
var offset = 0
|
|
fun add(item: FromItem) {
|
|
// A LATERAL derived table's rows depend on the row of an earlier FROM item it
|
|
// correlates with, so they cannot be resolved once up front like every other source -
|
|
// only its column *names* are needed here (for offset bookkeeping and validating
|
|
// references to its alias elsewhere in the query), computed structurally from the
|
|
// subquery's own AST without running it. Real rows are produced per left row in
|
|
// [executeJoin].
|
|
val (schema, rows) = if (item is SubquerySource && item.lateral) {
|
|
lateralSchema(item) to emptyList()
|
|
} else {
|
|
resolveSource(item)
|
|
}
|
|
val chosen = sourceLabel(item)
|
|
if (sources.any { it.alias.equals(chosen, ignoreCase = true) }) {
|
|
throw KestrelException("duplicate table alias: $chosen")
|
|
}
|
|
sources.add(Source(chosen, schema.name, schema, rows, offset))
|
|
offset += schema.columns.size
|
|
}
|
|
add(statement.from)
|
|
for (join in statement.joins) add(join.from)
|
|
return sources
|
|
}
|
|
|
|
private inner class Plan(val statement: SelectStatement, val sources: List<Source>) {
|
|
val aggregated: Boolean =
|
|
statement.groupBy.isNotEmpty() ||
|
|
containsAggregate(statement.projection) ||
|
|
(statement.having != null && containsAggregate(statement.having))
|
|
|
|
private val columnNames: List<String> = sources.flatMap { it.schema.columnNames }
|
|
|
|
private val aliasExpressions: Map<String, Expression> =
|
|
(statement.projection as? ExpressionProjection)?.items
|
|
?.mapNotNull { item -> item.alias?.let { it.lowercase() to item.expression } }
|
|
?.toMap() ?: emptyMap()
|
|
|
|
private val orderExpression: Expression? = statement.orderBy?.let { resolveOrderAlias(it.expression) }
|
|
|
|
private fun resolveOrderAlias(expression: Expression): Expression {
|
|
if (expression is ColumnExpression && expression.qualifier == null) {
|
|
aliasExpressions[expression.name.lowercase()]?.let { return it }
|
|
}
|
|
return expression
|
|
}
|
|
|
|
fun resolveIndex(expression: ColumnExpression): Int {
|
|
val qualifier = expression.qualifier
|
|
val matches = mutableListOf<Int>()
|
|
for (source in sources) {
|
|
if (qualifier != null && !source.alias.equals(qualifier, ignoreCase = true)) continue
|
|
val local = source.schema.columnIndex(expression.name) ?: continue
|
|
matches.add(source.offset + local)
|
|
}
|
|
return when {
|
|
matches.isEmpty() -> throw KestrelException("no such column: ${describe(expression)}")
|
|
matches.size > 1 -> throw KestrelException("ambiguous column reference: ${describe(expression)}")
|
|
else -> matches.single()
|
|
}
|
|
}
|
|
|
|
private fun sourceOfGlobal(global: Int): Int =
|
|
sources.indexOfLast { global >= it.offset }
|
|
|
|
private fun referencedSources(expression: Expression): Set<Int> =
|
|
collectColumns(expression).map { sourceOfGlobal(resolveIndex(it)) }.toSet()
|
|
|
|
fun validate() {
|
|
statement.where?.let { validateColumns(it) }
|
|
orderExpression?.let { validateColumns(it) }
|
|
statement.groupBy.forEach { validateColumns(it) }
|
|
statement.having?.let { validateColumns(it) }
|
|
if (statement.projection is ExpressionProjection) {
|
|
statement.projection.items.forEach { validateColumns(it.expression) }
|
|
}
|
|
statement.joins.forEachIndexed { position, join ->
|
|
validateColumns(join.on)
|
|
val allowed = position + 1
|
|
if (referencedSources(join.on).any { it > allowed }) {
|
|
throw KestrelException("ON condition for ${join.tableName} references a table joined later")
|
|
}
|
|
}
|
|
if (aggregated) validateAggregation()
|
|
}
|
|
|
|
private fun validateColumns(expression: Expression) {
|
|
collectColumns(expression).forEach { resolveIndex(it) }
|
|
}
|
|
|
|
private fun validateAggregation() {
|
|
if (statement.projection is StarProjection) {
|
|
throw KestrelException("cannot use * with GROUP BY or aggregate functions")
|
|
}
|
|
val groupColumns = statement.groupBy.filterIsInstance<ColumnExpression>()
|
|
.map { resolveIndex(it) }.toSet()
|
|
if (statement.projection is ExpressionProjection) {
|
|
statement.projection.items.forEach { requireGrouped(it.expression, groupColumns) }
|
|
}
|
|
statement.having?.let { requireGrouped(it, groupColumns) }
|
|
orderExpression?.let { requireGrouped(it, groupColumns) }
|
|
}
|
|
|
|
private fun requireGrouped(expression: Expression, groupColumns: Set<Int>) {
|
|
when (expression) {
|
|
is FunctionExpression -> Unit
|
|
is ColumnExpression -> if (resolveIndex(expression) !in groupColumns) {
|
|
throw KestrelException(
|
|
"column ${describe(expression)} must appear in GROUP BY or be used in an aggregate function"
|
|
)
|
|
}
|
|
is BinaryExpression -> {
|
|
requireGrouped(expression.left, groupColumns)
|
|
requireGrouped(expression.right, groupColumns)
|
|
}
|
|
is UnaryExpression -> requireGrouped(expression.operand, groupColumns)
|
|
is LiteralExpression -> Unit
|
|
}
|
|
}
|
|
|
|
private fun rowResolver(row: List<Value>) =
|
|
kestrel.engine.ColumnResolver { expression -> row[resolveIndex(expression)] }
|
|
|
|
private fun evaluateRow(expression: Expression, row: List<Value>): Value =
|
|
Evaluator(rowResolver(row)).evaluate(expression)
|
|
|
|
fun materialize(): List<List<Value>> {
|
|
var combined = primaryRows().map { it.values }
|
|
statement.joins.forEachIndexed { position, join ->
|
|
combined = executeJoin(combined, join, position + 1)
|
|
}
|
|
statement.where?.let { predicate ->
|
|
combined = combined.filter { isTrue(evaluateRow(predicate, it)) }
|
|
}
|
|
return combined
|
|
}
|
|
|
|
private fun primaryRows(): List<Row> {
|
|
val access = primaryAccess()
|
|
val source = sources.first()
|
|
return when (access) {
|
|
is PrimaryAccess.FullScan -> source.rows
|
|
is PrimaryAccess.IndexScan -> access.rowIndices.map { source.rows[it] }
|
|
}
|
|
}
|
|
|
|
private fun executeJoin(left: List<List<Value>>, join: JoinClause, rightIndex: Int): List<List<Value>> {
|
|
val right = sources[rightIndex]
|
|
val nullRight = List(right.schema.columns.size) { Value.NullValue }
|
|
val hashKeys = hashJoinKeys(join, rightIndex)
|
|
val result = mutableListOf<List<Value>>()
|
|
if (hashKeys != null) {
|
|
val buckets = HashMap<Value, MutableList<Row>>()
|
|
for (rightRow in right.rows) {
|
|
val key = evaluateOnRight(hashKeys.buildExpression, right, rightRow)
|
|
if (key !is Value.NullValue) buckets.getOrPut(key) { mutableListOf() }.add(rightRow)
|
|
}
|
|
for (leftRow in left) {
|
|
val probe = evaluateRow(hashKeys.probeExpression, leftRow)
|
|
val matches = if (probe is Value.NullValue) emptyList() else buckets[probe].orEmpty()
|
|
if (matches.isNotEmpty()) {
|
|
matches.forEach { result.add(leftRow + it.values) }
|
|
} else if (join.type == JoinType.Left) {
|
|
result.add(leftRow + nullRight)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
for (leftRow in left) {
|
|
var matched = false
|
|
for (rightRow in right.rows) {
|
|
val candidate = leftRow + rightRow.values
|
|
if (isTrue(evaluateRow(join.on, candidate))) {
|
|
result.add(candidate)
|
|
matched = true
|
|
}
|
|
}
|
|
if (!matched && join.type == JoinType.Left) result.add(leftRow + nullRight)
|
|
}
|
|
return result
|
|
}
|
|
|
|
private fun evaluateOnRight(expression: Expression, right: Source, rightRow: Row): Value {
|
|
val resolver = kestrel.engine.ColumnResolver { columnExpression ->
|
|
rightRow.values[resolveIndex(columnExpression) - right.offset]
|
|
}
|
|
return Evaluator(resolver).evaluate(expression)
|
|
}
|
|
|
|
private fun hashJoinKeys(join: JoinClause, rightIndex: Int): HashKeys? {
|
|
val condition = join.on
|
|
if (condition !is BinaryExpression || condition.operator != BinaryOperator.Equal) return null
|
|
val built = (0 until rightIndex).toSet()
|
|
val leftRefs = referencedSources(condition.left)
|
|
val rightRefs = referencedSources(condition.right)
|
|
return when {
|
|
rightRefs == setOf(rightIndex) && leftRefs.all { it in built } ->
|
|
HashKeys(buildExpression = condition.right, probeExpression = condition.left)
|
|
leftRefs == setOf(rightIndex) && rightRefs.all { it in built } ->
|
|
HashKeys(buildExpression = condition.left, probeExpression = condition.right)
|
|
else -> null
|
|
}
|
|
}
|
|
|
|
fun primaryAccess(): PrimaryAccess {
|
|
if (statement.joins.isNotEmpty()) return PrimaryAccess.FullScan
|
|
val source = sources.first()
|
|
return indexPlanner.plan(source.tableName, source.alias, source.schema, statement.where)
|
|
}
|
|
|
|
fun projectRows(combined: List<List<Value>>): SelectResult {
|
|
var rows = combined
|
|
orderExpression?.let { expression ->
|
|
val comparator = Comparator<List<Value>> { first, second ->
|
|
ValueOrdering.compare(evaluateRow(expression, first), evaluateRow(expression, second))
|
|
}
|
|
rows = rows.sortedWith(if (statement.orderBy!!.descending) comparator.reversed() else comparator)
|
|
}
|
|
statement.limit?.let {
|
|
if (it < 0) throw KestrelException("LIMIT must not be negative")
|
|
rows = rows.take(it)
|
|
}
|
|
return when (val projection = statement.projection) {
|
|
is StarProjection -> SelectResult(columnNames, rows)
|
|
is ExpressionProjection -> {
|
|
val labels = projection.items.mapIndexed { index, item -> columnLabel(item, index) }
|
|
val projected = rows.map { row -> projection.items.map { evaluateRow(it.expression, row) } }
|
|
SelectResult(labels, projected)
|
|
}
|
|
}
|
|
}
|
|
|
|
fun aggregate(combined: List<List<Value>>): SelectResult =
|
|
GroupAggregation(::resolveIndex, ::evaluateRow, this@SelectRunner::columnLabel)
|
|
.aggregate(statement, orderExpression, combined)
|
|
|
|
fun describe(): List<String> {
|
|
val lines = mutableListOf<String>()
|
|
var body = joinLines()
|
|
if (aggregated) {
|
|
val keys = statement.groupBy.map { describeExpression(it) }
|
|
val header = if (keys.isEmpty()) "Aggregate" else "GroupAggregate keys: ${keys.joinToString(", ")}"
|
|
body = listOf(header) + body.map { " $it" }
|
|
}
|
|
statement.orderBy?.let { body = listOf("Sort") + body.map { " $it" } }
|
|
statement.limit?.let { body = listOf("Limit ${statement.limit}") + body.map { " $it" } }
|
|
lines.addAll(body)
|
|
return lines
|
|
}
|
|
|
|
private fun joinLines(): List<String> {
|
|
var lines = listOf(scanLine(sources.first(), primaryAccess()))
|
|
statement.joins.forEachIndexed { position, join ->
|
|
val right = sources[position + 1]
|
|
val hash = hashJoinKeys(join, position + 1) != null
|
|
val header = if (hash) "Hash Join ${describeExpression(join.on)}" else "Nested Loop ${joinLabel(join)}"
|
|
lines = listOf(header) + lines.map { " -> $it" } + listOf(" -> Seq Scan on ${right.tableName}")
|
|
}
|
|
return lines
|
|
}
|
|
|
|
private fun scanLine(source: Source, access: PrimaryAccess): String = when (access) {
|
|
is PrimaryAccess.FullScan -> "Seq Scan on ${source.tableName}"
|
|
is PrimaryAccess.IndexScan ->
|
|
"Index Scan using ${access.indexName} on ${source.tableName} (${access.description})"
|
|
}
|
|
|
|
private fun joinLabel(join: JoinClause): String =
|
|
if (join.type == JoinType.Left) "Left" else "Inner"
|
|
}
|
|
|
|
private fun columnLabel(item: SelectItem, index: Int): String =
|
|
item.alias ?: when (val expression = item.expression) {
|
|
is ColumnExpression -> expression.name
|
|
is FunctionExpression -> expression.name.lowercase()
|
|
else -> "column${index + 1}"
|
|
}
|
|
|
|
private fun collectColumns(expression: Expression): List<ColumnExpression> = when (expression) {
|
|
is ColumnExpression -> listOf(expression)
|
|
is BinaryExpression -> collectColumns(expression.left) + collectColumns(expression.right)
|
|
is UnaryExpression -> collectColumns(expression.operand)
|
|
is FunctionExpression -> expression.argument?.let { collectColumns(it) } ?: emptyList()
|
|
is LiteralExpression -> emptyList()
|
|
}
|
|
|
|
private fun containsAggregate(projection: kestrel.sql.ast.Projection): Boolean = when (projection) {
|
|
is StarProjection -> false
|
|
is ExpressionProjection -> projection.items.any { containsAggregate(it.expression) }
|
|
}
|
|
|
|
private fun containsAggregate(expression: Expression): Boolean = when (expression) {
|
|
is FunctionExpression -> true
|
|
is BinaryExpression -> containsAggregate(expression.left) || containsAggregate(expression.right)
|
|
is UnaryExpression -> containsAggregate(expression.operand)
|
|
else -> false
|
|
}
|
|
|
|
private fun isTrue(value: Value): Boolean = value is Value.BooleanValue && value.value
|
|
|
|
private fun describe(expression: ColumnExpression): String =
|
|
expression.qualifier?.let { "$it.${expression.name}" } ?: expression.name
|
|
|
|
private fun describeExpression(expression: Expression): String = when (expression) {
|
|
is ColumnExpression -> describe(expression)
|
|
is LiteralExpression -> literalText(expression.value)
|
|
is FunctionExpression -> "${expression.name}(${if (expression.star) "*" else expression.argument?.let { describeExpression(it) } ?: ""})"
|
|
is BinaryExpression -> "${describeExpression(expression.left)} ${expression.operator.symbol} ${describeExpression(expression.right)}"
|
|
is UnaryExpression -> "${expression.operator.symbol} ${describeExpression(expression.operand)}"
|
|
}
|
|
|
|
private fun literalText(value: Value): String = when (value) {
|
|
is Value.IntegerValue -> value.value.toString()
|
|
is Value.RealValue -> value.value.toString()
|
|
is Value.TextValue -> "'${value.value}'"
|
|
is Value.BooleanValue -> value.value.toString()
|
|
is Value.NullValue -> "NULL"
|
|
}
|
|
|
|
private class Source(
|
|
val alias: String,
|
|
val tableName: String,
|
|
val schema: TableSchema,
|
|
val rows: List<Row>,
|
|
val offset: Int
|
|
)
|
|
|
|
private class HashKeys(val buildExpression: Expression, val probeExpression: Expression)
|
|
}
|