diff --git a/src/main/kotlin/kestrel/catalog/Catalog.kt b/src/main/kotlin/kestrel/catalog/Catalog.kt index d95358d..2989441 100644 --- a/src/main/kotlin/kestrel/catalog/Catalog.kt +++ b/src/main/kotlin/kestrel/catalog/Catalog.kt @@ -1,53 +1,711 @@ package kestrel.catalog import kestrel.KestrelException +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.storage.Collation import kestrel.storage.Row import kestrel.storage.Table +import kestrel.storage.Value class CatalogSnapshot( val tables: Map>>, - val indexDefinitions: List + val indexDefinitions: List, + val views: Map, + val triggers: Map, + /** + * A reference copy of the map, not a deep copy of each [Sequence] - see [Sequence]'s own doc + * for why that is exactly what makes ROLLBACK undo CREATE/DROP/ALTER SEQUENCE while leaving a + * NEXTVAL advance in place. + */ + val sequences: Map ) class Catalog { private val tables = linkedMapOf() private val indexes = linkedMapOf() + private val views = linkedMapOf() + private val triggers = linkedMapOf() + private val sequences = linkedMapOf() fun createTable(schema: TableSchema) { val key = schema.name.lowercase() if (tables.containsKey(key)) { throw KestrelException("table '${schema.name}' already exists") } + if (views.containsKey(key)) { + throw KestrelException("a view named '${schema.name}' already exists") + } tables[key] = Table(schema) } - fun dropTable(tableName: String) { + fun dropTable(tableName: String, cascade: Boolean = false) { val key = tableName.lowercase() - if (tables.remove(key) == null) { + if (!tables.containsKey(key)) { + if (views.containsKey(key)) { + throw KestrelException("'$tableName' is a view, use DROP VIEW") + } throw KestrelException("no such table: $tableName") } + dropDependentViews(tableName, cascade) + dropDependentForeignKeys(tableName, cascade) + tables.remove(key) indexes.entries.removeAll { it.value.definition.tableName.equals(tableName, ignoreCase = true) } + // A trigger without its table makes no sense to keep around - unlike a FOREIGN KEY + // reference, nothing else depends on it existing, so this always happens, CASCADE or not. + triggers.entries.removeAll { it.value.tableName.equals(tableName, ignoreCase = true) } } - fun table(tableName: String): Table = - tables[tableName.lowercase()] ?: throw KestrelException("no such table: $tableName") + /** + * Mirrors Postgres semantics: a FOREIGN KEY pointing at [tableName] blocks the drop under + * RESTRICT, while CASCADE only drops the constraint itself (the referencing table and its + * rows survive) - dropping a table never implies dropping every table that happens to + * reference it. A self-referencing FK is excluded because it disappears together with the + * table being dropped, with nothing left dangling in any other table. + */ + private fun dropDependentForeignKeys(tableName: String, cascade: Boolean) { + val referencing = tables.values.filter { table -> + !table.schema.name.equals(tableName, ignoreCase = true) && + table.schema.foreignKeys.any { it.referencedTable.equals(tableName, ignoreCase = true) } + } + if (referencing.isEmpty()) return + if (!cascade) { + val names = referencing.map { it.schema.name } + throw KestrelException( + "cannot drop '$tableName' because table(s) ${names.joinToString(", ")} have a FOREIGN KEY " + + "referencing it; use CASCADE" + ) + } + for (table in referencing) { + val key = table.schema.name.lowercase() + val newSchema = TableSchema( + table.schema.name, + table.schema.columns, + table.schema.foreignKeys.filterNot { it.referencedTable.equals(tableName, ignoreCase = true) }, + table.schema.checks, + table.schema.uniqueConstraints, + table.schema.compositePrimaryKey, + table.schema.temporary + ) + val replacement = Table(newSchema) + replacement.rows.addAll(table.rows) + tables[key] = replacement + } + } + + /** + * TRUNCATE's own dependency handling, deliberately NOT built on [dropDependentForeignKeys] or + * `ForeignKeyEnforcer.cascadeDelete`: wiping every row of [tableNames] makes EVERY row of a + * table with a FOREIGN KEY into any of them dangling at once, not just rows that happened to + * match specific parent rows about to disappear one at a time - so RESTRICT (default) blocks on + * any other table's FOREIGN KEY referencing one of [tableNames] regardless of that FK's own + * `onDelete` action (real TRUNCATE never consults it - there is no per-row delete to run it + * against), and [cascade] means "also truncate the whole referencing table", computed as a + * transitive closure exactly like [dropDependentViews]'s [transitiveViewDependents], not a + * per-row plan/apply cascade. A self-reference (a table whose own FOREIGN KEY points at itself, + * or two requested tables that reference each other) never blocks or duplicates, since every + * requested table starts in the closure before the first pass over `tables` even runs. + * + * Returns every table actually truncated - [tableNames] plus, under CASCADE, every transitive + * referencer - in an unspecified order, so the caller can report and (for RESTART IDENTITY) + * restart sequences for all of them, not just the ones named in the statement. + */ + fun truncateTables(tableNames: List, cascade: Boolean, restartIdentity: Boolean): List { + val requestedKeys = tableNames.map { name -> + val key = name.lowercase() + if (!tables.containsKey(key)) { + if (views.containsKey(key)) throw KestrelException("'$name' is a view, TRUNCATE only applies to tables") + throw KestrelException("no such table: $name") + } + key + } + val closure = truncateClosure(requestedKeys.toCollection(linkedSetOf()), cascade, tableNames) + for (key in closure) { + val table = tables.getValue(key) + table.rows.clear() + if (restartIdentity) restartOwnedSequences(table) + } + for (key in closure) reindexTable(tables.getValue(key).schema.name) + return closure.map { tables.getValue(it).schema.name } + } + + private fun truncateClosure(found: LinkedHashSet, cascade: Boolean, requestedNames: List): LinkedHashSet { + var frontier: Set = found + while (true) { + val referencing = tables.values.filter { table -> + table.schema.name.lowercase() !in found && + table.schema.foreignKeys.any { fk -> fk.referencedTable.lowercase() in frontier } + } + if (referencing.isEmpty()) return found + if (!cascade) { + val names = referencing.map { it.schema.name } + throw KestrelException( + "cannot truncate ${requestedNames.joinToString(", ")} because table(s) " + + "${names.joinToString(", ")} have a FOREIGN KEY referencing them; use CASCADE" + ) + } + val next = referencing.map { it.schema.name.lowercase() }.toSet() + found.addAll(next) + frontier = next + } + } + + /** + * `ALTER TABLE t ALTER COLUMN col TYPE ...` - replaces an EXISTING column's declared + * [Column.type]/[Column.precision]/[Column.scale] in place, keeping its name/position/ + * notNull/default/generated/collation untouched. Unlike [addColumn]'s per-row `valueForRow` + * callback, [newRows] arrives already fully built by [kestrel.engine.Executor] - every row's + * `USING` (or implicit "recompute the column as its own old value") expression evaluated and + * coerced, and every NOT NULL/CHECK re-validated against the NEW schema - BEFORE this is ever + * called, so this method never observes a row that fails validation partway through: either + * all of [newRows] is already known-good and gets installed atomically, or the caller never + * reaches this method at all. + */ + fun changeColumnType(tableName: String, columnName: String, newColumn: Column, newRows: List) { + val schema = table(tableName).schema + val columnIndex = schema.columnIndex(columnName) + ?: throw KestrelException("no such column: $columnName in table $tableName") + val newColumns = schema.columns.toMutableList().also { it[columnIndex] = newColumn } + val newSchema = TableSchema( + schema.name, newColumns, schema.foreignKeys, schema.checks, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, newRows) + } + + /** + * RESTART IDENTITY has no explicit "sequence OWNED BY column" catalog entry to consult (Postgres + * has one; this engine does not) - a column counts as identity-backed exactly when its own + * DEFAULT is a bare `NEXTVAL('seq')`, the same structural signal `SqlWriter`/`executeAddColumn` + * already treat as "this column is an auto-increment". A dangling reference to an + * already-dropped sequence (DROP SEQUENCE has no dependent-column check of its own) is silently + * skipped rather than thrown, matching how the rest of this engine only ever discovers that gap + * lazily, at the next NEXTVAL/INSERT. + */ + private fun restartOwnedSequences(table: Table) { + for (column in table.schema.columns) { + val default = column.default + if (default is NextValueExpression) { + val sequence = sequenceOrNull(default.sequenceName) ?: continue + sequences[default.sequenceName.lowercase()] = sequence.altered(sequence.definition, restartAt = sequence.definition.startValue) + } + } + } + + /** + * Appends a new column (already validated by the executor) to every existing row. + * [valueForRow] computes the new column's value from each row's OLD (pre-append) values - a + * constant lambda for a plain DEFAULT, or a fresh per-row expression evaluation for GENERATED + * ALWAYS AS, mirroring how INSERT already treats those two cases differently. + */ + fun addColumn(tableName: String, column: Column, valueForRow: (Row) -> Value) { + val table = table(tableName) + val schema = table.schema + if (schema.columnIndex(column.name) != null) { + throw KestrelException("duplicate column name: ${column.name}") + } + val newSchema = TableSchema( + schema.name, + schema.columns + column, + schema.foreignKeys, + schema.checks, + schema.uniqueConstraints, + schema.compositePrimaryKey, + schema.temporary + ) + val newRows = table.rows.map { row -> Row(row.values + valueForRow(row)) } + replaceTable(tableName, newSchema, newRows) + } + + /** + * RESTRICT (default) rejects the drop if [columnName] is used by another table's FOREIGN KEY, + * this table's own FOREIGN KEY, a UNIQUE constraint, a CHECK constraint, or an index - CASCADE + * removes all of those alongside the column itself. Dropping a PRIMARY KEY column is never + * allowed, with or without CASCADE: this engine has no standalone "DROP CONSTRAINT", so + * redesign the PRIMARY KEY (or drop the table) instead. + */ + fun dropColumn(tableName: String, columnName: String, cascade: Boolean) { + val table = table(tableName) + val schema = table.schema + schema.columnIndex(columnName) ?: throw KestrelException("no such column: $columnName in table $tableName") + if (schema.columns.size == 1) { + throw KestrelException("cannot drop $columnName: table $tableName would be left with no columns") + } + if (schema.primaryKeyColumns.any { it.equals(columnName, ignoreCase = true) }) { + throw KestrelException("cannot drop column $columnName because it is part of the PRIMARY KEY of $tableName") + } + + // A GENERATED column can only ever read this table's own non-GENERATED columns (never + // another GENERATED one - validateGeneratedDefinitions forbids that chain), so dropping + // columnName orphans any GENERATED column whose expression reads it, exactly like a CHECK + // constraint referencing it - it must be dropped right along with columnName (RESTRICT + // blocks the whole drop instead, same as every other dependent kind below). GENERATED can + // never be a PRIMARY KEY column (Executor.validateGeneratedDefinitions), so there is no + // "cascaded into dropping the PRIMARY KEY" case to guard against here. + val ownGeneratedDependents = schema.columns.filter { column -> + !column.name.equals(columnName, ignoreCase = true) && + column.generated?.let { CheckExpressionColumns.references(it, columnName) } == true + } + val droppedNames = (ownGeneratedDependents.map { it.name } + columnName).map { it.lowercase() }.toSet() + val droppedIndices = schema.columns.withIndex() + .filter { (_, column) -> column.name.lowercase() in droppedNames } + .map { it.index } + .toSet() + + val referencingForeignKeys = tables.values.filter { other -> + !other.schema.name.equals(tableName, ignoreCase = true) && + other.schema.foreignKeys.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.any { it.lowercase() in droppedNames } + } + } + val ownForeignKeys = schema.foreignKeys.filter { fk -> fk.columns.any { it.lowercase() in droppedNames } } + val ownUniqueConstraints = + schema.uniqueConstraints.filter { unique -> unique.columns.any { it.lowercase() in droppedNames } } + val ownChecks = schema.checks.filter { check -> droppedNames.any { name -> CheckExpressionColumns.references(check.expression, name) } } + val dependentIndexes = indexes.values.filter { + it.definition.tableName.equals(tableName, ignoreCase = true) && + it.definition.columnNames.any { col -> col.lowercase() in droppedNames } + } + + if (!cascade) { + val reasons = mutableListOf() + if (ownGeneratedDependents.isNotEmpty()) { + reasons.add( + "column(s) ${ownGeneratedDependents.joinToString(", ") { it.name }} are GENERATED columns that depend on it" + ) + } + if (referencingForeignKeys.isNotEmpty()) { + reasons.add( + "table(s) ${referencingForeignKeys.joinToString(", ") { it.schema.name }} have a FOREIGN KEY referencing it" + ) + } + if (ownForeignKeys.isNotEmpty()) reasons.add("it is used by one of its own FOREIGN KEY constraints") + if (ownUniqueConstraints.isNotEmpty()) reasons.add("it is used by a UNIQUE constraint") + if (ownChecks.isNotEmpty()) reasons.add("it is used by a CHECK constraint") + if (dependentIndexes.isNotEmpty()) { + reasons.add("index(es) ${dependentIndexes.joinToString(", ") { it.definition.name }} are defined on it") + } + if (reasons.isNotEmpty()) { + throw KestrelException( + "cannot drop column $columnName from $tableName because ${reasons.joinToString("; ")}; use CASCADE" + ) + } + } + + for (other in referencingForeignKeys) { + val key = other.schema.name.lowercase() + val newSchema = TableSchema( + other.schema.name, + other.schema.columns, + other.schema.foreignKeys.filterNot { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.any { it.lowercase() in droppedNames } + }, + other.schema.checks, + other.schema.uniqueConstraints, + other.schema.compositePrimaryKey, + other.schema.temporary + ) + val replacement = Table(newSchema) + replacement.rows.addAll(other.rows) + tables[key] = replacement + } + for (index in dependentIndexes) indexes.remove(index.definition.name.lowercase()) + + val newColumns = schema.columns.filterNot { it.name.lowercase() in droppedNames } + val newForeignKeys = schema.foreignKeys - ownForeignKeys.toSet() + val newUniqueConstraints = schema.uniqueConstraints - ownUniqueConstraints.toSet() + val newChecks = schema.checks - ownChecks.toSet() + val newSchema = TableSchema(schema.name, newColumns, newForeignKeys, newChecks, newUniqueConstraints, schema.compositePrimaryKey, schema.temporary) + val newRows = table.rows.map { row -> Row(row.values.filterIndexed { valueIndex, _ -> valueIndex !in droppedIndices }) } + replaceTable(tableName, newSchema, newRows) + } + + /** + * Always fully propagates rather than being rejected the way [dropColumn] can be: the + * identifier is rewritten everywhere it is used by name rather than removed, so nothing + * becomes newly invalid - this table's own PRIMARY KEY/FOREIGN KEY/UNIQUE/CHECK definitions, + * any other table's FOREIGN KEY referencing this column, and any index defined on it. + */ + fun renameColumn(tableName: String, oldName: String, newName: String) { + val table = table(tableName) + val schema = table.schema + if (schema.columnIndex(oldName) == null) { + throw KestrelException("no such column: $oldName in table $tableName") + } + if (!oldName.equals(newName, ignoreCase = true) && schema.columnIndex(newName) != null) { + throw KestrelException("column $newName already exists in table $tableName") + } + + val newColumns = schema.columns.map { column -> + val renamed = if (column.name.equals(oldName, ignoreCase = true)) column.copy(name = newName) else column + // A GENERATED column can only ever reference this table's own columns, so renaming one + // of them must rewrite that reference too, exactly like a CHECK constraint's own + // reference to it (below) - otherwise the GENERATED expression would silently keep + // pointing at a name that no longer exists. + renamed.generated?.let { renamed.copy(generated = CheckExpressionColumns.rename(it, oldName, newName)) } ?: renamed + } + val newForeignKeys = schema.foreignKeys.map { fk -> + val selfReference = fk.referencedTable.equals(tableName, ignoreCase = true) + fk.copy( + columns = renameInList(fk.columns, oldName, newName), + referencedColumns = if (selfReference) renameInList(fk.referencedColumns, oldName, newName) else fk.referencedColumns + ) + } + val newUniqueConstraints = schema.uniqueConstraints.map { it.copy(columns = renameInList(it.columns, oldName, newName)) } + val newChecks = schema.checks.map { it.copy(expression = CheckExpressionColumns.rename(it.expression, oldName, newName)) } + val newCompositePrimaryKey = renameInList(schema.compositePrimaryKey, oldName, newName) + val newSchema = TableSchema(schema.name, newColumns, newForeignKeys, newChecks, newUniqueConstraints, newCompositePrimaryKey, schema.temporary) + replaceTable(tableName, newSchema, table.rows) + + for ((key, other) in tables.entries.toList()) { + if (other.schema.name.equals(tableName, ignoreCase = true)) continue + val touched = other.schema.foreignKeys.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.any { it.equals(oldName, ignoreCase = true) } + } + if (!touched) continue + val updatedForeignKeys = other.schema.foreignKeys.map { fk -> + if (fk.referencedTable.equals(tableName, ignoreCase = true)) { + fk.copy(referencedColumns = renameInList(fk.referencedColumns, oldName, newName)) + } else { + fk + } + } + val updatedSchema = TableSchema( + other.schema.name, + other.schema.columns, + updatedForeignKeys, + other.schema.checks, + other.schema.uniqueConstraints, + other.schema.compositePrimaryKey, + other.schema.temporary + ) + val replacement = Table(updatedSchema) + replacement.rows.addAll(other.rows) + tables[key] = replacement + } + + for ((key, index) in indexes.entries.toList()) { + if (!index.definition.tableName.equals(tableName, ignoreCase = true)) continue + if (index.definition.columnNames.none { it.equals(oldName, ignoreCase = true) }) continue + val renamedDefinition = index.definition.copy(columnNames = renameInList(index.definition.columnNames, oldName, newName)) + val renamed = SortedIndex(renamedDefinition, collationsFor(newSchema, renamedDefinition.columnNames)) + reindexOne(tables.getValue(tableName.lowercase()), renamed) + indexes[key] = renamed + } + } + + /** + * Renames a table and propagates the identifier everywhere it is used by name, mirroring + * [renameColumn]'s "always fully propagates" contract: this table's own FOREIGN KEY(s) that + * self-reference it, any other table's FOREIGN KEY pointing at it, every index defined on it, + * and every view whose defining query mentions it anywhere (FROM/JOIN, derived tables, CTEs, + * set operations, scalar/EXISTS/IN subqueries - see [RelationRename]). + */ + fun renameTable(oldName: String, newName: String) { + val existing = table(oldName) + if (!oldName.equals(newName, ignoreCase = true)) { + val key = newName.lowercase() + if (tables.containsKey(key)) throw KestrelException("table '$newName' already exists") + if (views.containsKey(key)) throw KestrelException("a view named '$newName' already exists") + } + + val selfReferencing = existing.schema.foreignKeys.any { it.referencedTable.equals(oldName, ignoreCase = true) } + val newForeignKeys = if (selfReferencing) { + existing.schema.foreignKeys.map { fk -> + if (fk.referencedTable.equals(oldName, ignoreCase = true)) fk.copy(referencedTable = newName) else fk + } + } else { + existing.schema.foreignKeys + } + val newSchema = TableSchema( + newName, + existing.schema.columns, + newForeignKeys, + existing.schema.checks, + existing.schema.uniqueConstraints, + existing.schema.compositePrimaryKey, + existing.schema.temporary + ) + tables.remove(oldName.lowercase()) + val replacement = Table(newSchema) + replacement.rows.addAll(existing.rows) + tables[newName.lowercase()] = replacement + + for ((key, other) in tables.entries.toList()) { + if (other.schema.name.equals(newName, ignoreCase = true)) continue + val touched = other.schema.foreignKeys.any { it.referencedTable.equals(oldName, ignoreCase = true) } + if (!touched) continue + val updatedForeignKeys = other.schema.foreignKeys.map { fk -> + if (fk.referencedTable.equals(oldName, ignoreCase = true)) fk.copy(referencedTable = newName) else fk + } + val updatedSchema = TableSchema( + other.schema.name, + other.schema.columns, + updatedForeignKeys, + other.schema.checks, + other.schema.uniqueConstraints, + other.schema.compositePrimaryKey, + other.schema.temporary + ) + val replacementOther = Table(updatedSchema) + replacementOther.rows.addAll(other.rows) + tables[key] = replacementOther + } + + for ((key, index) in indexes.entries.toList()) { + if (!index.definition.tableName.equals(oldName, ignoreCase = true)) continue + val renamedDefinition = index.definition.copy(tableName = newName) + indexes[key] = SortedIndex(renamedDefinition, collationsFor(newSchema, renamedDefinition.columnNames)).also { renamed -> + reindexOne(tables.getValue(newName.lowercase()), renamed) + } + } + + for ((key, view) in views.entries.toList()) { + if (!DependencyAnalysis.referencesRelation(view.query, oldName)) continue + views[key] = view.copy(query = RelationRename.rename(view.query, oldName, newName)) + } + + for ((key, trigger) in triggers.entries.toList()) { + if (!trigger.tableName.equals(oldName, ignoreCase = true)) continue + triggers[key] = trigger.copy(tableName = newName) + } + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name FOREIGN KEY (...) REFERENCES ...` - installs a FOREIGN KEY + * onto an already-existing table. Unlike [createTable]'s FOREIGN KEY definitions, this never + * touches existing rows itself: [kestrel.engine.Executor.executeAddConstraint] validates every + * existing row against [foreignKey] (via [kestrel.engine.ForeignKeyEnforcer.validateRow]) BEFORE + * calling this, so by the time the schema actually changes the constraint is already known to + * hold for every row already in the table. + */ + fun addForeignKey(tableName: String, foreignKey: ForeignKeyConstraint) { + val table = table(tableName) + val schema = table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys + foreignKey, schema.checks, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, table.rows) + } + + /** [addForeignKey]'s counterpart for `ADD CONSTRAINT name UNIQUE (...)` - see its own doc for the validate-before-install contract. */ + fun addUniqueConstraint(tableName: String, unique: UniqueConstraint) { + val table = table(tableName) + val schema = table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks, + schema.uniqueConstraints + unique, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, table.rows) + } + + /** [addForeignKey]'s counterpart for `ADD CONSTRAINT name CHECK (...)` - see its own doc for the validate-before-install contract. */ + fun addCheckConstraint(tableName: String, check: CheckConstraint) { + val table = table(tableName) + val schema = table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks + check, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, table.rows) + } + + /** + * `ALTER TABLE t DROP CONSTRAINT name [RESTRICT | CASCADE]` - finds a named CHECK, UNIQUE, or + * FOREIGN KEY constraint on [tableName] (case-insensitive) and removes it. A PRIMARY KEY is + * never a match: it has no name to compare against (see [dropColumn]'s own doc on why + * redesigning the PRIMARY KEY, or dropping the table, is the only way to get rid of one), so a + * name that happens to target one simply falls through to "no such constraint" like any other + * unknown name. Only a UNIQUE constraint has cross-table fallout worth a RESTRICT/CASCADE + * decision - see [dropUniqueConstraint]. + */ + fun dropConstraint(tableName: String, name: String, cascade: Boolean) { + val table = table(tableName) + val schema = table.schema + val foreignKey = schema.foreignKeys.find { it.name?.equals(name, ignoreCase = true) == true } + val check = schema.checks.find { it.name?.equals(name, ignoreCase = true) == true } + val unique = schema.uniqueConstraints.find { it.name?.equals(name, ignoreCase = true) == true } + when { + foreignKey != null -> { + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys - foreignKey, schema.checks, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, table.rows) + } + check != null -> { + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks - check, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + replaceTable(tableName, newSchema, table.rows) + } + unique != null -> dropUniqueConstraint(tableName, unique, cascade) + else -> throw KestrelException("no such constraint: $name in table $tableName") + } + } + + /** + * A FOREIGN KEY's `referencedColumns` must exactly match one of the parent's enforced-unique + * key groups ([TableSchema.enforcedUniqueKeyGroups] - PRIMARY KEY or a UNIQUE constraint) to + * have been accepted in the first place, so dropping a UNIQUE constraint can silently invalidate + * any FOREIGN KEY (on this table itself, via a self-reference, or on any other table) that leans + * on exactly this column set for its uniqueness guarantee - UNLESS another remaining group + * (another UNIQUE constraint over the same columns, or the PRIMARY KEY) still covers it, in + * which case nothing actually depends on THIS particular constraint continuing to exist. + * RESTRICT (default) blocks the drop when a real dependent exists; CASCADE drops that FOREIGN + * KEY right along with it, mirroring [dropDependentForeignKeys]. + */ + private fun dropUniqueConstraint(tableName: String, unique: UniqueConstraint, cascade: Boolean) { + val schema = table(tableName).schema + val remainingUniques = schema.uniqueConstraints - unique + val targetSet = unique.columns.map { it.lowercase() }.toSet() + val stillEnforced = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks, remainingUniques, + schema.compositePrimaryKey, schema.temporary + ).enforcedUniqueKeyGroups().any { group -> group.map { it.lowercase() }.toSet() == targetSet } + + fun dependsOnTarget(fk: ForeignKeyConstraint) = + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.map { it.lowercase() }.toSet() == targetSet + + val dependents = if (stillEnforced) emptyList() else tables.values.filter { other -> other.schema.foreignKeys.any(::dependsOnTarget) } + if (!cascade && dependents.isNotEmpty()) { + val names = dependents.map { it.schema.name }.distinct() + throw KestrelException( + "cannot drop constraint ${unique.name} because table(s) ${names.joinToString(", ")} have a " + + "FOREIGN KEY referencing it; use CASCADE" + ) + } + for (other in dependents) { + val key = other.schema.name.lowercase() + val newSchema = TableSchema( + other.schema.name, other.schema.columns, other.schema.foreignKeys.filterNot(::dependsOnTarget), + other.schema.checks, other.schema.uniqueConstraints, other.schema.compositePrimaryKey, other.schema.temporary + ) + val replacement = Table(newSchema) + replacement.rows.addAll(other.rows) + tables[key] = replacement + } + // Re-fetched: a self-referencing FOREIGN KEY on tableName itself may have just been rewritten + // above (dependents is not restricted to OTHER tables - a self-reference is a real dependent + // here, unlike dropDependentForeignKeys' table-drop case where it disappears for free). + val current = table(tableName) + val newSchema = TableSchema( + current.schema.name, current.schema.columns, current.schema.foreignKeys, current.schema.checks, + current.schema.uniqueConstraints - unique, current.schema.compositePrimaryKey, current.schema.temporary + ) + replaceTable(tableName, newSchema, current.rows) + } + + private fun renameInList(names: List, oldName: String, newName: String): List = + names.map { if (it.equals(oldName, ignoreCase = true)) newName else it } + + private fun replaceTable(tableName: String, newSchema: TableSchema, rows: List) { + val key = tableName.lowercase() + val replacement = Table(newSchema) + replacement.rows.addAll(rows) + tables[key] = replacement + } + + fun table(tableName: String): Table { + val key = tableName.lowercase() + tables[key]?.let { return it } + if (views.containsKey(key)) { + throw KestrelException("cannot modify view: $tableName") + } + throw KestrelException("no such table: $tableName") + } fun tableOrNull(tableName: String): Table? = tables[tableName.lowercase()] val tableNames: List get() = tables.values.map { it.schema.name } + fun createView(name: String, query: QueryStatement, orReplace: Boolean) { + val key = name.lowercase() + if (tables.containsKey(key)) { + throw KestrelException("a table named '$name' already exists") + } + if (!orReplace && views.containsKey(key)) { + throw KestrelException("view '$name' already exists") + } + views[key] = ViewDefinition(name, query) + } + + fun dropView(name: String, cascade: Boolean = false) { + val key = name.lowercase() + if (!views.containsKey(key)) { + if (tables.containsKey(key)) { + throw KestrelException("'$name' is a table, use DROP TABLE") + } + throw KestrelException("no such view: $name") + } + dropDependentViews(name, cascade) + views.remove(key) + } + + /** + * Computes the full transitive closure of views depending on [name] (directly or + * through other views) before touching anything, so a "diamond" - two different + * dependents that both end up pulling in the same further-removed view - can't + * make a later removal fail with "no such view" because an earlier one already + * deleted it. + */ + private fun dropDependentViews(name: String, cascade: Boolean) { + val closure = transitiveViewDependents(name.lowercase()) + if (closure.isEmpty()) return + if (!cascade) { + val names = closure.mapNotNull { views[it]?.name } + throw KestrelException( + "cannot drop '$name' because view(s) ${names.joinToString(", ")} depend on it; use CASCADE" + ) + } + for (key in closure) views.remove(key) + } + + private fun transitiveViewDependents(rootKey: String): Set { + val found = linkedSetOf() + var frontier = setOf(rootKey) + while (true) { + val next = views.values + .filter { it.name.lowercase() != rootKey && it.name.lowercase() !in found } + .filter { view -> frontier.any { DependencyAnalysis.referencesRelation(view.query, it) } } + .map { it.name.lowercase() } + .toSet() + if (next.isEmpty()) return found + found.addAll(next) + frontier = next + } + } + + fun viewOrNull(name: String): ViewDefinition? = views[name.lowercase()] + + val viewNames: List get() = views.values.map { it.name } + fun createIndex(definition: IndexDefinition) { val key = definition.name.lowercase() if (indexes.containsKey(key)) { throw KestrelException("index '${definition.name}' already exists") } + require(definition.columnNames.isNotEmpty()) { "index must cover at least one column" } val table = table(definition.tableName) - if (table.schema.columnIndex(definition.columnName) == null) { - throw KestrelException("no such column: ${definition.columnName} in table ${definition.tableName}") + for (columnName in definition.columnNames) { + if (table.schema.columnIndex(columnName) == null) { + throw KestrelException("no such column: $columnName in table ${definition.tableName}") + } } - val index = SortedIndex(definition) - indexes[key] = index + // Populated before registering in [indexes] so that a UNIQUE index created over a table + // with pre-existing duplicate values fails cleanly, without leaving a half-built index + // behind for later statements to trip over. + val index = SortedIndex(definition, collationsFor(table.schema, definition.columnNames)) reindexOne(table, index) + indexes[key] = index } fun dropIndex(indexName: String) { @@ -56,9 +714,9 @@ class Catalog { } } - fun indexOn(tableName: String, columnName: String): SortedIndex? = indexes.values.firstOrNull { - it.definition.tableName.equals(tableName, ignoreCase = true) && - it.definition.columnName.equals(columnName, ignoreCase = true) + /** Every index defined on [tableName], single-column or composite, in creation order. */ + fun indexesOn(tableName: String): List = indexes.values.filter { + it.definition.tableName.equals(tableName, ignoreCase = true) } val indexNames: List get() = indexes.values.map { it.definition.name } @@ -72,21 +730,100 @@ class Catalog { .forEach { reindexOne(table, it) } } + /** + * Fires in [name]'s creation order for a given table/timing/event, matching the order + * multiple triggers on the same event fire in Postgres/SQLite. + */ + fun createTrigger(definition: TriggerDefinition) { + val key = definition.name.lowercase() + if (triggers.containsKey(key)) { + throw KestrelException("trigger '${definition.name}' already exists") + } + table(definition.tableName) + triggers[key] = definition + } + + fun dropTrigger(name: String) { + if (triggers.remove(name.lowercase()) == null) { + throw KestrelException("no such trigger: $name") + } + } + + fun triggersOn(tableName: String, timing: TriggerTiming, event: TriggerEvent): List = + triggers.values.filter { + it.tableName.equals(tableName, ignoreCase = true) && it.timing == timing && it.event == event + } + + fun hasTriggers(tableName: String): Boolean = triggers.values.any { it.tableName.equals(tableName, ignoreCase = true) } + + val triggerNames: List get() = triggers.values.map { it.name } + + fun triggerDefinitions(): List = triggers.values.toList() + + /** + * `name` must not already be a sequence (redundant CREATE); unlike tables/views, sequence + * names live in their own namespace here (Postgres puts them in the same relation namespace as + * tables, but this engine has no need for that overlap check to matter for anything it + * implements). + */ + fun createSequence(definition: SequenceDefinition) { + val key = definition.name.lowercase() + if (sequences.containsKey(key)) { + throw KestrelException("sequence '${definition.name}' already exists") + } + sequences[key] = Sequence(definition) + } + + fun dropSequence(name: String) { + if (sequences.remove(name.lowercase()) == null) { + throw KestrelException("no such sequence: $name") + } + } + + /** + * Always installs a brand new [Sequence] instance (via [Sequence.altered]) rather than mutating + * the existing one in place - see [Sequence]'s own doc for why that is what makes ALTER SEQUENCE + * (unlike NEXTVAL) transactional. + */ + fun alterSequence(name: String, newDefinition: SequenceDefinition, restartAt: Long?) { + val key = name.lowercase() + val existing = sequences[key] ?: throw KestrelException("no such sequence: $name") + sequences[key] = existing.altered(newDefinition, restartAt) + } + + fun sequence(name: String): Sequence = sequenceOrNull(name) ?: throw KestrelException("no such sequence: $name") + + fun sequenceOrNull(name: String): Sequence? = sequences[name.lowercase()] + + val sequenceNames: List get() = sequences.values.map { it.definition.name } + + fun sequenceDefinitions(): List = sequences.values.map { it.definition } + fun snapshot(): CatalogSnapshot = CatalogSnapshot( tables = tables.mapValues { (_, table) -> table.schema to table.rows.toList() }, - indexDefinitions = indexes.values.map { it.definition } + indexDefinitions = indexes.values.map { it.definition }, + views = views.toMap(), + triggers = triggers.toMap(), + sequences = sequences.toMap() ) fun restore(snapshot: CatalogSnapshot) { tables.clear() indexes.clear() + views.clear() + triggers.clear() + sequences.clear() + views.putAll(snapshot.views) + triggers.putAll(snapshot.triggers) + sequences.putAll(snapshot.sequences) for ((key, image) in snapshot.tables) { val table = Table(image.first) table.rows.addAll(image.second) tables[key] = table } for (definition in snapshot.indexDefinitions) { - val index = SortedIndex(definition) + val schema = tables.getValue(definition.tableName.lowercase()).schema + val index = SortedIndex(definition, collationsFor(schema, definition.columnNames)) indexes[definition.name.lowercase()] = index reindexOne(tables.getValue(definition.tableName.lowercase()), index) } @@ -94,7 +831,13 @@ class Catalog { private fun reindexOne(table: Table, index: SortedIndex) { index.clear() - val columnIndex = table.schema.columnIndex(index.definition.columnName)!! - table.rows.forEachIndexed { rowIndex, row -> index.add(row.values[columnIndex], rowIndex) } + val columnIndices = index.definition.columnNames.map { table.schema.columnIndex(it)!! } + table.rows.forEachIndexed { rowIndex, row -> + index.add(columnIndices.map { row.values[it] }, rowIndex) + } } + + /** One [Collation] per entry of [columnNames], in order - [Collation.Binary] for any name [schema] cannot resolve. */ + private fun collationsFor(schema: TableSchema, columnNames: List): List = + columnNames.map { schema.column(it)?.collation ?: Collation.Binary } } diff --git a/src/main/kotlin/kestrel/catalog/CheckExpressionColumns.kt b/src/main/kotlin/kestrel/catalog/CheckExpressionColumns.kt new file mode 100644 index 0000000..5aae404 --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/CheckExpressionColumns.kt @@ -0,0 +1,97 @@ +package kestrel.catalog + +import kestrel.sql.ast.BetweenExpression +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.WindowExpression + +/** + * Walks a CHECK constraint's expression looking for (or renaming) a plain column reference by + * name. Used by [Catalog] to decide whether dropping a column must also drop a dependent CHECK + * (`ALTER TABLE ... DROP COLUMN`) and to keep a CHECK in sync when a column is renamed + * (`ALTER TABLE ... RENAME COLUMN`). The subquery and window-function variants are handled only to + * keep the `when` exhaustive: `Executor.requireNoSubquery` already rejects any CHECK expression + * containing a subquery, an aggregate, or a window function at CREATE TABLE time, so those branches + * are unreachable in practice. + */ +internal object CheckExpressionColumns { + fun references(expression: Expression, columnName: String): Boolean = when (expression) { + is ColumnExpression -> expression.name.equals(columnName, ignoreCase = true) + is LiteralExpression -> false + is BinaryExpression -> references(expression.left, columnName) || references(expression.right, columnName) + is UnaryExpression -> references(expression.operand, columnName) + is CallExpression -> expression.arguments.any { references(it, columnName) } + is FunctionExpression -> expression.argument?.let { references(it, columnName) } ?: false + is IsNullExpression -> references(expression.operand, columnName) + is InExpression -> references(expression.operand, columnName) || expression.values.any { references(it, columnName) } + is BetweenExpression -> + references(expression.operand, columnName) || + references(expression.low, columnName) || + references(expression.high, columnName) + is LikeExpression -> references(expression.operand, columnName) || references(expression.pattern, columnName) + is CaseExpression -> + (expression.operand?.let { references(it, columnName) } ?: false) || + expression.branches.any { references(it.condition, columnName) || references(it.result, columnName) } || + (expression.elseExpression?.let { references(it, columnName) } ?: false) + is CastExpression -> references(expression.operand, columnName) + is CollateExpression -> references(expression.operand, columnName) + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression, is WindowExpression, + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> false + } + + fun rename(expression: Expression, oldName: String, newName: String): Expression = when (expression) { + is ColumnExpression -> + if (expression.name.equals(oldName, ignoreCase = true)) expression.copy(name = newName) else expression + is LiteralExpression -> expression + is BinaryExpression -> expression.copy( + left = rename(expression.left, oldName, newName), + right = rename(expression.right, oldName, newName) + ) + is UnaryExpression -> expression.copy(operand = rename(expression.operand, oldName, newName)) + is CallExpression -> expression.copy(arguments = expression.arguments.map { rename(it, oldName, newName) }) + is FunctionExpression -> expression.copy(argument = expression.argument?.let { rename(it, oldName, newName) }) + is IsNullExpression -> expression.copy(operand = rename(expression.operand, oldName, newName)) + is InExpression -> expression.copy( + operand = rename(expression.operand, oldName, newName), + values = expression.values.map { rename(it, oldName, newName) } + ) + is BetweenExpression -> expression.copy( + operand = rename(expression.operand, oldName, newName), + low = rename(expression.low, oldName, newName), + high = rename(expression.high, oldName, newName) + ) + is LikeExpression -> expression.copy( + operand = rename(expression.operand, oldName, newName), + pattern = rename(expression.pattern, oldName, newName) + ) + is CaseExpression -> expression.copy( + operand = expression.operand?.let { rename(it, oldName, newName) }, + branches = expression.branches.map { + it.copy(condition = rename(it.condition, oldName, newName), result = rename(it.result, oldName, newName)) + }, + elseExpression = expression.elseExpression?.let { rename(it, oldName, newName) } + ) + is CastExpression -> expression.copy(operand = rename(expression.operand, oldName, newName)) + is CollateExpression -> expression.copy(operand = rename(expression.operand, oldName, newName)) + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression, is WindowExpression, + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> expression + } +} diff --git a/src/main/kotlin/kestrel/catalog/DependencyAnalysis.kt b/src/main/kotlin/kestrel/catalog/DependencyAnalysis.kt new file mode 100644 index 0000000..c5ab303 --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/DependencyAnalysis.kt @@ -0,0 +1,123 @@ +package kestrel.catalog + +import kestrel.sql.ast.BetweenExpression +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CommonTableExpression +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FromItem +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.JoinClause +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.StarProjection +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WithQuery + +/** + * Walks a view's defining query looking for a relation name anywhere it could appear: + * FROM/JOIN targets, derived tables, CTE bodies, set-operation branches, and every + * subquery expression (scalar/EXISTS/IN) nested inside WHERE/HAVING/projection/GROUP BY. + * Used by [Catalog] to find what would break (RESTRICT) or need to cascade (CASCADE) + * when a table or view is dropped. + */ +internal object DependencyAnalysis { + fun referencesRelation(query: QueryStatement, name: String): Boolean = when (query) { + is SelectStatement -> referencesInSelect(query, name) + is SetOperationStatement -> referencesRelation(query.left, name) || referencesRelation(query.right, name) + is WithQuery -> query.ctes.any { referencesInCte(it, name) } || referencesRelation(query.body, name) + } + + private fun referencesInCte(cte: CommonTableExpression, name: String): Boolean = + referencesRelation(cte.query, name) + + private fun referencesInSelect(select: SelectStatement, name: String): Boolean { + if (referencesFromItem(select.from, name)) return true + if (select.joins.any { referencesJoin(it, name) }) return true + if (select.where?.let { referencesInExpression(it, name) } == true) return true + if (select.groupBy.any { referencesInExpression(it, name) }) return true + if (select.having?.let { referencesInExpression(it, name) } == true) return true + if (select.distinctOn.any { referencesInExpression(it, name) }) return true + return referencesInProjection(select.projection, name) + } + + private fun referencesJoin(join: JoinClause, name: String): Boolean = + referencesFromItem(join.from, name) || (join.on?.let { referencesInExpression(it, name) } == true) + + private fun referencesFromItem(item: FromItem, name: String): Boolean = when (item) { + is TableSource -> item.name.equals(name, ignoreCase = true) + is SubquerySource -> referencesRelation(item.select, name) + } + + private fun referencesInProjection(projection: Projection, name: String): Boolean = when (projection) { + is StarProjection -> false + is ExpressionProjection -> projection.items.any { referencesInExpression(it.expression, name) } + } + + private fun referencesInExpression(expression: Expression, name: String): Boolean = when (expression) { + is ScalarSubqueryExpression -> referencesRelation(expression.select, name) + is ExistsExpression -> referencesRelation(expression.select, name) + is InSubqueryExpression -> + referencesRelation(expression.select, name) || referencesInExpression(expression.operand, name) + is BinaryExpression -> referencesInExpression(expression.left, name) || referencesInExpression(expression.right, name) + is UnaryExpression -> referencesInExpression(expression.operand, name) + is CallExpression -> expression.arguments.any { referencesInExpression(it, name) } + is IsNullExpression -> referencesInExpression(expression.operand, name) + is InExpression -> + referencesInExpression(expression.operand, name) || expression.values.any { referencesInExpression(it, name) } + is BetweenExpression -> + referencesInExpression(expression.operand, name) || + referencesInExpression(expression.low, name) || + referencesInExpression(expression.high, name) + is LikeExpression -> + referencesInExpression(expression.operand, name) || referencesInExpression(expression.pattern, name) + is CaseExpression -> + (expression.operand?.let { referencesInExpression(it, name) } ?: false) || + expression.branches.any { + referencesInExpression(it.condition, name) || referencesInExpression(it.result, name) + } || + (expression.elseExpression?.let { referencesInExpression(it, name) } ?: false) + is CastExpression -> referencesInExpression(expression.operand, name) + is CollateExpression -> referencesInExpression(expression.operand, name) + is FunctionExpression -> + (expression.argument?.let { referencesInExpression(it, name) } ?: false) || + (expression.separator?.let { referencesInExpression(it, name) } ?: false) || + expression.orderBy.any { referencesInExpression(it.expression, name) } || + (expression.filter?.let { referencesInExpression(it, name) } ?: false) + is WindowExpression -> + (expression.argument?.let { referencesInExpression(it, name) } ?: false) || + (expression.offset?.let { referencesInExpression(it, name) } ?: false) || + (expression.defaultValue?.let { referencesInExpression(it, name) } ?: false) || + (expression.frame?.start?.let { referencesInFrameBound(it, name) } ?: false) || + (expression.frame?.end?.let { referencesInFrameBound(it, name) } ?: false) || + expression.partitionBy.any { referencesInExpression(it, name) } || + expression.orderBy.any { referencesInExpression(it.expression, name) } + is ColumnExpression, is LiteralExpression, is NextValueExpression, is CurrentValueExpression, + is MergeActionExpression, is ParameterExpression -> false + } + + private fun referencesInFrameBound(bound: FrameBound, name: String): Boolean = + bound.offset?.let { referencesInExpression(it, name) } ?: false +} diff --git a/src/main/kotlin/kestrel/catalog/RelationRename.kt b/src/main/kotlin/kestrel/catalog/RelationRename.kt new file mode 100644 index 0000000..a55542c --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/RelationRename.kt @@ -0,0 +1,182 @@ +package kestrel.catalog + +import kestrel.sql.ast.BetweenExpression +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CommonTableExpression +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FromItem +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.GroupingSpec +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.RollupSpec +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.StarProjection +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WithQuery + +/** + * Rewrites every [TableSource] named [oldName] to [newName] throughout a view's defining query - + * FROM/JOIN targets, derived tables, CTE bodies, set-operation branches, and every subquery + * expression (scalar/EXISTS/IN) nested inside WHERE/HAVING/projection/GROUP BY/ORDER BY. Used by + * [Catalog] to keep views working after `ALTER TABLE ... RENAME TO`. + * + * A CTE whose own name equals [oldName] shadows the real table for the rest of that WITH clause + * (same rule SQL itself uses to resolve the name): once such a CTE is seen, later CTEs and the + * body are left untouched, because a bare reference to that name from there on means the CTE, not + * the table being renamed - renaming it there would silently change what the query means. + */ +internal object RelationRename { + fun rename(query: QueryStatement, oldName: String, newName: String): QueryStatement = when (query) { + is SelectStatement -> renameInSelect(query, oldName, newName) + is SetOperationStatement -> query.copy( + left = rename(query.left, oldName, newName), + right = rename(query.right, oldName, newName) + ) + is WithQuery -> renameInWith(query, oldName, newName) + } + + private fun renameInWith(query: WithQuery, oldName: String, newName: String): WithQuery { + val newCtes = mutableListOf() + var shadowed = false + for (cte in query.ctes) { + newCtes.add(if (shadowed) cte else cte.copy(query = rename(cte.query, oldName, newName))) + if (cte.name.equals(oldName, ignoreCase = true)) shadowed = true + } + val newBody = if (shadowed) query.body else rename(query.body, oldName, newName) + return query.copy(ctes = newCtes, body = newBody) + } + + private fun renameInSelect(select: SelectStatement, oldName: String, newName: String): SelectStatement = + select.copy( + from = renameFromItem(select.from, oldName, newName), + joins = select.joins.map { join -> + join.copy( + from = renameFromItem(join.from, oldName, newName), + on = join.on?.let { renameInExpression(it, oldName, newName) } + ) + }, + where = select.where?.let { renameInExpression(it, oldName, newName) }, + groupBy = select.groupBy.map { renameInExpression(it, oldName, newName) }, + groupingSpec = select.groupingSpec?.let { renameInGroupingSpec(it, oldName, newName) }, + having = select.having?.let { renameInExpression(it, oldName, newName) }, + projection = renameInProjection(select.projection, oldName, newName), + orderBy = select.orderBy?.let { orderBy -> + orderBy.copy(keys = orderBy.keys.map { it.copy(expression = renameInExpression(it.expression, oldName, newName)) }) + }, + distinctOn = select.distinctOn.map { renameInExpression(it, oldName, newName) } + ) + + /** + * `groupBy` itself only ever carries the union of a [GroupingSpec]'s expressions (see + * [SelectStatement.groupBy]'s doc), never the spec's own per-set structure - so renaming that + * union alone would leave a stale table name buried inside a subquery nested in + * [RollupSpec]/[CubeSpec]/[GroupingSetsSpec], the one place [GroupAggregation] actually reads for + * execution. Kept in exact lockstep with `groupBy`'s own rewrite above. + */ + private fun renameInGroupingSpec(spec: GroupingSpec, oldName: String, newName: String): GroupingSpec = when (spec) { + is RollupSpec -> spec.copy(expressions = spec.expressions.map { renameInExpression(it, oldName, newName) }) + is CubeSpec -> spec.copy(expressions = spec.expressions.map { renameInExpression(it, oldName, newName) }) + is GroupingSetsSpec -> spec.copy(sets = spec.sets.map { set -> set.map { renameInExpression(it, oldName, newName) } }) + } + + private fun renameFromItem(item: FromItem, oldName: String, newName: String): FromItem = when (item) { + is TableSource -> if (item.name.equals(oldName, ignoreCase = true)) item.copy(name = newName) else item + is SubquerySource -> item.copy(select = rename(item.select, oldName, newName)) + } + + private fun renameInProjection(projection: Projection, oldName: String, newName: String): Projection = when (projection) { + is StarProjection -> projection + is ExpressionProjection -> + projection.copy(items = projection.items.map { it.copy(expression = renameInExpression(it.expression, oldName, newName)) }) + } + + private fun renameInExpression(expression: Expression, oldName: String, newName: String): Expression = when (expression) { + is ScalarSubqueryExpression -> expression.copy(select = renameInSelect(expression.select, oldName, newName)) + is ExistsExpression -> expression.copy(select = renameInSelect(expression.select, oldName, newName)) + is InSubqueryExpression -> expression.copy( + operand = renameInExpression(expression.operand, oldName, newName), + select = renameInSelect(expression.select, oldName, newName) + ) + is BinaryExpression -> expression.copy( + left = renameInExpression(expression.left, oldName, newName), + right = renameInExpression(expression.right, oldName, newName) + ) + is UnaryExpression -> expression.copy(operand = renameInExpression(expression.operand, oldName, newName)) + is CallExpression -> expression.copy(arguments = expression.arguments.map { renameInExpression(it, oldName, newName) }) + is IsNullExpression -> expression.copy(operand = renameInExpression(expression.operand, oldName, newName)) + is InExpression -> expression.copy( + operand = renameInExpression(expression.operand, oldName, newName), + values = expression.values.map { renameInExpression(it, oldName, newName) } + ) + is BetweenExpression -> expression.copy( + operand = renameInExpression(expression.operand, oldName, newName), + low = renameInExpression(expression.low, oldName, newName), + high = renameInExpression(expression.high, oldName, newName) + ) + is LikeExpression -> expression.copy( + operand = renameInExpression(expression.operand, oldName, newName), + pattern = renameInExpression(expression.pattern, oldName, newName) + ) + is CaseExpression -> expression.copy( + operand = expression.operand?.let { renameInExpression(it, oldName, newName) }, + branches = expression.branches.map { + it.copy( + condition = renameInExpression(it.condition, oldName, newName), + result = renameInExpression(it.result, oldName, newName) + ) + }, + elseExpression = expression.elseExpression?.let { renameInExpression(it, oldName, newName) } + ) + is CastExpression -> expression.copy(operand = renameInExpression(expression.operand, oldName, newName)) + is CollateExpression -> expression.copy(operand = renameInExpression(expression.operand, oldName, newName)) + is FunctionExpression -> expression.copy( + argument = expression.argument?.let { renameInExpression(it, oldName, newName) }, + separator = expression.separator?.let { renameInExpression(it, oldName, newName) }, + orderBy = expression.orderBy.map { it.copy(expression = renameInExpression(it.expression, oldName, newName)) }, + filter = expression.filter?.let { renameInExpression(it, oldName, newName) } + ) + is WindowExpression -> expression.copy( + argument = expression.argument?.let { renameInExpression(it, oldName, newName) }, + partitionBy = expression.partitionBy.map { renameInExpression(it, oldName, newName) }, + orderBy = expression.orderBy.map { it.copy(expression = renameInExpression(it.expression, oldName, newName)) }, + offset = expression.offset?.let { renameInExpression(it, oldName, newName) }, + defaultValue = expression.defaultValue?.let { renameInExpression(it, oldName, newName) }, + frame = expression.frame?.let { frame -> + frame.copy( + start = renameInFrameBound(frame.start, oldName, newName), + end = renameInFrameBound(frame.end, oldName, newName) + ) + } + ) + is ColumnExpression, is LiteralExpression, is NextValueExpression, is CurrentValueExpression, + is MergeActionExpression, is ParameterExpression -> expression + } + + private fun renameInFrameBound(bound: FrameBound, oldName: String, newName: String): FrameBound = + bound.copy(offset = bound.offset?.let { renameInExpression(it, oldName, newName) }) +} diff --git a/src/main/kotlin/kestrel/catalog/Sequence.kt b/src/main/kotlin/kestrel/catalog/Sequence.kt new file mode 100644 index 0000000..39f8baa --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/Sequence.kt @@ -0,0 +1,76 @@ +package kestrel.catalog + +import kestrel.KestrelException + +/** + * Mirrors Postgres' own internal sequence representation (`pg_sequences.last_value`/`is_called`) + * rather than "the next value to hand out" directly, because the two ALTER SEQUENCE behaviors this + * engine supports need genuinely different arithmetic: a plain `ALTER SEQUENCE ... INCREMENT BY n` + * (no RESTART) must recompute the NEXT value from the LAST one actually returned using the NEW + * increment (so incrementing by 100 right after NEXTVAL returned 1 gives 101, not "whatever was + * precomputed under the old increment"), while `RESTART [WITH n]` must instead make the very next + * NEXTVAL return `n` directly, bypassing that arithmetic entirely for one call - exactly the same + * shape as a fresh sequence's first-ever NEXTVAL returning `START WITH` untouched. [called] `false` + * captures both of those "return this value as-is, don't advance from it" cases uniformly. + * + * [value]/[called] live entirely in plain `var`s deliberately NOT covered by [Catalog]'s + * snapshot/restore copying of its `sequences` map entries - mirrors Postgres: a ROLLBACK undoes + * `CREATE`/`DROP`/`ALTER SEQUENCE` (the catalog entry itself, since [Catalog.restore] reinstalls + * whichever [Sequence] *instance* a snapshot captured a reference to) but never undoes a NEXTVAL + * advance, even one made by the very transaction being rolled back - the whole point of a sequence + * is to hand out values that never repeat, even across an aborted transaction, so two concurrent + * transactions can both call NEXTVAL without either blocking on the other's eventual + * commit/rollback decision. [Catalog.alterSequence] relies on this same identity trick in reverse: + * it always installs a brand new [Sequence] instance (see [altered]) rather than mutating this one + * in place, so ALTER SEQUENCE (unlike NEXTVAL) rolls back cleanly too. + */ +class Sequence(val definition: SequenceDefinition, initialValue: Long = definition.startValue, initialCalled: Boolean = false) { + private var value: Long = initialValue + private var called: Boolean = initialCalled + + /** What the next call to [next] would return, without consuming it - used by [kestrel.persistence.SqlWriter] to dump a sequence's current position so a reload resumes from there instead of restarting at [SequenceDefinition.startValue]. */ + fun peekNext(): Long = if (called) advance(value) else value + + fun next(): Long = peekNext().also { + value = it + called = true + } + + /** + * The raw `(value, called)` pair this instance currently holds, with no arithmetic and so no way + * to throw - unlike [peekNext], which may run [advance] and hit an exhausted-without-CYCLE bound. + * Used by [kestrel.store.sql.DiskCatalog] to persist a NEXTVAL advance (or an ALTER SEQUENCE's + * resulting position) exactly as it stands, rather than one step further ahead: computing that + * next step eagerly - right after a [next] call that legitimately landed exactly on + * [kestrel.catalog.SequenceDefinition.maxValue]/[SequenceDefinition.minValue] - would spuriously + * raise the "exhausted" exception a whole [next] call early, before anything ever actually asked + * for a value beyond the bound. + */ + fun snapshotPosition(): Pair = value to called + + /** + * Builds the [Sequence] `ALTER SEQUENCE` installs in place of this one: [newDefinition] takes + * over immediately. With [restartAt] given (Postgres' `RESTART [WITH n]`, resolved to + * [SequenceDefinition.startValue] by the caller for a bare `RESTART`), the next NEXTVAL returns + * exactly that value; otherwise this sequence's own last-returned value and "has it ever been + * called" flag both carry over unchanged, so the next NEXTVAL keeps advancing from where this + * one left off - just using [newDefinition]'s (possibly different) increment/bounds. + */ + fun altered(newDefinition: SequenceDefinition, restartAt: Long? = null): Sequence = + if (restartAt != null) Sequence(newDefinition, restartAt, initialCalled = false) + else Sequence(newDefinition, value, called) + + private fun advance(previous: Long): Long { + val ascending = definition.incrementBy > 0 + val candidate = runCatching { Math.addExact(previous, definition.incrementBy) }.getOrNull() + val exhausted = candidate == null || + (ascending && candidate > definition.maxValue) || + (!ascending && candidate < definition.minValue) + if (!exhausted) return candidate!! + if (!definition.cycle) { + val bound = if (ascending) "maximum value (${definition.maxValue})" else "minimum value (${definition.minValue})" + throw KestrelException("sequence '${definition.name}' has reached its $bound and does not CYCLE") + } + return if (ascending) definition.minValue else definition.maxValue + } +} diff --git a/src/main/kotlin/kestrel/catalog/SequenceDefinition.kt b/src/main/kotlin/kestrel/catalog/SequenceDefinition.kt new file mode 100644 index 0000000..68123e5 --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/SequenceDefinition.kt @@ -0,0 +1,26 @@ +package kestrel.catalog + +/** + * A sequence's static configuration - separate from [Sequence]'s mutable counter so that a + * `CREATE SEQUENCE`/`ALTER SEQUENCE`/`DROP SEQUENCE` (all fully transactional, unlike a NEXTVAL + * advance - see [Sequence]'s own doc) can be represented and rolled back like any other catalog + * object, by simply installing or removing an immutable value rather than mutating shared state. + * + * [startValue] is never changed by `ALTER SEQUENCE` (Postgres has no way to do this either) - it is + * only ever the value a bare `RESTART` (no `WITH n`) resets the counter back to, recorded once at + * `CREATE SEQUENCE` time and carried forward unchanged by every later `ALTER SEQUENCE` (see + * [Sequence.altered]). + */ +data class SequenceDefinition( + val name: String, + val incrementBy: Long, + val minValue: Long, + val maxValue: Long, + val startValue: Long, + val cycle: Boolean +) { + init { + require(incrementBy != 0L) { "sequence '$name' INCREMENT BY cannot be zero" } + require(minValue <= maxValue) { "sequence '$name' MINVALUE ($minValue) must not exceed MAXVALUE ($maxValue)" } + } +} diff --git a/src/main/kotlin/kestrel/catalog/SortedIndex.kt b/src/main/kotlin/kestrel/catalog/SortedIndex.kt index da83be9..2933009 100644 --- a/src/main/kotlin/kestrel/catalog/SortedIndex.kt +++ b/src/main/kotlin/kestrel/catalog/SortedIndex.kt @@ -2,27 +2,123 @@ package kestrel.catalog import java.util.NavigableMap import java.util.TreeMap +import kestrel.KestrelException +import kestrel.storage.Collation import kestrel.storage.Value import kestrel.storage.ValueOrdering -data class IndexDefinition(val name: String, val tableName: String, val columnName: String) +data class IndexDefinition( + val name: String, + val tableName: String, + val columnNames: List, + val unique: Boolean = false +) { + val columnDisplayName: String get() = columnNames.joinToString(", ") +} -class SortedIndex(val definition: IndexDefinition) { - private val entries: TreeMap> = TreeMap(ValueOrdering) +/** + * A key used to navigate [SortedIndex.entries]. Every key actually stored in the map is + * [Exact] with exactly `definition.columnNames.size` values. [Low]/[High] only ever appear as + * *boundary* arguments to `tailMap`/`headMap` when a caller supplies fewer values than the + * index has columns (an equality prefix over the leading columns, or a range bound on a + * trailing column while earlier columns are pinned by equality) - they tell [IndexKeyOrdering] + * to treat the omitted trailing columns as -infinity/+infinity, which brackets exactly the + * stored keys that share the given prefix without needing a real sentinel [Value] for every + * column type (there's no universal "biggest TEXT" or "biggest DECIMAL"). + */ +private sealed class IndexKey(val values: List) { + class Exact(values: List) : IndexKey(values) + class Low(values: List) : IndexKey(values) + class High(values: List) : IndexKey(values) +} + +/** + * [collations] carries one [Collation] per indexed column, in the same order as + * [IndexDefinition.columnNames] - a plain-[Collation.Binary] entry for a column with no declared + * `COLLATE` (see [kestrel.catalog.Column.collation]), matching this engine's pre-COLLATE ordering + * exactly. Threading it through this comparator (rather than always comparing bytes/`String` + * order via the plain [ValueOrdering.compare]) is what makes an equality/range lookup through + * this index (see [SortedIndex.equalTo]/[SortedIndex.rangeOf], both driven entirely by this + * comparator) agree with what a full-scan `WHERE` filter through [kestrel.engine.Evaluator] + * would decide for the same column - without it, `WHERE name = 'BOB'` on a `NOCASE`-declared, + * indexed `name` column would silently miss a stored `'bob'` row once + * [kestrel.engine.IndexAccessPlanner] chooses to use the index, a real (not just incomplete) + * correctness bug rather than a documented scope-cut. + */ +private class IndexKeyOrdering(private val collations: List) : Comparator { + override fun compare(left: IndexKey, right: IndexKey): Int { + val common = minOf(left.values.size, right.values.size) + for (i in 0 until common) { + val cmp = ValueOrdering.compare(left.values[i], right.values[i], collations[i]) + if (cmp != 0) return cmp + } + if (left.values.size == right.values.size) return 0 + // Sizes differ only when comparing a real (Exact, full-length) stored key against a + // boundary that carries fewer values than the index has columns; the shorter side's + // kind decides whether the missing tail reads as -infinity (Low) or +infinity (High). + return if (left.values.size < right.values.size) { + when (left) { is IndexKey.Low -> -1; is IndexKey.High -> 1; is IndexKey.Exact -> -1 } + } else { + when (right) { is IndexKey.Low -> 1; is IndexKey.High -> -1; is IndexKey.Exact -> 1 } + } + } +} + +class SortedIndex( + val definition: IndexDefinition, + collations: List = List(definition.columnNames.size) { Collation.Binary } +) { + private val entries: TreeMap> = TreeMap(IndexKeyOrdering(collations)) fun clear() = entries.clear() - fun add(key: Value, rowIndex: Int) { - if (key is Value.NullValue) return - entries.getOrPut(key) { mutableListOf() }.add(rowIndex) + /** + * NULLs are exempt from the uniqueness check and from lookups entirely, mirroring the + * standard SQL UNIQUE constraint (many NULLs never collide with each other or with anything + * else). For a composite key that means ANY component being NULL takes the whole row out of + * the index, not just the NULL column - a partial composite key is not a meaningful sort key. + */ + fun add(key: List, rowIndex: Int) { + require(key.size == definition.columnNames.size) { "index key arity mismatch for ${definition.name}" } + if (key.any { it is Value.NullValue }) return + val wrapped = IndexKey.Exact(key) + if (definition.unique && entries[wrapped]?.isNotEmpty() == true) { + throw KestrelException("UNIQUE constraint failed: ${definition.tableName}(${definition.columnDisplayName})") + } + entries.getOrPut(wrapped) { mutableListOf() }.add(rowIndex) } - fun equalTo(key: Value): List = entries[key]?.toList() ?: emptyList() + /** Exact lookup; [key] must supply a value for every indexed column. */ + fun equalTo(key: List): List = entries[IndexKey.Exact(key)]?.toList() ?: emptyList() - fun rangeOf(lower: Value?, lowerInclusive: Boolean, upper: Value?, upperInclusive: Boolean): List { - var view: NavigableMap> = entries - if (lower != null) view = view.tailMap(lower, lowerInclusive) - if (upper != null) view = view.headMap(upper, upperInclusive) + /** + * Range/prefix scan. [lower]/[upper] may each supply fewer values than the index has + * columns - the missing trailing columns are then left unconstrained (a pure equality + * prefix scan passes the same, full prefix, as both [lower] and [upper] with both bounds + * inclusive). + */ + fun rangeOf(lower: List?, lowerInclusive: Boolean, upper: List?, upperInclusive: Boolean): List { + var view: NavigableMap> = entries + if (lower != null) view = view.tailMap(lowerBoundaryKey(lower, lowerInclusive), lowerInclusive) + if (upper != null) view = view.headMap(upperBoundaryKey(upper, upperInclusive), upperInclusive) return view.values.flatten() } + + /** + * A shorter-than-full lower boundary must still admit stored keys whose extra trailing + * columns tie on the given prefix when the bound is inclusive (so it reads as -infinity, + * [IndexKey.Low]); an exclusive lower bound must reject those same tying keys entirely (so + * it reads as +infinity, [IndexKey.High]) - otherwise e.g. `age > 50` on a composite + * (age, name) index would let `age = 50` rows back in through the unconstrained `name` tail. + */ + private fun lowerBoundaryKey(values: List, inclusive: Boolean): IndexKey { + if (values.size == definition.columnNames.size) return IndexKey.Exact(values) + return if (inclusive) IndexKey.Low(values) else IndexKey.High(values) + } + + /** Mirror image of [lowerBoundaryKey]: an inclusive upper bound reads as +infinity, an exclusive one as -infinity. */ + private fun upperBoundaryKey(values: List, inclusive: Boolean): IndexKey { + if (values.size == definition.columnNames.size) return IndexKey.Exact(values) + return if (inclusive) IndexKey.High(values) else IndexKey.Low(values) + } } diff --git a/src/main/kotlin/kestrel/catalog/TableSchema.kt b/src/main/kotlin/kestrel/catalog/TableSchema.kt index 25c6a81..de4fbc2 100644 --- a/src/main/kotlin/kestrel/catalog/TableSchema.kt +++ b/src/main/kotlin/kestrel/catalog/TableSchema.kt @@ -1,15 +1,106 @@ package kestrel.catalog +import kestrel.sql.ast.Expression +import kestrel.storage.Collation import kestrel.storage.ColumnType data class Column( val name: String, val type: ColumnType, val notNull: Boolean, - val primaryKey: Boolean + val primaryKey: Boolean, + val precision: Int? = null, + val scale: Int? = null, + /** + * A constant-only DEFAULT expression (validated by the executor to contain no column + * reference, subquery, or aggregate), used to fill a column omitted from an INSERT's column + * list and to backfill existing rows when the column is added later via ALTER TABLE. + */ + val default: Expression? = null, + /** + * `GENERATED ALWAYS AS (expression) STORED` - recomputed from this row's OTHER column values + * on every INSERT/UPDATE (the executor rejects an explicit value for this column outright, + * matching standard SQL). Mutually exclusive with [default] and never itself references + * another GENERATED column (validated by the executor at CREATE TABLE time, both engines) - + * see [kestrel.engine.Executor.validateGeneratedDefinitions]. + */ + val generated: Expression? = null, + /** + * `col TYPE COLLATE name` - the collating sequence a bare reference to this column compares + * with when the query itself writes no explicit `COLLATE` (see [kestrel.sql.ast.CollateExpression], + * which always wins over this when present). Threaded through [kestrel.engine.ColumnResolver.declaredCollation] + * by every resolver built directly over a table's own schema (both engines' row/EXCLUDED/MERGE + * resolvers, [kestrel.engine.SelectRunner]'s per-source column lookup) so `WHERE`/`IN`/`BETWEEN`/ + * a searched-`CASE` operand honor it automatically - a resolver with no such schema to consult + * (a set operation's output row, a trigger's NEW/OLD side falling through to its own schema + * notwithstanding) defaults to [Collation.Binary] via [kestrel.engine.ColumnResolver]'s own + * default method, exactly like a column never had one declared. + */ + val collation: Collation = Collation.Binary ) -class TableSchema(val name: String, val columns: List) { +enum class ForeignKeyAction { NoAction, Restrict, Cascade, SetNull } + +val ForeignKeyAction.sqlKeyword: String + get() = when (this) { + ForeignKeyAction.NoAction -> "NO ACTION" + ForeignKeyAction.Restrict -> "RESTRICT" + ForeignKeyAction.Cascade -> "CASCADE" + ForeignKeyAction.SetNull -> "SET NULL" + } + +/** + * `referencedColumns` is required to carry an enforced uniqueness guarantee in the referenced + * table as a group - either it is exactly the table's PRIMARY KEY (single-column or composite), + * or it is named (as an exact set) by one of that table's [UniqueConstraint]s. Those are the only + * two ways this engine guarantees a set of columns has no duplicate value combination to safely + * match a foreign key against. `columns`/`referencedColumns` always have the same size (validated + * at CREATE TABLE time) and are positionally paired: `columns[i]` corresponds to + * `referencedColumns[i]`. + */ +data class ForeignKeyConstraint( + val columns: List, + val referencedTable: String, + val referencedColumns: List, + val onDelete: ForeignKeyAction = ForeignKeyAction.NoAction, + val onUpdate: ForeignKeyAction = ForeignKeyAction.NoAction, + /** `CONSTRAINT name FOREIGN KEY (...)` - see [CheckConstraint.name]/[UniqueConstraint.name]'s own doc. */ + val name: String? = null +) + +data class CheckConstraint(val name: String?, val expression: Expression) + +/** + * A UNIQUE constraint independent of the table's PRIMARY KEY. `columns` holds one entry for an + * inline `col TYPE UNIQUE` column definition, or several for a table-level `UNIQUE (a, b, ...)`. + * Per standard SQL, a row where any of these columns is NULL is exempt from the check (NULLs + * are never considered duplicates of each other). + */ +data class UniqueConstraint(val name: String?, val columns: List) + +class TableSchema( + val name: String, + val columns: List, + val foreignKeys: List = emptyList(), + val checks: List = emptyList(), + val uniqueConstraints: List = emptyList(), + /** + * Table-level `PRIMARY KEY (a, b, ...)`. Empty when the table instead declares its PRIMARY KEY + * inline on a single column (`col TYPE PRIMARY KEY`, tracked by [Column.primaryKey]) - see + * [primaryKeyColumns], which unifies both spellings into one canonical list. + */ + val compositePrimaryKey: List = emptyList(), + /** + * `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` - a table exactly like any other at runtime + * (same constraints, indexes, triggers all work), except [kestrel.persistence.SqlWriter] never + * writes it (or its rows, or any index/trigger defined on it) to a checkpoint or the redo log, + * so it does not survive a process restart, matching Postgres. Every place in [Catalog] that + * rebuilds a [TableSchema] from an existing one's fields (ADD/DROP/RENAME COLUMN, RENAME TABLE, + * a referencing table's FOREIGN KEY cleanup) must carry this flag forward explicitly - there is + * no `copy()` to do it automatically, since this stays a plain class, not a data class. + */ + val temporary: Boolean = false +) { private val indexByName: Map = columns.withIndex().associate { (index, column) -> column.name.lowercase() to index } @@ -19,5 +110,36 @@ class TableSchema(val name: String, val columns: List) { fun column(columnName: String): Column? = columnIndex(columnName)?.let { columns[it] } - val primaryKeyIndex: Int? get() = columns.indexOfFirst { it.primaryKey }.takeIf { it >= 0 } + /** + * The table's PRIMARY KEY column names in declared order, regardless of whether it came from + * an inline `col TYPE PRIMARY KEY` (exactly one [Column] with `primaryKey = true`) or a + * table-level `PRIMARY KEY (a, b, ...)` clause ([compositePrimaryKey]). Empty when the table + * has no PRIMARY KEY at all. + */ + val primaryKeyColumns: List get() = + compositePrimaryKey.ifEmpty { columns.filter { it.primaryKey }.map { it.name } } + + val primaryKeyIndices: List get() = primaryKeyColumns.map { columnIndex(it)!! } + + /** Convenience for callers that only ever deal with single-column PRIMARY KEYs. */ + val primaryKeyIndex: Int? get() = primaryKeyIndices.singleOrNull() + + /** + * Every column group in this table with a guaranteed-unique combination of values: the + * PRIMARY KEY (if any, single-column or composite) plus each UNIQUE constraint. A valid + * FOREIGN KEY target must match one of these groups exactly (as a set of names). + */ + fun enforcedUniqueKeyGroups(): List> { + val groups = mutableListOf>() + if (primaryKeyColumns.isNotEmpty()) groups.add(primaryKeyColumns) + for (unique in uniqueConstraints) groups.add(unique.columns) + return groups + } + + /** True when [columnNames], taken together as a set, exactly matches one enforced-unique key group. */ + fun hasEnforcedUniqueness(columnNames: List): Boolean { + val target = columnNames.map { it.lowercase() }.toSet() + if (target.size != columnNames.size) return false + return enforcedUniqueKeyGroups().any { group -> group.map { it.lowercase() }.toSet() == target } + } } diff --git a/src/main/kotlin/kestrel/catalog/TriggerDefinition.kt b/src/main/kotlin/kestrel/catalog/TriggerDefinition.kt new file mode 100644 index 0000000..089b705 --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/TriggerDefinition.kt @@ -0,0 +1,25 @@ +package kestrel.catalog + +import kestrel.sql.ast.Expression +import kestrel.sql.ast.Statement +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming + +/** + * `body` only ever holds INSERT/UPDATE/DELETE statements (enforced by the parser) - see + * [kestrel.sql.ast.CreateTriggerStatement] for why. `tableName` is kept in sync with + * [Catalog.renameTable] the same way [IndexDefinition.tableName] is; it is NOT rewritten when a + * column referenced by [whenClause] or [body] is renamed (unlike CHECK constraints, which use + * [CheckExpressionColumns.rename]) - a documented scope-cut, since [body] statements can each + * target a different table and no expression-rewrite utility for arbitrary nested statements + * exists yet. A rename of a column such a reference depends on simply surfaces as "no such column" + * the next time the trigger fires. + */ +data class TriggerDefinition( + val name: String, + val timing: TriggerTiming, + val event: TriggerEvent, + val tableName: String, + val whenClause: Expression?, + val body: List +) diff --git a/src/main/kotlin/kestrel/catalog/ViewDefinition.kt b/src/main/kotlin/kestrel/catalog/ViewDefinition.kt new file mode 100644 index 0000000..be7e0e5 --- /dev/null +++ b/src/main/kotlin/kestrel/catalog/ViewDefinition.kt @@ -0,0 +1,5 @@ +package kestrel.catalog + +import kestrel.sql.ast.QueryStatement + +data class ViewDefinition(val name: String, val query: QueryStatement) diff --git a/src/main/kotlin/kestrel/cli/Main.kt b/src/main/kotlin/kestrel/cli/Main.kt index 777d038..4763ed5 100644 --- a/src/main/kotlin/kestrel/cli/Main.kt +++ b/src/main/kotlin/kestrel/cli/Main.kt @@ -2,9 +2,11 @@ package kestrel.cli import java.io.File import kestrel.persistence.Persistence +import kestrel.store.sql.DiskCatalog fun main(arguments: Array) { var dataDir: File? = null + var diskDir: File? = null var scriptPath: String? = null var index = 0 while (index < arguments.size) { @@ -16,13 +18,27 @@ fun main(arguments: Array) { argument.startsWith("--data=") -> { dataDir = File(argument.substringAfter('=')); index += 1 } + argument == "--disk" -> { + diskDir = File(arguments[index + 1]); index += 2 + } + argument.startsWith("--disk=") -> { + diskDir = File(argument.substringAfter('=')); index += 1 + } else -> { scriptPath = argument; index += 1 } } } + if (dataDir != null && diskDir != null) { + System.err.println("Error: --data (in-memory engine, text WAL) and --disk (disk-backed engine) are mutually exclusive") + return + } val persistence = dataDir?.let { Persistence(it) } - val repl = Repl(persistence = persistence) + val diskCatalog = diskDir?.let { + it.mkdirs() + DiskCatalog.open(File(it, "kestrel.data"), File(it, "kestrel.wal")) + } + val repl = Repl(persistence = persistence, diskCatalog = diskCatalog) val reader = if (scriptPath != null) File(scriptPath).bufferedReader() else System.`in`.bufferedReader() reader.use { repl.run(it) } } diff --git a/src/main/kotlin/kestrel/cli/Repl.kt b/src/main/kotlin/kestrel/cli/Repl.kt index 8120ca6..2368f4d 100644 --- a/src/main/kotlin/kestrel/cli/Repl.kt +++ b/src/main/kotlin/kestrel/cli/Repl.kt @@ -4,42 +4,89 @@ import java.io.BufferedReader import kestrel.KestrelException import kestrel.LexException import kestrel.catalog.Catalog +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.sqlKeyword +import kestrel.engine.AlterSequenceResult +import kestrel.engine.AlterTableResult import kestrel.engine.BeginResult import kestrel.engine.CommitResult import kestrel.engine.CreateIndexResult +import kestrel.engine.CreateSequenceResult import kestrel.engine.CreateTableResult +import kestrel.engine.CreateTriggerResult +import kestrel.engine.CreateViewResult +import kestrel.engine.DeallocateResult import kestrel.engine.DeleteResult import kestrel.engine.DropIndexResult +import kestrel.engine.DropSequenceResult import kestrel.engine.DropTableResult +import kestrel.engine.DropTriggerResult +import kestrel.engine.DropViewResult import kestrel.engine.ExplainResult import kestrel.engine.Executor import kestrel.engine.InsertResult +import kestrel.engine.MergeResult +import kestrel.engine.PrepareResult +import kestrel.engine.ReleaseSavepointResult import kestrel.engine.RollbackResult +import kestrel.engine.RollbackToSavepointResult +import kestrel.engine.SavepointResult import kestrel.engine.SelectResult import kestrel.engine.StatementResult +import kestrel.engine.TruncateTableResult import kestrel.engine.UpdateResult import kestrel.persistence.Persistence import kestrel.persistence.SqlWriter +import kestrel.sql.ast.AlterTableStatement import kestrel.sql.ast.BeginStatement import kestrel.sql.ast.CommitStatement +import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropIndexStatement +import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.ExecuteStatement +import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.ReleaseSavepointStatement import kestrel.sql.ast.RollbackStatement +import kestrel.sql.ast.RollbackToSavepointStatement +import kestrel.sql.ast.SavepointStatement import kestrel.sql.ast.Statement +import kestrel.sql.ast.TruncateTableStatement +import kestrel.sql.ast.UpdateStatement import kestrel.sql.lexer.Lexer +import kestrel.storage.displayWith +import kestrel.sql.lexer.Token import kestrel.sql.lexer.TokenType import kestrel.sql.parser.Parser +import kestrel.store.sql.DiskCatalog +import kestrel.store.sql.DiskExecutor +/** + * `diskCatalog` swaps the storage backend for every statement from the in-memory [Catalog]/ + * [Executor] to the real disk-backed [DiskCatalog]/[DiskExecutor] pair (pager/WAL/B+Tree) - the two + * engines are mutually exclusive, never blended in one session, and [persistence] (the in-memory + * engine's own text-based WAL replay) makes no sense once statements are already durable on disk. + */ class Repl( private val output: Appendable = System.out, private val errorOutput: Appendable = System.err, - private val persistence: Persistence? = null + private val persistence: Persistence? = null, + private val diskCatalog: DiskCatalog? = null ) { private val catalog = Catalog() private val executor = Executor(catalog) + private val diskExecutor = diskCatalog?.let { DiskExecutor(it) } private val pending = StringBuilder() private var pendingCommit: MutableList? = null + private val savepointMarks = mutableListOf>() private var replaying = false init { + require(persistence == null || diskCatalog == null) { + "Repl cannot combine text-based Persistence with a disk-backed DiskCatalog - pick one storage backend" + } if (persistence != null) { val restored = persistence.load() if (restored.isNotBlank()) { @@ -63,6 +110,11 @@ class Repl( } private fun finish() { + if (diskExecutor != null) { + if (diskExecutor.inTransaction) diskExecutor.execute(RollbackStatement) + diskCatalog?.close() + return + } if (executor.inTransaction) executor.execute(RollbackStatement) pendingCommit = null persistence?.checkpoint(SqlWriter.dump(catalog)) @@ -89,12 +141,40 @@ class Repl( return } val meaningful = tokens.dropLast(1) - if (meaningful.isNotEmpty() && meaningful.last().type == TokenType.Semicolon) { + if (meaningful.isNotEmpty() && meaningful.last().type == TokenType.Semicolon && !endsInsideOpenBlock(meaningful)) { executeBuffer(pending.toString()) pending.setLength(0) } } + /** + * A trigger body's `BEGIN ... END` wraps statements that each end in their own `;`, so the + * naive "buffer ends in a semicolon" check above would fire mid-body, on the first inner + * statement, well before the trigger definition is actually complete - unlike every other + * statement this REPL buffers line by line, a trigger body is the only one with `;` nested + * inside it. Distinguishing a trigger-body `BEGIN` from a bare `BEGIN`/`BEGIN TRANSACTION` + * (which never takes a matching `END`) only needs one fact: a fresh top-level statement's + * first token is always either the very start of the buffer or right after a `;` - a + * trigger-body `BEGIN` is never in that position, since `FOR EACH ROW` (or a `WHEN` clause) + * always precedes it. `CASE ... END` nests on the same stack since it needs the exact same + * "don't fire on an inner `;`" protection for a `CASE` that spans multiple typed lines. + */ + private fun endsInsideOpenBlock(tokens: List): Boolean { + var depth = 0 + for (index in tokens.indices) { + when (tokens[index].type) { + TokenType.Case -> depth++ + TokenType.Begin -> { + val previous = tokens.getOrNull(index - 1)?.type + if (previous != null && previous != TokenType.Semicolon) depth++ + } + TokenType.End -> if (depth > 0) depth-- + else -> Unit + } + } + return depth > 0 + } + private fun flushPending() { if (pending.isNotBlank()) { executeBuffer(pending.toString()) @@ -111,46 +191,162 @@ class Repl( } for (statement in statements) { try { - val result = executor.execute(statement) + // EXECUTE is resolved to its concrete, parameter-free statement HERE, once, rather + // than inside Executor.execute's own dispatch - so recordForPersistence below sees + // (and durably logs) the same substituted INSERT/UPDATE/DELETE/SELECT that actually + // ran, not the literal `EXECUTE name(...)` text. That literal text is meaningless to + // replay anyway: a freshly reopened session has no PREPARE-d statement named `name` + // (see PrepareStatement's own doc - prepared statements are session-scoped, never + // persisted). Resolving exactly once also matters for correctness, not just tidiness: + // an EXECUTE argument could itself be a volatile expression, so resolving twice + // (once to run it, once again to decide what to persist) would evaluate it twice. + val effective = if (diskExecutor == null && statement is ExecuteStatement) { + executor.resolveExecute(statement) + } else { + statement + } + // Computed BEFORE execute() runs, not after - DROP TABLE/RENAME TABLE/DROP INDEX + // mutate (or remove) the very catalog entry this asks about, so checking afterward + // could never again tell a table that WAS temporary apart from one that never + // existed at all. CREATE TABLE is the one case checked from the statement itself + // rather than the catalog, since the table it names does not exist yet either way. + val temporaryTargets = if (diskExecutor == null) temporaryTableTargets(effective) else emptySet() + val result = if (diskExecutor != null) diskExecutor.execute(statement) else executor.execute(effective) printResult(result) - if (!replaying) recordForPersistence(statement) + if (diskExecutor == null && !replaying) recordForPersistence(effective, temporaryTargets) } catch (exception: KestrelException) { printError(exception.message) } } } - private fun recordForPersistence(statement: Statement) { + /** + * Every table name [statement] itself creates/drops/alters/indexes/reads-and-writes-as-its-own- + * target that is currently a TEMP table (see [kestrel.catalog.TableSchema.temporary]) - + * deliberately NOT a general "every table this statement's body could possibly touch" scan + * (an INSERT's SELECT source, a MERGE's USING source, a view/trigger's query body): those are + * read-only references that either replay fine (the source table still exists after reload) or + * fail loudly with "no such table" if it does not, which is an acceptable, honest failure mode - + * see [kestrel.persistence.SqlWriter.dump]'s own note on the same tradeoff for CREATE VIEW. + */ + private fun temporaryTableTargets(statement: Statement): Set = when (statement) { + is CreateTableStatement -> if (statement.temporary) setOf(statement.tableName) else emptySet() + is DropTableStatement -> temporaryOf(statement.tableName) + is TruncateTableStatement -> statement.tableNames.filter { isTemporaryTable(it) }.toSet() + is AlterTableStatement -> temporaryOf(statement.tableName) + is CreateIndexStatement -> temporaryOf(statement.tableName) + is DropIndexStatement -> catalog.indexDefinitions() + .firstOrNull { it.name.equals(statement.indexName, ignoreCase = true) } + ?.let { temporaryOf(it.tableName) } ?: emptySet() + is InsertStatement -> temporaryOf(statement.tableName) + is UpdateStatement -> temporaryOf(statement.tableName) + is DeleteStatement -> temporaryOf(statement.tableName) + is MergeStatement -> temporaryOf(statement.targetTable) + else -> emptySet() + } + + private fun isTemporaryTable(tableName: String): Boolean = catalog.tableOrNull(tableName)?.schema?.temporary == true + + private fun temporaryOf(tableName: String): Set = if (isTemporaryTable(tableName)) setOf(tableName) else emptySet() + + private fun recordForPersistence(statement: Statement, temporaryTargets: Set) { when (statement) { - is BeginStatement -> pendingCommit = mutableListOf() + is BeginStatement -> { + pendingCommit = mutableListOf() + savepointMarks.clear() + } is CommitStatement -> { pendingCommit?.let { persistence?.appendWal(it) } pendingCommit = null + savepointMarks.clear() } - is RollbackStatement -> pendingCommit = null - else -> if (SqlWriter.canPersist(statement)) { - val sql = SqlWriter.write(statement) - val buffer = pendingCommit - if (buffer != null) buffer.add(sql) else persistence?.appendWal(listOf(sql)) + is RollbackStatement -> { + pendingCommit = null + savepointMarks.clear() + } + is SavepointStatement -> { + pendingCommit?.let { savepointMarks.add(statement.name to it.size) } + } + is ReleaseSavepointStatement -> { + val index = savepointMarks.indexOfLast { it.first.equals(statement.name, ignoreCase = true) } + while (savepointMarks.size > index) savepointMarks.removeAt(savepointMarks.size - 1) + } + is RollbackToSavepointStatement -> { + val index = savepointMarks.indexOfLast { it.first.equals(statement.name, ignoreCase = true) } + val mark = savepointMarks[index].second + pendingCommit?.let { buffer -> while (buffer.size > mark) buffer.removeAt(buffer.size - 1) } + while (savepointMarks.size > index + 1) savepointMarks.removeAt(savepointMarks.size - 1) + } + // A TRUNCATE naming both TEMP and regular tables in one statement is the only + // persistable statement that can target more than one table (every other case above + // fully skips or fully persists) - the regular ones still need their truncation durably + // logged even though the TEMP ones must not appear in the text at all, since after + // reload a TEMP table's own CREATE never replayed and "TRUNCATE TABLE that_name" would + // otherwise fail with "no such table" for a name that was only ever supposed to vanish + // quietly. + is TruncateTableStatement -> { + val persistedNames = statement.tableNames.filterNot { it in temporaryTargets } + if (persistedNames.isNotEmpty()) { + appendToWal(SqlWriter.write(statement.copy(tableNames = persistedNames))) + } + } + else -> if (temporaryTargets.isEmpty() && SqlWriter.canPersist(statement)) { + appendToWal(SqlWriter.write(statement)) } } } + private fun appendToWal(sql: String) { + val buffer = pendingCommit + if (buffer != null) buffer.add(sql) else persistence?.appendWal(listOf(sql)) + } + private fun printResult(result: StatementResult) { if (replaying) return when (result) { is SelectResult -> output.appendLine(TableRenderer.render(result)) - is InsertResult -> output.appendLine("${rowsWord(result.count)} inserted") - is UpdateResult -> output.appendLine("${rowsWord(result.count)} updated") - is DeleteResult -> output.appendLine("${rowsWord(result.count)} deleted") + is InsertResult -> { + output.appendLine("${rowsWord(result.count)} inserted") + result.returning?.let { output.appendLine(TableRenderer.render(it)) } + } + is UpdateResult -> { + output.appendLine("${rowsWord(result.count)} updated") + result.returning?.let { output.appendLine(TableRenderer.render(it)) } + } + is DeleteResult -> { + output.appendLine("${rowsWord(result.count)} deleted") + result.returning?.let { output.appendLine(TableRenderer.render(it)) } + } is CreateTableResult -> output.appendLine("Table ${result.tableName} created") is DropTableResult -> output.appendLine("Table ${result.tableName} dropped") + is TruncateTableResult -> output.appendLine("Table(s) ${result.tableNames.joinToString(", ")} truncated") + is MergeResult -> { + output.appendLine( + "${rowsWord(result.insertedCount)} inserted, ${rowsWord(result.updatedCount)} updated, " + + "${rowsWord(result.deletedCount)} deleted" + ) + result.returning?.let { output.appendLine(TableRenderer.render(it)) } + } + is AlterTableResult -> output.appendLine("Table ${result.tableName} altered") is CreateIndexResult -> output.appendLine("Index ${result.indexName} created") is DropIndexResult -> output.appendLine("Index ${result.indexName} dropped") + is CreateViewResult -> output.appendLine("View ${result.viewName} created") + is DropViewResult -> output.appendLine("View ${result.viewName} dropped") + is CreateTriggerResult -> output.appendLine("Trigger ${result.triggerName} created") + is DropTriggerResult -> output.appendLine("Trigger ${result.triggerName} dropped") + is CreateSequenceResult -> output.appendLine("Sequence ${result.sequenceName} created") + is DropSequenceResult -> output.appendLine("Sequence ${result.sequenceName} dropped") + is AlterSequenceResult -> output.appendLine("Sequence ${result.sequenceName} altered") is ExplainResult -> result.plan.forEach { output.appendLine(it) } BeginResult -> output.appendLine("Transaction started") CommitResult -> output.appendLine("Transaction committed") RollbackResult -> output.appendLine("Transaction rolled back") + is SavepointResult -> output.appendLine("Savepoint ${result.name} created") + is ReleaseSavepointResult -> output.appendLine("Savepoint ${result.name} released") + is RollbackToSavepointResult -> output.appendLine("Rolled back to savepoint ${result.name}") + is PrepareResult -> output.appendLine("Prepared statement ${result.name} created") + is DeallocateResult -> + output.appendLine(if (result.all) "All prepared statements deallocated" else "Prepared statement ${result.name} deallocated") } } @@ -158,14 +354,15 @@ class Repl( val parts = command.split(Regex("\\s+"), limit = 2) when (parts[0]) { ".exit", ".quit" -> return true - ".tables" -> { - val names = catalog.tableNames - if (names.isEmpty()) output.appendLine("(no tables)") else names.forEach { output.appendLine(it) } - } - ".indexes" -> { - val names = catalog.indexNames - if (names.isEmpty()) output.appendLine("(no indexes)") else names.forEach { output.appendLine(it) } - } + ".tables" -> printNames(tableNames(), "(no tables)") + ".indexes" -> printNames(indexNames(), "(no indexes)") + ".views" -> printNames(viewNames(), "(no views)") + ".triggers" -> printNames(triggerNames(), "(no triggers)") + ".sequences" -> printNames(sequenceNames(), "(no sequences)") + ".prepared" -> printNames( + diskExecutor?.preparedStatementNames ?: executor.preparedStatementNames, + "(no prepared statements)" + ) ".schema" -> { if (parts.size < 2 || parts[1].isBlank()) { printError("usage: .schema ") @@ -178,25 +375,87 @@ class Repl( return false } + private fun printNames(names: List, emptyMessage: String) { + if (names.isEmpty()) output.appendLine(emptyMessage) else names.forEach { output.appendLine(it) } + } + + private fun tableNames(): List = diskCatalog?.tableNames ?: catalog.tableNames + private fun viewNames(): List = diskCatalog?.viewNames ?: catalog.viewNames + private fun triggerNames(): List = diskCatalog?.triggerNames ?: catalog.triggerNames + private fun sequenceNames(): List = diskCatalog?.sequenceNames ?: catalog.sequenceNames + + private fun indexNames(): List { + val disk = diskCatalog ?: return catalog.indexNames + return disk.tableNames.flatMap { disk.table(it).secondaryIndexes.map { index -> index.name } } + } + + /** Every `CREATE [UNIQUE] INDEX ...;` line for [tableName], one engine or the other - never both. */ + private fun indexLines(tableName: String): List { + val disk = diskCatalog + if (disk != null) { + val table = disk.tableOrNull(tableName) ?: return emptyList() + return table.secondaryIndexes.map { index -> + val uniqueKeyword = if (index.unique) "UNIQUE " else "" + "CREATE ${uniqueKeyword}INDEX ${index.name} ON $tableName (${index.columnNames.joinToString(", ")});" + } + } + return catalog.indexDefinitions() + .filter { it.tableName.equals(tableName, ignoreCase = true) } + .map { index -> + val uniqueKeyword = if (index.unique) "UNIQUE " else "" + "CREATE ${uniqueKeyword}INDEX ${index.name} ON $tableName (${index.columnDisplayName});" + } + } + + /** Every `CREATE TRIGGER ...` block for [tableName], one engine or the other - never both. */ + private fun triggerLines(tableName: String): List { + val definitions = diskCatalog?.triggerDefinitions() ?: catalog.triggerDefinitions() + return definitions + .filter { it.tableName.equals(tableName, ignoreCase = true) } + .map { SqlWriter.triggerText(it) } + } + private fun printSchema(tableName: String) { - val table = catalog.tableOrNull(tableName) - if (table == null) { + val schema = diskCatalog?.tableOrNull(tableName)?.schema + ?: diskCatalog?.temporaryTableOrNull(tableName)?.schema + ?: catalog.tableOrNull(tableName)?.schema + if (schema == null) { printError("no such table: $tableName") return } - val schema = table.schema val builder = StringBuilder() builder.appendLine("CREATE TABLE ${schema.name} (") - schema.columns.forEachIndexed { index, column -> + val lines = mutableListOf() + schema.columns.forEach { column -> val constraints = buildString { if (column.primaryKey) append(" PRIMARY KEY") if (column.notNull && !column.primaryKey) append(" NOT NULL") + column.default?.let { append(" DEFAULT ${SqlWriter.expressionText(it)}") } } - val comma = if (index < schema.columns.size - 1) "," else "" - builder.appendLine(" ${column.name} ${column.type.displayName}$constraints$comma") + lines.add(" ${column.name} ${column.type.displayWith(column.precision, column.scale)}$constraints") } - builder.append(");") + if (schema.compositePrimaryKey.isNotEmpty()) { + lines.add(" PRIMARY KEY (${schema.compositePrimaryKey.joinToString(", ")})") + } + schema.foreignKeys.forEach { fk -> + val onDelete = if (fk.onDelete != ForeignKeyAction.NoAction) " ON DELETE ${fk.onDelete.sqlKeyword}" else "" + val onUpdate = if (fk.onUpdate != ForeignKeyAction.NoAction) " ON UPDATE ${fk.onUpdate.sqlKeyword}" else "" + lines.add( + " FOREIGN KEY (${fk.columns.joinToString(", ")}) REFERENCES ${fk.referencedTable} " + + "(${fk.referencedColumns.joinToString(", ")})$onDelete$onUpdate" + ) + } + schema.checks.forEach { check -> + lines.add(" CHECK (${SqlWriter.expressionText(check.expression)})") + } + schema.uniqueConstraints.forEach { unique -> + lines.add(" UNIQUE (${unique.columns.joinToString(", ")})") + } + builder.append(lines.joinToString(",\n")) + builder.append("\n);") output.appendLine(builder.toString()) + indexLines(schema.name).forEach { output.appendLine(it) } + triggerLines(schema.name).forEach { output.appendLine(it) } } private fun printError(message: String?) { diff --git a/src/main/kotlin/kestrel/cli/TableRenderer.kt b/src/main/kotlin/kestrel/cli/TableRenderer.kt index 1209ce9..3265494 100644 --- a/src/main/kotlin/kestrel/cli/TableRenderer.kt +++ b/src/main/kotlin/kestrel/cli/TableRenderer.kt @@ -31,6 +31,8 @@ object TableRenderer { is Value.RealValue -> value.value.toString() is Value.TextValue -> value.value is Value.BooleanValue -> value.value.toString() + is Value.DecimalValue -> value.value.toPlainString() + is Value.DateValue -> value.value.toString() is Value.NullValue -> "NULL" } diff --git a/src/main/kotlin/kestrel/engine/AggregateFunctions.kt b/src/main/kotlin/kestrel/engine/AggregateFunctions.kt new file mode 100644 index 0000000..6732cf2 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/AggregateFunctions.kt @@ -0,0 +1,147 @@ +package kestrel.engine + +import java.math.BigDecimal +import java.math.MathContext +import java.math.RoundingMode +import kestrel.KestrelException +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.OrderKey +import kestrel.storage.Collation +import kestrel.storage.Value +import kestrel.storage.ValueOrdering +import kestrel.storage.typeName + +/** + * COUNT/SUM/AVG/MIN/MAX over a fixed list of rows - the one piece of arithmetic shared by grouped + * aggregation ([GroupAggregation], which collapses a whole group into one row) and windowed + * aggregation (a [kestrel.sql.ast.WindowExpression] with [kestrel.sql.ast.WindowFunctionKind.Aggregate], + * which instead computes the same thing once per partition/frame and keeps every row). Both callers + * hand in their own `evaluateRow` so the argument expression is resolved the same way it would be + * anywhere else in that context (correlated outer columns, subqueries, and so on). + */ +internal object AggregateFunctions { + fun compute( + name: String, + argument: Expression?, + star: Boolean, + rows: List>, + evaluateRow: (Expression, List) -> Value, + separator: Expression? = null, + orderBy: List = emptyList(), + distinct: Boolean = false, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } + ): Value { + val upperName = name.uppercase() + if (upperName == "COUNT") { + if (star) return Value.IntegerValue(rows.size.toLong()) + val arg = argument ?: return Value.IntegerValue(rows.size.toLong()) + val nonNull = rows.map { evaluateRow(arg, it) }.filter { it !is Value.NullValue } + val count = if (distinct) nonNull.distinctByCollation(resolveCollation(arg, declaredCollationOf)).size else nonNull.size + return Value.IntegerValue(count.toLong()) + } + if (upperName == "STRING_AGG") return stringAgg(argument, separator, orderBy, rows, evaluateRow, declaredCollationOf) + val arg = argument ?: throw KestrelException("aggregate function $name requires an argument") + val nonNullValues = rows.map { evaluateRow(arg, it) }.filter { it !is Value.NullValue } + val values = if (distinct) nonNullValues.distinctByCollation(resolveCollation(arg, declaredCollationOf)) else nonNullValues + return when (upperName) { + "SUM" -> sum(values) + "AVG" -> average(values) + "MIN" -> values.minWithOrNull(ValueOrdering) ?: Value.NullValue + "MAX" -> values.maxWithOrNull(ValueOrdering) ?: Value.NullValue + else -> throw KestrelException("unknown function: $name") + } + } + + /** + * NULL [argument] values are skipped (standard STRING_AGG behavior, matching how SUM/AVG/MIN/MAX + * above already drop NULLs before combining). With no [orderBy], values keep [rows]' own order - + * for a GROUP BY aggregate that is source/scan order, which is exactly what every other order- + * sensitive-but-unordered SQL construct in this engine already falls back to. [separator] is + * evaluated once, against whichever surviving row ends up first after any [orderBy] sort - it is + * expected to be a per-call constant (a literal in every realistic query), so which row supplies it + * only matters if someone writes a genuinely row-varying separator expression, an edge case no SQL + * engine defines cleanly either. + */ + private fun stringAgg( + argument: Expression?, + separator: Expression?, + orderBy: List, + rows: List>, + evaluateRow: (Expression, List) -> Value, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } + ): Value { + val valueArgument = argument ?: throw KestrelException("STRING_AGG requires a value argument") + val separatorArgument = separator ?: throw KestrelException("STRING_AGG requires a separator argument") + var entries = rows.mapNotNull { row -> + val value = evaluateRow(valueArgument, row) + if (value is Value.NullValue) null else row to text(value, "STRING_AGG") + } + if (entries.isEmpty()) return Value.NullValue + if (orderBy.isNotEmpty()) { + val comparator = Comparator, String>> { first, second -> + var result = 0 + for (key in orderBy) { + val comparison = compareOrderKeyValues( + evaluateRow(key.expression, first.first), + evaluateRow(key.expression, second.first), + key, + declaredCollationOf + ) + if (comparison != 0) { + result = comparison + break + } + } + result + } + entries = entries.sortedWith(comparator) + } + val separatorText = text(evaluateRow(separatorArgument, entries.first().first), "STRING_AGG") + return Value.TextValue(entries.joinToString(separatorText) { it.second }) + } + + private fun text(value: Value, function: String): String = when (value) { + is Value.TextValue -> value.value + else -> throw KestrelException("function $function requires a TEXT argument but got ${value.typeName}") + } + + private fun sum(values: List): Value { + if (values.isEmpty()) return Value.NullValue + if (values.all { it is Value.IntegerValue }) { + return Value.IntegerValue(values.sumOf { (it as Value.IntegerValue).value }) + } + if (isExactDecimalMix(values)) { + return Value.DecimalValue(values.fold(BigDecimal.ZERO) { acc, value -> acc + asBigDecimal(value) }) + } + return Value.RealValue(values.sumOf { numeric(it) }) + } + + private fun average(values: List): Value { + if (values.isEmpty()) return Value.NullValue + if (isExactDecimalMix(values)) { + val total = values.fold(BigDecimal.ZERO) { acc, value -> acc + asBigDecimal(value) } + return Value.DecimalValue(total.divide(BigDecimal(values.size), MathContext(DECIMAL_AVERAGE_PRECISION, RoundingMode.HALF_UP))) + } + return Value.RealValue(values.sumOf { numeric(it) } / values.size) + } + + /** SUM/AVG stay exact (BigDecimal) rather than falling back to Double when a DECIMAL value is involved. */ + private fun isExactDecimalMix(values: List): Boolean = + values.any { it is Value.DecimalValue } && values.all { it is Value.DecimalValue || it is Value.IntegerValue } + + private fun asBigDecimal(value: Value): BigDecimal = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + else -> throw KestrelException("aggregate requires numeric operands but got ${value.typeName}") + } + + private fun numeric(value: Value): Double = when (value) { + is Value.IntegerValue -> value.value.toDouble() + is Value.RealValue -> value.value + is Value.DecimalValue -> value.value.toDouble() + else -> throw KestrelException("aggregate requires numeric operands but got ${value.typeName}") + } + + private const val DECIMAL_AVERAGE_PRECISION = 20 +} diff --git a/src/main/kotlin/kestrel/engine/CollationEquality.kt b/src/main/kotlin/kestrel/engine/CollationEquality.kt new file mode 100644 index 0000000..293df86 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/CollationEquality.kt @@ -0,0 +1,94 @@ +package kestrel.engine + +import kestrel.catalog.TableSchema +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.Expression +import kestrel.storage.Collation +import kestrel.storage.Value + +/** + * Resolves which [Collation] governs equality/ordering for [expression]'s own value, with the same + * two-tier precedence [compareOrderKeyValues] already uses for `ORDER BY`: an explicit `COLLATE` on + * [expression] itself wins; failing that, a bare [ColumnExpression]'s own declared `COLLATE` (see + * [kestrel.catalog.Column.collation]), reported by [declaredCollationOf]; [Collation.Binary] + * otherwise. Shared by [compareOrderKeyValues] and every equality-based dedup below so both sides of + * the COLLATE story (ordering and equality) agree on which collation a given expression uses. + */ +fun resolveCollation(expression: Expression, declaredCollationOf: (ColumnExpression) -> Collation?): Collation = + (expression as? CollateExpression)?.collation + ?: (expression as? ColumnExpression)?.let(declaredCollationOf) + ?: Collation.Binary + +/** + * Folds [values] (one grouping/`DISTINCT` key, whose entries line up positionally with + * [expressions]) into a key usable as a [Map]/[Set] bucket or a `distinctBy` selector: a + * [Value.TextValue] under a collation resolved (via [resolveCollation]) to anything but + * [Collation.Binary] is rewritten through [Collation.canonicalize] first, so e.g. `'Bob'` and + * `'bob'` land in the same bucket under a `NOCASE`-declared column, while every other value keeps + * its own `equals`/`hashCode` - a call site with no declared collation anywhere behaves exactly as + * it did before this function existed, since [Collation.Binary]'s [Collation.canonicalize] is the + * identity. + */ +fun collationFoldedKey( + values: List, + expressions: List, + declaredCollationOf: (ColumnExpression) -> Collation? +): List = values.mapIndexed { index, value -> + if (value !is Value.TextValue) value + else Value.TextValue(resolveCollation(expressions[index], declaredCollationOf).canonicalize(value.value)) +} + +/** + * [collationFoldedKey]'s counterpart for a PRIMARY KEY / `UNIQUE` constraint / `UNIQUE` index + * uniqueness check, where the "expression" naming each key column is always just that column + * itself - a bare index into [schema], not an arbitrary [Expression] tree - so the collation to + * apply is always [kestrel.catalog.Column.collation] directly rather than something resolved + * through [resolveCollation]'s explicit-`COLLATE`-wins rule (a `PRIMARY KEY`/`UNIQUE` column list + * has no expressions to attach an explicit `COLLATE` to in the first place). Used by + * [kestrel.engine.Executor]'s in-memory `enforcePrimaryKeyUniqueness`/`enforceUniqueConstraints`/ + * `enforceUniqueIndexes`, [kestrel.store.sql.DiskTable]'s heap-scanned + * `rejectDuplicateUniqueConstraints`/`backfillSecondaryIndex`, and (via + * [kestrel.store.sql.DiskTable]'s `encodePrimaryKey`/`compositeKey`/`equalRangeFor`) the disk + * engine's own B+Tree-backed PRIMARY KEY and `CREATE UNIQUE INDEX` key encoding itself - folding + * only ever changes what bytes a [kestrel.store.codec.KeyCodec] index key encodes to, never the + * row values a [kestrel.store.heap.RecordId] resolves to, so it is safe for lookup/ordering + * without corrupting any stored row. + */ +fun collationFoldedUniquenessKey(schema: TableSchema, columnIndices: List, values: List): List = + values.mapIndexed { position, value -> + if (value !is Value.TextValue) value + else Value.TextValue(schema.columns[columnIndices[position]].collation.canonicalize(value.value)) + } + +/** + * Single-column counterpart of [collationFoldedKey], for an aggregate's own `DISTINCT` dedup + * (`COUNT`/`SUM`/`AVG`/`MIN`/`MAX(DISTINCT ...)`), which has already reduced its argument down to a + * flat list of values rather than whole rows. + */ +fun List.distinctByCollation(collation: Collation): List = + distinctBy { value -> if (value is Value.TextValue) collation.canonicalize(value.value) else value } + +/** + * SQLite's own two-tier rule for a binary comparison's collating sequence, in decreasing + * precedence: (1) an explicit `COLLATE` on either operand, left first; (2) either operand's own + * declared column `COLLATE` (via [declaredCollationOf], with [Collation.Binary] treated the same + * as "no declaration" so it never out-ranks the *other* side purely by appearing first - see + * [resolveCollation]'s own doc for the identical reasoning), left first; (3) [Collation.Binary]. + * Kept as two separate passes (not one combined per-side [resolveCollation] check) so an explicit + * `COLLATE` on the *right* operand still outranks the *left* operand's own declared column + * collation. Shared by [kestrel.engine.Evaluator]'s `=`/`<>`/`BETWEEN`/`CASE` comparisons and the + * hash-join fast path in [kestrel.engine.SelectRunner] (`hashJoinKeys`/`executeJoin`), so a + * `JOIN ... ON a.x = b.x` bucket lookup agrees with what evaluating the exact same condition + * row-by-row through [kestrel.engine.Evaluator] would have decided. + */ +fun pairCollation( + left: Expression, + right: Expression, + declaredCollationOf: (ColumnExpression) -> Collation? +): Collation { + fun explicitOf(expression: Expression) = (expression as? CollateExpression)?.collation + fun declaredOf(expression: Expression) = + (expression as? ColumnExpression)?.let(declaredCollationOf)?.takeIf { it != Collation.Binary } + return explicitOf(left) ?: explicitOf(right) ?: declaredOf(left) ?: declaredOf(right) ?: Collation.Binary +} diff --git a/src/main/kotlin/kestrel/engine/Evaluator.kt b/src/main/kotlin/kestrel/engine/Evaluator.kt index 72902d7..215a341 100644 --- a/src/main/kotlin/kestrel/engine/Evaluator.kt +++ b/src/main/kotlin/kestrel/engine/Evaluator.kt @@ -1,40 +1,345 @@ package kestrel.engine +import java.math.BigDecimal +import java.math.RoundingMode +import java.time.LocalDate import kestrel.KestrelException import kestrel.catalog.TableSchema +import kestrel.engine.function.ScalarFunctions +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.ExistsExpression import kestrel.sql.ast.Expression import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.CurrentValueExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectStatement import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UnaryOperator +import kestrel.sql.ast.WindowExpression +import kestrel.storage.Collation +import kestrel.storage.ColumnType +import kestrel.storage.DecimalColumns import kestrel.storage.Row import kestrel.storage.Value import kestrel.storage.typeName fun interface ColumnResolver { fun resolve(expression: ColumnExpression): Value + + /** + * The [Collation] [expression] was declared with in its `CREATE TABLE ... COLLATE name` (its + * [kestrel.catalog.Column.collation]), or `null` when this resolver has no such column to + * report one for - a computed/derived row (a set operation's output, a LATERAL subquery's own + * result), an ambiguous reference across more than one source, or simply a resolver that never + * bothered to override this default. [Evaluator.collationOf] only ever calls this for a bare + * [ColumnExpression] with no [kestrel.sql.ast.CollateExpression] of its own - an explicit + * `COLLATE` in the query always wins over whatever this returns. A default method (not the SAM + * member) so every existing lambda-shaped `ColumnResolver { expression -> ... }` construction + * keeps compiling unchanged and simply reports "no declared collation" - only the resolvers + * built directly over a [kestrel.catalog.TableSchema] bother to override it. + */ + fun declaredCollation(expression: ColumnExpression): Collation? = null } -class Evaluator(private val resolver: ColumnResolver? = null) { - constructor(schema: TableSchema?, row: Row?) : this(schema?.let { active -> - ColumnResolver { expression -> - val index = active.columnIndex(expression.name) - ?: throw KestrelException("no such column: ${expression.name}") - row!!.values[index] - } - }) +fun interface SubqueryRunner { + fun run(select: SelectStatement, outer: ColumnResolver?): SelectResult +} + +/** + * Bridges NEXTVAL/CURRVAL evaluation to whichever [kestrel.catalog.Catalog] and per-session state + * actually own sequence counters - kept as its own interface (not a `fun interface` like + * [ColumnResolver]/[SubqueryRunner]) because it needs two independent operations, not one. + * [kestrel.engine.Executor] is the only implementation; every other [Evaluator] construction site + * (SELECT projections, WHERE clauses read-side, CHECK constraints, trigger `WHEN` clauses) omits it, + * so [evaluateNextValue]/[evaluateCurrentValue] throw there exactly like a missing [SubqueryRunner] + * already does for a subquery - deliberately: a volatile, side-effecting call being evaluated an + * engine-implementation-defined number of times (once for filtering, again for projecting the same + * row, ...) is exactly the ambiguity real databases warn about, so this engine only ever allows it + * where a row's assignment is evaluated exactly once by construction (see + * [kestrel.engine.Executor.executeInsertInner]/`executeUpdateInner`). + */ +interface SequenceAccess { + fun nextValue(sequenceName: String): Long + fun currentValue(sequenceName: String): Long +} + +class Evaluator( + private val resolver: ColumnResolver? = null, + private val subqueries: SubqueryRunner? = null, + private val sequences: SequenceAccess? = null, + /** Bound only by [kestrel.engine.Executor.evaluateMergeReturning]'s per-row [Evaluator] - see [MergeActionExpression]'s own doc for why every other call site leaves this null and lets MERGE_ACTION() throw. */ + private val mergeAction: String? = null +) { + constructor(schema: TableSchema?, row: Row?, subqueries: SubqueryRunner? = null, sequences: SequenceAccess? = null) : this( + schema?.let { active -> + object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val index = active.columnIndex(expression.name) + ?: throw KestrelException("no such column: ${expression.name}") + return row!!.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = + active.column(expression.name)?.collation + } + }, + subqueries, + sequences + ) fun evaluate(expression: Expression): Value = when (expression) { is LiteralExpression -> expression.value is ColumnExpression -> evaluateColumn(expression) is UnaryExpression -> evaluateUnary(expression) + is CollateExpression -> evaluate(expression.operand) is BinaryExpression -> evaluateBinary(expression) is FunctionExpression -> throw KestrelException( "aggregate function ${expression.name} is not allowed here" ) + is WindowExpression -> throw KestrelException( + "window function ${expression.name} is not allowed here" + ) + is CallExpression -> evaluateCall(expression) + is IsNullExpression -> evaluateIsNull(expression) + is InExpression -> evaluateIn(expression) + is BetweenExpression -> evaluateBetween(expression) + is LikeExpression -> evaluateLike(expression) + is CaseExpression -> evaluateCase(expression) + is ScalarSubqueryExpression -> evaluateScalarSubquery(expression) + is ExistsExpression -> evaluateExists(expression) + is InSubqueryExpression -> evaluateInSubquery(expression) + is CastExpression -> evaluateCast(expression) + is NextValueExpression -> evaluateNextValue(expression) + is CurrentValueExpression -> evaluateCurrentValue(expression) + is MergeActionExpression -> Value.TextValue(mergeAction ?: throw KestrelException("MERGE_ACTION() is not allowed here")) + is ParameterExpression -> throw KestrelException( + "parameter reference \$${expression.index} is only valid inside a PREPARE-d statement, resolved by EXECUTE" + ) + } + + private fun evaluateNextValue(expression: NextValueExpression): Value = Value.IntegerValue( + (sequences ?: throw KestrelException("NEXTVAL is not allowed here")).nextValue(expression.sequenceName) + ) + + private fun evaluateCurrentValue(expression: CurrentValueExpression): Value = Value.IntegerValue( + (sequences ?: throw KestrelException("CURRVAL is not allowed here")).currentValue(expression.sequenceName) + ) + + private fun runSubquery(select: SelectStatement): SelectResult = + (subqueries ?: throw KestrelException("subqueries are not allowed here")).run(select, resolver) + + private fun evaluateScalarSubquery(expression: ScalarSubqueryExpression): Value { + val result = runSubquery(expression.select) + if (result.columns.size != 1) { + throw KestrelException("scalar subquery must return exactly one column but returned ${result.columns.size}") + } + return when { + result.rows.isEmpty() -> Value.NullValue + result.rows.size > 1 -> throw KestrelException("scalar subquery returned more than one row") + else -> result.rows.single().single() + } + } + + private fun evaluateExists(expression: ExistsExpression): Value = + Value.BooleanValue(runSubquery(expression.select).rows.isNotEmpty()) + + private fun evaluateInSubquery(expression: InSubqueryExpression): Value { + val operand = evaluate(expression.operand) + if (operand is Value.NullValue) return Value.NullValue + val collation = collationOf(expression.operand) ?: Collation.Binary + val result = runSubquery(expression.select) + if (result.columns.size != 1) { + throw KestrelException("IN subquery must return exactly one column but returned ${result.columns.size}") + } + var sawNull = false + for (row in result.rows) { + val candidate = row.single() + if (candidate is Value.NullValue) { + sawNull = true + continue + } + if (compareOperands(operand, candidate, collation) == 0) return Value.BooleanValue(!expression.negated) + } + return if (sawNull) Value.NullValue else Value.BooleanValue(expression.negated) + } + + private fun evaluateCast(expression: CastExpression): Value { + val operand = evaluate(expression.operand) + if (operand is Value.NullValue) return Value.NullValue + return when (expression.targetType) { + ColumnType.Integer -> castToInteger(operand) + ColumnType.Real -> castToReal(operand) + ColumnType.Text -> Value.TextValue(castToText(operand)) + ColumnType.Boolean -> castToBoolean(operand) + ColumnType.Decimal -> castToDecimal(operand, expression.precision, expression.scale) + ColumnType.Date -> castToDate(operand) + } + } + + private fun castToInteger(value: Value): Value = Value.IntegerValue( + when (value) { + is Value.IntegerValue -> value.value + is Value.RealValue -> value.value.toLong() + is Value.DecimalValue -> value.value.toLong() + is Value.DateValue -> value.value.toEpochDay() + is Value.TextValue -> value.value.trim().toLongOrNull() + ?: throw KestrelException("cannot cast '${value.value}' to INTEGER") + else -> throw KestrelException("cannot cast ${value.typeName} to INTEGER") + } + ) + + private fun castToReal(value: Value): Value = Value.RealValue( + when (value) { + is Value.IntegerValue -> value.value.toDouble() + is Value.RealValue -> value.value + is Value.DecimalValue -> value.value.toDouble() + is Value.TextValue -> value.value.trim().toDoubleOrNull() + ?: throw KestrelException("cannot cast '${value.value}' to REAL") + else -> throw KestrelException("cannot cast ${value.typeName} to REAL") + } + ) + + private fun castToText(value: Value): String = when (value) { + is Value.IntegerValue -> value.value.toString() + is Value.RealValue -> value.value.toString() + is Value.DecimalValue -> value.value.toPlainString() + is Value.BooleanValue -> value.value.toString() + is Value.DateValue -> value.value.toString() + is Value.TextValue -> value.value + else -> throw KestrelException("cannot cast ${value.typeName} to TEXT") + } + + private fun castToBoolean(value: Value): Value = Value.BooleanValue( + when (value) { + is Value.BooleanValue -> value.value + is Value.IntegerValue -> value.value != 0L + is Value.TextValue -> when (value.value.trim().lowercase()) { + "true" -> true + "false" -> false + else -> throw KestrelException("cannot cast '${value.value}' to BOOLEAN") + } + else -> throw KestrelException("cannot cast ${value.typeName} to BOOLEAN") + } + ) + + private fun castToDecimal(value: Value, precision: Int?, scale: Int?): Value { + val raw = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + is Value.RealValue -> BigDecimal.valueOf(value.value) + is Value.TextValue -> value.value.trim().toBigDecimalOrNull() + ?: throw KestrelException("cannot cast '${value.value}' to DECIMAL") + else -> throw KestrelException("cannot cast ${value.typeName} to DECIMAL") + } + return Value.DecimalValue(DecimalColumns.normalize(raw, precision, scale, "CAST")) + } + + private fun castToDate(value: Value): Value = Value.DateValue( + when (value) { + is Value.DateValue -> value.value + is Value.IntegerValue -> LocalDate.ofEpochDay(value.value) + is Value.TextValue -> runCatching { LocalDate.parse(value.value.trim()) }.getOrElse { + throw KestrelException("cannot cast '${value.value}' to DATE") + } + else -> throw KestrelException("cannot cast ${value.typeName} to DATE") + } + ) + + private fun evaluateCall(expression: CallExpression): Value { + if (expression.name.equals("GROUPING", ignoreCase = true)) { + throw KestrelException("GROUPING() can only be used in the SELECT list, HAVING, or ORDER BY of a query with GROUP BY") + } + val function = ScalarFunctions.lookup(expression.name) + ?: throw KestrelException("unknown function: ${expression.name}") + return function.apply(expression.arguments.map { evaluate(it) }) + } + + private fun evaluateIsNull(expression: IsNullExpression): Value { + val isNull = evaluate(expression.operand).isNull + return Value.BooleanValue(if (expression.negated) !isNull else isNull) + } + + private fun evaluateIn(expression: InExpression): Value { + val operand = evaluate(expression.operand) + if (operand is Value.NullValue) return Value.NullValue + val collation = collationOf(expression.operand) ?: Collation.Binary + var sawNull = false + for (candidateExpression in expression.values) { + val candidate = evaluate(candidateExpression) + if (candidate is Value.NullValue) { + sawNull = true + continue + } + if (compareOperands(operand, candidate, collation) == 0) return Value.BooleanValue(!expression.negated) + } + return if (sawNull) Value.NullValue else Value.BooleanValue(expression.negated) + } + + private fun evaluateBetween(expression: BetweenExpression): Value { + val operand = evaluate(expression.operand) + val low = evaluate(expression.low) + val high = evaluate(expression.high) + if (operand is Value.NullValue || low is Value.NullValue || high is Value.NullValue) return Value.NullValue + val withinRange = compareOperands(operand, low, pairCollation(expression.operand, expression.low)) >= 0 && + compareOperands(operand, high, pairCollation(expression.operand, expression.high)) <= 0 + return Value.BooleanValue(if (expression.negated) !withinRange else withinRange) + } + + private fun evaluateLike(expression: LikeExpression): Value { + val operand = evaluate(expression.operand) + val pattern = evaluate(expression.pattern) + if (operand is Value.NullValue || pattern is Value.NullValue) return Value.NullValue + if (operand !is Value.TextValue || pattern !is Value.TextValue) { + throw KestrelException("LIKE requires text operands but got ${operand.typeName} and ${pattern.typeName}") + } + val matches = likeToRegex(pattern.value).matches(operand.value) + return Value.BooleanValue(if (expression.negated) !matches else matches) + } + + private fun likeToRegex(pattern: String): Regex { + val builder = StringBuilder("(?s)") + for (character in pattern) { + when (character) { + '%' -> builder.append(".*") + '_' -> builder.append(".") + else -> builder.append(Regex.escape(character.toString())) + } + } + return Regex(builder.toString()) + } + + private fun evaluateCase(expression: CaseExpression): Value { + val operand = expression.operand?.let { evaluate(it) } + for (branch in expression.branches) { + val matched = if (operand != null) { + if (operand is Value.NullValue) { + false + } else { + val candidate = evaluate(branch.condition) + candidate !is Value.NullValue && + compareOperands(operand, candidate, pairCollation(expression.operand!!, branch.condition)) == 0 + } + } else { + asBooleanOrNull(evaluate(branch.condition)) == true + } + if (matched) return evaluate(branch.result) + } + return expression.elseExpression?.let { evaluate(it) } ?: Value.NullValue } private fun evaluateColumn(expression: ColumnExpression): Value { @@ -49,6 +354,7 @@ class Evaluator(private val resolver: ColumnResolver? = null) { UnaryOperator.Negate -> when (operand) { is Value.IntegerValue -> Value.IntegerValue(-operand.value) is Value.RealValue -> Value.RealValue(-operand.value) + is Value.DecimalValue -> Value.DecimalValue(-operand.value) else -> throw KestrelException("unary minus requires a numeric operand but got ${operand.typeName}") } UnaryOperator.Not -> when (operand) { @@ -68,7 +374,7 @@ class Evaluator(private val resolver: ColumnResolver? = null) { return when (operator) { BinaryOperator.Add, BinaryOperator.Subtract, BinaryOperator.Multiply, BinaryOperator.Divide -> evaluateArithmetic(operator, left, right) - else -> evaluateComparison(operator, left, right) + else -> evaluateComparison(operator, left, right, pairCollation(expression.left, expression.right)) } } @@ -98,6 +404,12 @@ class Evaluator(private val resolver: ColumnResolver? = null) { private fun evaluateArithmetic(operator: BinaryOperator, left: Value, right: Value): Value { if (left is Value.NullValue || right is Value.NullValue) return Value.NullValue + if (left is Value.DateValue || right is Value.DateValue) { + return evaluateDateArithmetic(operator, left, right) + } + if (left is Value.DecimalValue || right is Value.DecimalValue) { + return evaluateDecimalArithmetic(operator, left, right) + } if (!isNumeric(left) || !isNumeric(right)) { throw KestrelException("arithmetic requires numeric operands but got ${left.typeName} and ${right.typeName}") } @@ -133,9 +445,9 @@ class Evaluator(private val resolver: ColumnResolver? = null) { ) } - private fun evaluateComparison(operator: BinaryOperator, left: Value, right: Value): Value { + private fun evaluateComparison(operator: BinaryOperator, left: Value, right: Value, collation: Collation): Value { if (left is Value.NullValue || right is Value.NullValue) return Value.NullValue - val ordering = compareOperands(left, right) + val ordering = compareOperands(left, right, collation) val result = when (operator) { BinaryOperator.Equal -> ordering == 0 BinaryOperator.NotEqual -> ordering != 0 @@ -148,14 +460,59 @@ class Evaluator(private val resolver: ColumnResolver? = null) { return Value.BooleanValue(result) } - private fun compareOperands(left: Value, right: Value): Int = when { + /** + * [collation] only ever changes the outcome for two [Value.TextValue]s - every other pairing + * ignores it, matching SQLite (COLLATE is meaningless outside of text comparison). Defaults to + * [Collation.Binary] (this engine's behavior before COLLATE existed) at every call site that + * has no [CollateExpression] to resolve one from. + */ + private fun compareOperands(left: Value, right: Value, collation: Collation = Collation.Binary): Int = when { left is Value.IntegerValue && right is Value.IntegerValue -> left.value.compareTo(right.value) + left is Value.DecimalValue || right is Value.DecimalValue -> { + val a = decimalOf(left) ?: throw KestrelException("cannot compare ${left.typeName} with ${right.typeName}") + val b = decimalOf(right) ?: throw KestrelException("cannot compare ${left.typeName} with ${right.typeName}") + a.compareTo(b) + } isNumeric(left) && isNumeric(right) -> asDouble(left).compareTo(asDouble(right)) - left is Value.TextValue && right is Value.TextValue -> left.value.compareTo(right.value) + left is Value.TextValue && right is Value.TextValue -> collation.compare(left.value, right.value) left is Value.BooleanValue && right is Value.BooleanValue -> left.value.compareTo(right.value) + left is Value.DateValue && right is Value.DateValue -> left.value.compareTo(right.value) else -> throw KestrelException("cannot compare ${left.typeName} with ${right.typeName}") } + /** The [Collation] an explicit `COLLATE` on [expression] itself names, or `null` when it is not a [CollateExpression] at all. */ + private fun explicitCollationOf(expression: Expression): Collation? = (expression as? CollateExpression)?.collation + + /** + * [expression]'s own declared `COLLATE` (see [kestrel.catalog.Column.collation]) when it is a + * bare [ColumnExpression] whose [resolver] reports one - `null` for anything else, including a + * column that never declared one at all ([ColumnResolver.declaredCollation] reporting + * [Collation.Binary], its class default, is treated the exact same as reporting nothing: a + * column that never opted into a non-default sequence should not out-rank an operand on the + * *other* side of a comparison that did, purely by virtue of appearing on the left - see + * [pairCollation]). + */ + private fun columnDeclaredCollationOf(expression: Expression): Collation? = + (expression as? ColumnExpression)?.let { resolver?.declaredCollation(it) }?.takeIf { it != Collation.Binary } + + /** + * The [Collation] [expression] compares with on its own, combining both of [pairCollation]'s + * tiers into one - used where there is only a single operand to consult + * ([evaluateIn]/[evaluateInSubquery]'s list/subquery operand), which can still tell "no COLLATE + * here at all" apart from "explicitly COLLATE BINARY" (both [explicitCollationOf] and + * [columnDeclaredCollationOf] return `null` only for the former). + */ + private fun collationOf(expression: Expression): Collation? = explicitCollationOf(expression) ?: columnDeclaredCollationOf(expression) + + /** + * Delegates to the shared [kestrel.engine.pairCollation] (SQLite's two-tier binary-comparison + * rule) bound to this evaluator's own [resolver] - see that function's doc for the precedence + * rule itself and why it must stay two separate passes rather than one combined + * [collationOf]-per-side check. + */ + private fun pairCollation(left: Expression, right: Expression): Collation = + kestrel.engine.pairCollation(left, right) { resolver?.declaredCollation(it) } + private fun isNumeric(value: Value): Boolean = value is Value.IntegerValue || value is Value.RealValue private fun asDouble(value: Value): Double = when (value) { @@ -163,4 +520,47 @@ class Evaluator(private val resolver: ColumnResolver? = null) { is Value.RealValue -> value.value else -> error("not a numeric value") } + + private fun decimalOf(value: Value): BigDecimal? = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> null + } + + private fun evaluateDateArithmetic(operator: BinaryOperator, left: Value, right: Value): Value = when { + left is Value.DateValue && right is Value.DateValue -> when (operator) { + BinaryOperator.Subtract -> Value.IntegerValue(left.value.toEpochDay() - right.value.toEpochDay()) + else -> throw KestrelException("DATE ${operator.symbol} DATE is not supported; only subtraction (day difference) is") + } + left is Value.DateValue && right is Value.IntegerValue -> when (operator) { + BinaryOperator.Add -> Value.DateValue(left.value.plusDays(right.value)) + BinaryOperator.Subtract -> Value.DateValue(left.value.minusDays(right.value)) + else -> throw KestrelException("DATE only supports + and - with an INTEGER day offset") + } + left is Value.IntegerValue && right is Value.DateValue && operator == BinaryOperator.Add -> + Value.DateValue(right.value.plusDays(left.value)) + else -> throw KestrelException("arithmetic requires numeric operands but got ${left.typeName} and ${right.typeName}") + } + + private fun evaluateDecimalArithmetic(operator: BinaryOperator, left: Value, right: Value): Value { + val a = decimalOf(left) ?: throw KestrelException("arithmetic requires numeric operands but got ${left.typeName} and ${right.typeName}") + val b = decimalOf(right) ?: throw KestrelException("arithmetic requires numeric operands but got ${left.typeName} and ${right.typeName}") + return Value.DecimalValue( + when (operator) { + BinaryOperator.Add -> a + b + BinaryOperator.Subtract -> a - b + BinaryOperator.Multiply -> a * b + BinaryOperator.Divide -> { + if (b.signum() == 0) throw KestrelException("division by zero") + a.divide(b, maxOf(a.scale(), b.scale(), MIN_DIVISION_SCALE), RoundingMode.HALF_UP) + } + else -> error("unreachable arithmetic operator") + } + ) + } + + private companion object { + const val MIN_DIVISION_SCALE = 10 + } } diff --git a/src/main/kotlin/kestrel/engine/Executor.kt b/src/main/kotlin/kestrel/engine/Executor.kt index 2c910d3..48cfd8f 100644 --- a/src/main/kotlin/kestrel/engine/Executor.kt +++ b/src/main/kotlin/kestrel/engine/Executor.kt @@ -3,52 +3,215 @@ package kestrel.engine import kestrel.KestrelException import kestrel.catalog.Catalog import kestrel.catalog.CatalogSnapshot +import kestrel.catalog.CheckConstraint import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyConstraint import kestrel.catalog.IndexDefinition +import kestrel.catalog.Sequence +import kestrel.catalog.SequenceDefinition import kestrel.catalog.TableSchema +import kestrel.catalog.TriggerDefinition +import kestrel.catalog.UniqueConstraint +import kestrel.sql.ast.AddColumn +import kestrel.sql.ast.AddConstraint +import kestrel.sql.ast.AddedCheck +import kestrel.sql.ast.AddedConstraint +import kestrel.sql.ast.AddedForeignKey +import kestrel.sql.ast.AddedUnique +import kestrel.sql.ast.AlterSequenceStatement +import kestrel.sql.ast.AlterTableStatement import kestrel.sql.ast.BeginStatement +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.ChangeColumnType +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression import kestrel.sql.ast.CommitStatement +import kestrel.sql.ast.ConflictDoNothing +import kestrel.sql.ast.ConflictDoUpdate import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateSequenceStatement import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.DeallocateStatement import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropColumn +import kestrel.sql.ast.DropConstraint import kestrel.sql.ast.DropIndexStatement +import kestrel.sql.ast.DropMode +import kestrel.sql.ast.DropSequenceStatement import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.DropTriggerStatement +import kestrel.sql.ast.DropViewStatement +import kestrel.sql.ast.ExecuteStatement +import kestrel.sql.ast.ExistsExpression import kestrel.sql.ast.Expression import kestrel.sql.ast.ExplainStatement import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.OnConflictClause +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.PrepareStatement +import kestrel.sql.ast.PreparableStatement +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.ReleaseSavepointStatement +import kestrel.sql.ast.RenameColumn +import kestrel.sql.ast.RenameTable import kestrel.sql.ast.RollbackStatement -import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.RollbackToSavepointStatement +import kestrel.sql.ast.SavepointStatement +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectItem +import kestrel.sql.ast.StarProjection import kestrel.sql.ast.Statement +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.sql.ast.TruncateTableStatement import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import java.math.BigDecimal +import kestrel.storage.Collation import kestrel.storage.ColumnType +import kestrel.storage.DecimalColumns import kestrel.storage.Row +import kestrel.storage.Table import kestrel.storage.Value +import kestrel.storage.displayWith import kestrel.storage.typeName -class Executor(private val catalog: Catalog) { +class Executor(private val catalog: Catalog) : SequenceAccess { private val selectRunner = SelectRunner(catalog) + private val subqueryRunner = SubqueryRunner { select, outer -> selectRunner.run(select, outer) } private var transactionSnapshot: CatalogSnapshot? = null + private val savepoints = mutableListOf>() + + /** Guards against a trigger whose body (directly or transitively) fires itself forever. */ + private var triggerDepth = 0 + + /** + * Postgres' CURRVAL is scoped to "the current session" - this engine's closest equivalent is + * one [Executor] instance. Keyed by [Sequence] identity rather than by name so that ALTER + * SEQUENCE (which always installs a new [Sequence] instance, see [Sequence.altered]) or a + * DROP+CREATE of the same name invalidates any remembered value instead of silently handing + * back a value that belonged to a different configuration of "the same" name - a deliberately + * conservative choice over trying to decide case by case which ALTERs should or shouldn't + * preserve it. + */ + private val lastSequenceValues = HashMap() + + /** + * Session-scoped, deliberately outside [catalog] - see [PrepareStatement]'s own doc for why a + * prepared statement is not part of the transactional snapshot/restore every other catalog + * object participates in. + */ + private val preparedStatements = linkedMapOf() + + val preparedStatementNames: List get() = preparedStatements.keys.toList() + + override fun nextValue(sequenceName: String): Long { + val sequence = catalog.sequence(sequenceName) + return sequence.next().also { lastSequenceValues[sequence] = it } + } + + override fun currentValue(sequenceName: String): Long { + val sequence = catalog.sequence(sequenceName) + return lastSequenceValues[sequence] + ?: throw KestrelException("currval of sequence '$sequenceName' is not yet defined in this session - call NEXTVAL first") + } val inTransaction: Boolean get() = transactionSnapshot != null fun execute(statement: Statement): StatementResult = when (statement) { is CreateTableStatement -> executeCreateTable(statement) is DropTableStatement -> executeDropTable(statement) + is TruncateTableStatement -> executeTruncate(statement) + is AlterTableStatement -> executeAlterTable(statement) is CreateIndexStatement -> executeCreateIndex(statement) is DropIndexStatement -> executeDropIndex(statement) + is CreateViewStatement -> executeCreateView(statement) + is DropViewStatement -> executeDropView(statement) + is CreateTriggerStatement -> executeCreateTrigger(statement) + is DropTriggerStatement -> executeDropTrigger(statement) + is CreateSequenceStatement -> executeCreateSequence(statement) + is DropSequenceStatement -> executeDropSequence(statement) + is AlterSequenceStatement -> executeAlterSequence(statement) is InsertStatement -> executeInsert(statement) - is SelectStatement -> selectRunner.run(statement) + is QueryStatement -> selectRunner.runQuery(statement) is DeleteStatement -> executeDelete(statement) is UpdateStatement -> executeUpdate(statement) + is MergeStatement -> executeMerge(statement) is ExplainStatement -> ExplainResult(selectRunner.explain(statement.select)) is BeginStatement -> executeBegin() is CommitStatement -> executeCommit() is RollbackStatement -> executeRollback() + is SavepointStatement -> executeSavepoint(statement) + is ReleaseSavepointStatement -> executeReleaseSavepoint(statement) + is RollbackToSavepointStatement -> executeRollbackToSavepoint(statement) + is PrepareStatement -> executePrepare(statement) + is ExecuteStatement -> execute(resolveExecute(statement)) + is DeallocateStatement -> executeDeallocate(statement) + } + + private fun executePrepare(statement: PrepareStatement): StatementResult { + val key = statement.name.lowercase() + if (preparedStatements.containsKey(key)) { + throw KestrelException("prepared statement '${statement.name}' already exists") + } + preparedStatements[key] = statement.target + return PrepareResult(statement.name) + } + + /** + * Looks [statement]'s name up, evaluates every argument expression as a CONSTANT (a bare + * `Evaluator()` with no [ColumnResolver]/[SubqueryRunner]/[SequenceAccess] - a column reference, + * subquery, or NEXTVAL/CURRVAL in an EXECUTE argument throws exactly like one would in a DEFAULT + * expression, for the same reason: there is no row to resolve any of them against), and + * substitutes the results for every `$1`, `$2`, ... in the prepared statement. The result is + * always a concrete [PreparableStatement] - never [ExecuteStatement] itself - ready to run + * through [execute] like any statement typed directly. + */ + fun resolveExecute(statement: ExecuteStatement): Statement { + val target = preparedStatements[statement.name.lowercase()] + ?: throw KestrelException("no such prepared statement: ${statement.name}") + val required = ParameterSubstitution.parameterCount(target) + if (statement.arguments.size != required) { + throw KestrelException( + "prepared statement '${statement.name}' expects $required parameter(s) but got ${statement.arguments.size}" + ) + } + val values = statement.arguments.map { Evaluator().evaluate(it) } + return ParameterSubstitution.substitute(target, values) + } + + private fun executeDeallocate(statement: DeallocateStatement): StatementResult { + if (statement.all) { + preparedStatements.clear() + return DeallocateResult(null, all = true) + } + val name = statement.name!! + if (preparedStatements.remove(name.lowercase()) == null) { + throw KestrelException("no such prepared statement: $name") + } + return DeallocateResult(name, all = false) } private fun executeBegin(): StatementResult { @@ -60,6 +223,7 @@ class Executor(private val catalog: Catalog) { private fun executeCommit(): StatementResult { if (!inTransaction) throw KestrelException("no transaction is in progress") transactionSnapshot = null + savepoints.clear() return CommitResult } @@ -67,11 +231,47 @@ class Executor(private val catalog: Catalog) { val snapshot = transactionSnapshot ?: throw KestrelException("no transaction is in progress") catalog.restore(snapshot) transactionSnapshot = null + savepoints.clear() return RollbackResult } + /** + * A savepoint that reuses an already-established name is intentionally allowed (matches + * Postgres): it shadows the earlier one, and RELEASE/ROLLBACK TO always target the most + * recently established savepoint of that name via [indexOfLastSavepoint], leaving the older, + * shadowed one addressable again once the newer one is released or rolled back past. + */ + private fun executeSavepoint(statement: SavepointStatement): StatementResult { + if (!inTransaction) throw KestrelException("SAVEPOINT can only be used inside a transaction") + savepoints.add(statement.name to catalog.snapshot()) + return SavepointResult(statement.name) + } + + private fun executeReleaseSavepoint(statement: ReleaseSavepointStatement): StatementResult { + if (!inTransaction) throw KestrelException("RELEASE SAVEPOINT can only be used inside a transaction") + val index = indexOfLastSavepoint(statement.name) + while (savepoints.size > index) savepoints.removeAt(savepoints.size - 1) + return ReleaseSavepointResult(statement.name) + } + + private fun executeRollbackToSavepoint(statement: RollbackToSavepointStatement): StatementResult { + if (!inTransaction) throw KestrelException("ROLLBACK TO SAVEPOINT can only be used inside a transaction") + val index = indexOfLastSavepoint(statement.name) + catalog.restore(savepoints[index].second) + while (savepoints.size > index + 1) savepoints.removeAt(savepoints.size - 1) + return RollbackToSavepointResult(statement.name) + } + + private fun indexOfLastSavepoint(name: String): Int { + val index = savepoints.indexOfLast { it.first.equals(name, ignoreCase = true) } + if (index < 0) throw KestrelException("no such savepoint: $name") + return index + } + private fun executeCreateTable(statement: CreateTableStatement): StatementResult { - if (statement.columns.count { it.primaryKey } > 1) { + if (statement.columns.count { it.primaryKey } > 1 || + (statement.compositePrimaryKey.isNotEmpty() && statement.columns.any { it.primaryKey }) + ) { throw KestrelException("table '${statement.tableName}' declares more than one primary key") } val seen = mutableSetOf() @@ -80,17 +280,673 @@ class Executor(private val catalog: Catalog) { throw KestrelException("duplicate column name: ${column.name}") } } - catalog.createTable(TableSchema(statement.tableName, statement.columns)) + val columns = applyCompositePrimaryKey(statement) + validateForeignKeyDefinitions(statement, columns) + validateCheckDefinitions(statement) + validateUniqueDefinitions(statement) + validateDefaultDefinitions(columns) + validateGeneratedDefinitions(columns) + catalog.createTable( + TableSchema( + statement.tableName, + columns, + statement.foreignKeys, + statement.checks, + statement.uniqueConstraints, + statement.compositePrimaryKey, + statement.temporary + ) + ) return CreateTableResult(statement.tableName) } + /** + * A table-level `PRIMARY KEY (a, b, ...)` implies NOT NULL on every listed column, exactly + * like the inline `col TYPE PRIMARY KEY` spelling already does - applied here (rather than in + * the parser) because the parser builds [Column]s one at a time and has no view of the whole + * table-level clause yet when it does. + */ + private fun applyCompositePrimaryKey(statement: CreateTableStatement): List { + if (statement.compositePrimaryKey.isEmpty()) return statement.columns + val declared = statement.compositePrimaryKey.map { it.lowercase() }.toSet() + if (declared.size != statement.compositePrimaryKey.size) { + throw KestrelException("table '${statement.tableName}' declares a duplicate column in its PRIMARY KEY") + } + for (name in statement.compositePrimaryKey) { + if (statement.columns.none { it.name.equals(name, ignoreCase = true) }) { + throw KestrelException("PRIMARY KEY refers to unknown column: $name") + } + } + return statement.columns.map { column -> + if (column.name.lowercase() in declared) column.copy(notNull = true) else column + } + } + + private fun validateUniqueDefinitions(statement: CreateTableStatement) { + for (unique in statement.uniqueConstraints) { + for (columnName in unique.columns) { + if (statement.columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw KestrelException("UNIQUE constraint refers to unknown column: $columnName") + } + } + } + } + + /** + * A self-reference (a FK on the table pointing back at itself, e.g. an employee/manager + * hierarchy) is checked against a [TableSchema] built from [columns] (already normalized by + * [applyCompositePrimaryKey]) rather than the catalog, because the table doesn't exist there + * yet while it's being created. + */ + private fun validateForeignKeyDefinitions(statement: CreateTableStatement, columns: List) { + for (fk in statement.foreignKeys) { + if (fk.columns.size != fk.referencedColumns.size) { + throw KestrelException( + "FOREIGN KEY (${fk.columns.joinToString(", ")}) column count does not match " + + "REFERENCES ${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")})" + ) + } + for (columnName in fk.columns) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw KestrelException("FOREIGN KEY refers to unknown column: $columnName") + } + } + val selfReference = fk.referencedTable.equals(statement.tableName, ignoreCase = true) + val hasEnforcedUniqueness = if (selfReference) { + for (columnName in fk.referencedColumns) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw KestrelException( + "FOREIGN KEY references unknown column: ${fk.referencedTable}.$columnName" + ) + } + } + val selfSchema = TableSchema( + statement.tableName, + columns, + uniqueConstraints = statement.uniqueConstraints, + compositePrimaryKey = statement.compositePrimaryKey + ) + selfSchema.hasEnforcedUniqueness(fk.referencedColumns) + } else { + val referencedTable = catalog.tableOrNull(fk.referencedTable) + ?: throw KestrelException("FOREIGN KEY references unknown table: ${fk.referencedTable}") + for (columnName in fk.referencedColumns) { + if (referencedTable.schema.column(columnName) == null) { + throw KestrelException( + "FOREIGN KEY references unknown column: ${fk.referencedTable}.$columnName" + ) + } + } + referencedTable.schema.hasEnforcedUniqueness(fk.referencedColumns) + } + if (!hasEnforcedUniqueness) { + throw KestrelException( + "FOREIGN KEY must reference a PRIMARY KEY or UNIQUE column(s), but " + + "${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")}) is not one" + ) + } + } + } + + private fun validateCheckDefinitions(statement: CreateTableStatement) { + val schema = TableSchema(statement.tableName, statement.columns) + for (check in statement.checks) { + checkColumns(check.expression, schema) + requireNoSubquery(check.expression) + } + } + + private fun validateDefaultDefinitions(columns: List) { + for (column in columns) { + val default = column.default ?: continue + validateConstantDefault(default) + if (default is NextValueExpression) { + validateNextValueDefault(default, column) + continue + } + coerceForColumn(Evaluator().evaluate(default), column) + } + } + + /** + * A GENERATED column's expression is recomputed from this row's OTHER column values on every + * INSERT/UPDATE (see [applyGeneratedColumns]) - it must therefore be a pure function of that + * row's own columns: [checkColumns] rejects a reference to a nonexistent column plus any + * aggregate/window function, [requireNoSubquery] rejects a subquery and NEXTVAL/CURRVAL/ + * MERGE_ACTION() on top of that (same two-function combination [validateCheckDefinitions] uses + * for CHECK), and [requireNoGeneratedColumnReference] additionally forbids referencing another + * GENERATED column - not because it's unsound in general, but because allowing it would need a + * dependency-ordered evaluation pass instead of the current single independent pass over every + * GENERATED column (see [applyGeneratedColumns]), matching Postgres's own restriction. + */ + private fun validateGeneratedDefinitions(columns: List) { + val schema = TableSchema("", columns) + val generatedNames = columns.filter { it.generated != null }.map { it.name.lowercase() }.toSet() + for (column in columns) { + val generated = column.generated ?: continue + if (column.default != null) { + throw KestrelException("column ${column.name} cannot declare both DEFAULT and GENERATED ALWAYS AS") + } + checkColumns(generated, schema) + requireNoSubquery(generated) + requireNoGeneratedColumnReference(generated, generatedNames, column.name) + } + } + + private fun requireNoGeneratedColumnReference(expression: Expression, generatedNames: Set, ownerColumn: String) { + when (expression) { + is ColumnExpression -> if (expression.name.lowercase() in generatedNames) { + throw KestrelException( + "GENERATED column $ownerColumn cannot reference another GENERATED column: ${expression.name}" + ) + } + is LiteralExpression -> Unit + is BinaryExpression -> { + requireNoGeneratedColumnReference(expression.left, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.right, generatedNames, ownerColumn) + } + is UnaryExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is CallExpression -> expression.arguments.forEach { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + is IsNullExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is InExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + expression.values.forEach { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + } + is BetweenExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.low, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.high, generatedNames, ownerColumn) + } + is LikeExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.pattern, generatedNames, ownerColumn) + } + is CaseExpression -> { + expression.operand?.let { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + expression.branches.forEach { + requireNoGeneratedColumnReference(it.condition, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(it.result, generatedNames, ownerColumn) + } + expression.elseExpression?.let { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + } + is CastExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is CollateExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + // Already rejected earlier in validateGeneratedDefinitions by checkColumns/requireNoSubquery - + // kept here only so this `when` stays exhaustive. + is FunctionExpression, is WindowExpression, is ScalarSubqueryExpression, is ExistsExpression, + is InSubqueryExpression, is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, + is ParameterExpression -> Unit + } + } + + /** + * Recomputes every GENERATED column in [values] from its expression, in place - safe to call + * any time after every non-GENERATED column has its final value for this row (explicit + * assignment, DEFAULT, or otherwise), because [validateGeneratedDefinitions] guarantees a + * GENERATED expression only ever reads a non-GENERATED column of the same row, never another + * GENERATED one - so evaluation order among multiple GENERATED columns never matters. + */ + private fun applyGeneratedColumns(schema: TableSchema, values: MutableList) { + if (schema.columns.none { it.generated != null }) return + val evaluator = Evaluator(schema, Row(values), subqueryRunner) + for ((index, column) in schema.columns.withIndex()) { + val generated = column.generated ?: continue + values[index] = coerceForColumn(evaluator.evaluate(generated), column) + } + } + + /** Standard SQL forbids assigning a GENERATED column's value directly - it is always derived, never set. */ + private fun rejectGeneratedAssignment(column: Column, explicitValueGiven: Boolean) { + if (explicitValueGiven && column.generated != null) { + throw KestrelException("cannot assign an explicit value to GENERATED column ${column.name}") + } + } + + /** + * [NextValueExpression] skips the generic "evaluate once to see what type comes back" check + * [validateDefaultDefinitions] otherwise uses, because evaluating it for real would mean a bare + * `CREATE TABLE` actually consumes a sequence value before a single row exists - instead this + * checks the two things that actually matter statically: the sequence is real, and the column it + * defaults can hold what NEXTVAL returns. + */ + private fun validateNextValueDefault(default: NextValueExpression, column: Column) { + catalog.sequence(default.sequenceName) + if (column.type != ColumnType.Integer) { + throw KestrelException("NEXTVAL can only be used as a DEFAULT for an INTEGER column, but ${column.name} is ${column.type.displayName}") + } + } + + /** + * A DEFAULT must be a constant expression: no column reference (there is no row to read one + * from when a value is missing at INSERT time, or none at all yet when backfilling ALTER + * TABLE ADD COLUMN), no subquery, and no aggregate. [NextValueExpression] is the one exception - + * see [validateNextValueDefault] - because unlike everything else here it names an external, + * mutable counter rather than something computable from the expression alone. + * [CurrentValueExpression] is rejected outright: CURRVAL only ever means something after a prior + * NEXTVAL in the same session, which can never be true yet at CREATE TABLE/ADD COLUMN time. + */ + private fun validateConstantDefault(expression: Expression) { + when (expression) { + is ColumnExpression -> throw KestrelException("DEFAULT expression cannot reference a column") + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression -> + throw KestrelException("DEFAULT expression cannot contain a subquery") + is FunctionExpression -> throw KestrelException("DEFAULT expression cannot contain an aggregate function") + is BinaryExpression -> { + validateConstantDefault(expression.left) + validateConstantDefault(expression.right) + } + is UnaryExpression -> validateConstantDefault(expression.operand) + is CallExpression -> expression.arguments.forEach { validateConstantDefault(it) } + is IsNullExpression -> validateConstantDefault(expression.operand) + is InExpression -> { + validateConstantDefault(expression.operand) + expression.values.forEach { validateConstantDefault(it) } + } + is BetweenExpression -> { + validateConstantDefault(expression.operand) + validateConstantDefault(expression.low) + validateConstantDefault(expression.high) + } + is LikeExpression -> { + validateConstantDefault(expression.operand) + validateConstantDefault(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { validateConstantDefault(it) } + expression.branches.forEach { + validateConstantDefault(it.condition) + validateConstantDefault(it.result) + } + expression.elseExpression?.let { validateConstantDefault(it) } + } + is CastExpression -> validateConstantDefault(expression.operand) + is CollateExpression -> validateConstantDefault(expression.operand) + is WindowExpression -> throw KestrelException("DEFAULT expression cannot contain a window function") + is NextValueExpression -> Unit + is CurrentValueExpression -> throw KestrelException("DEFAULT expression cannot contain CURRVAL") + is MergeActionExpression -> throw KestrelException("DEFAULT expression cannot contain MERGE_ACTION()") + is ParameterExpression -> throw KestrelException("DEFAULT expression cannot contain a parameter reference") + is LiteralExpression -> Unit + } + } + + private fun requireNoSubquery(expression: Expression) { + when (expression) { + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression -> + throw KestrelException("CHECK constraints and GENERATED column expressions cannot contain subqueries") + is FunctionExpression -> throw KestrelException("aggregate functions are not allowed in CHECK constraints") + is BinaryExpression -> { + requireNoSubquery(expression.left) + requireNoSubquery(expression.right) + } + is UnaryExpression -> requireNoSubquery(expression.operand) + is CallExpression -> expression.arguments.forEach { requireNoSubquery(it) } + is IsNullExpression -> requireNoSubquery(expression.operand) + is InExpression -> { + requireNoSubquery(expression.operand) + expression.values.forEach { requireNoSubquery(it) } + } + is BetweenExpression -> { + requireNoSubquery(expression.operand) + requireNoSubquery(expression.low) + requireNoSubquery(expression.high) + } + is LikeExpression -> { + requireNoSubquery(expression.operand) + requireNoSubquery(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { requireNoSubquery(it) } + expression.branches.forEach { + requireNoSubquery(it.condition) + requireNoSubquery(it.result) + } + expression.elseExpression?.let { requireNoSubquery(it) } + } + is CastExpression -> requireNoSubquery(expression.operand) + is CollateExpression -> requireNoSubquery(expression.operand) + is WindowExpression -> throw KestrelException("window functions are not allowed in CHECK constraints") + is NextValueExpression, is CurrentValueExpression -> + throw KestrelException("NEXTVAL/CURRVAL are not allowed in a CHECK constraint, trigger WHEN clause, or GENERATED column expression") + is MergeActionExpression -> + throw KestrelException("MERGE_ACTION() is not allowed in a CHECK constraint, trigger WHEN clause, or GENERATED column expression") + is ParameterExpression -> + throw KestrelException("a parameter reference is not allowed in a CHECK constraint, trigger WHEN clause, or GENERATED column expression") + is ColumnExpression, is LiteralExpression -> Unit + } + } + + private fun enforceChecks(schema: TableSchema, row: Row) { + for (check in schema.checks) { + val result = Evaluator(schema, row, subqueryRunner).evaluate(check.expression) + if (result is Value.BooleanValue && !result.value) { + val label = check.name?.let { " $it" } ?: "" + throw KestrelException("CHECK constraint$label failed: ${schema.name}") + } + } + } + private fun executeDropTable(statement: DropTableStatement): StatementResult { - catalog.dropTable(statement.tableName) + catalog.dropTable(statement.tableName, cascade = statement.mode == DropMode.Cascade) return DropTableResult(statement.tableName) } + /** + * TRUNCATE never fires a row-level trigger (Postgres/SQLite both only ever fire TRUNCATE-shaped + * statement-level triggers this engine has no equivalent of) and never runs a FOREIGN KEY's + * `onDelete` action row by row - see [kestrel.catalog.Catalog.truncateTables]'s own doc for why + * its RESTRICT/CASCADE handling is structurally different from [ForeignKeyEnforcer.cascadeDelete]. + */ + private fun executeTruncate(statement: TruncateTableStatement): StatementResult { + val truncated = catalog.truncateTables(statement.tableNames, statement.mode == DropMode.Cascade, statement.restartIdentity) + return TruncateTableResult(truncated) + } + + private fun executeAlterTable(statement: AlterTableStatement): StatementResult { + when (val action = statement.action) { + is AddColumn -> executeAddColumn(statement.tableName, action.column) + is DropColumn -> catalog.dropColumn(statement.tableName, action.columnName, cascade = action.mode == DropMode.Cascade) + is RenameColumn -> catalog.renameColumn(statement.tableName, action.oldName, action.newName) + is RenameTable -> catalog.renameTable(statement.tableName, action.newName) + is AddConstraint -> executeAddConstraint(statement.tableName, action.constraint) + is DropConstraint -> catalog.dropConstraint(statement.tableName, action.name, cascade = action.mode == DropMode.Cascade) + is ChangeColumnType -> executeChangeColumnType(statement.tableName, action) + } + val resultName = (statement.action as? RenameTable)?.newName ?: statement.tableName + return AlterTableResult(resultName) + } + + private fun executeAddConstraint(tableName: String, added: AddedConstraint) { + when (added) { + is AddedCheck -> executeAddCheckConstraint(tableName, added.check) + is AddedUnique -> executeAddUniqueConstraint(tableName, added.unique) + is AddedForeignKey -> executeAddForeignKeyConstraint(tableName, added.foreignKey) + } + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name CHECK (...)` - validated exactly like a CREATE TABLE-time + * CHECK ([validateCheckDefinitions]: known columns only, no subquery), PLUS - since this table + * may already hold rows a brand new CREATE TABLE never has to worry about - every existing row + * is evaluated against it up front, the same per-row logic [enforceChecks] already runs on every + * future INSERT/UPDATE. + */ + private fun executeAddCheckConstraint(tableName: String, check: CheckConstraint) { + val schema = catalog.table(tableName).schema + check.name?.let { requireUniqueConstraintName(schema, it) } + checkColumns(check.expression, schema) + requireNoSubquery(check.expression) + for (row in catalog.table(tableName).rows) { + val result = Evaluator(schema, row, subqueryRunner).evaluate(check.expression) + if (result is Value.BooleanValue && !result.value) { + val label = check.name?.let { " $it" } ?: "" + throw KestrelException("CHECK constraint$label failed: an existing row in $tableName violates it") + } + } + catalog.addCheckConstraint(tableName, check) + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name UNIQUE (...)` - every existing row is checked for a + * duplicate combination of [unique]'s columns before the constraint is installed, the same + * [collationFoldedUniquenessKey]-folded, NULL-exempt logic [enforceUniqueConstraints] already + * runs on every future INSERT/UPDATE, just run once up front here against rows that already exist. + */ + private fun executeAddUniqueConstraint(tableName: String, unique: UniqueConstraint) { + val schema = catalog.table(tableName).schema + unique.name?.let { requireUniqueConstraintName(schema, it) } + for (columnName in unique.columns) { + if (schema.columnIndex(columnName) == null) { + throw KestrelException("UNIQUE constraint refers to unknown column: $columnName") + } + } + val columnIndices = unique.columns.map { schema.columnIndex(it)!! } + val seen = HashSet>() + for (row in catalog.table(tableName).rows) { + val rawKey = columnIndices.map { row.values[it] } + if (rawKey.any { it is Value.NullValue }) continue + if (!seen.add(collationFoldedUniquenessKey(schema, columnIndices, rawKey))) { + val label = unique.name?.let { " $it" } ?: "" + throw KestrelException( + "UNIQUE constraint$label failed: an existing row combination in " + + "$tableName(${unique.columns.joinToString(", ")}) is already duplicated" + ) + } + } + catalog.addUniqueConstraint(tableName, unique) + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name FOREIGN KEY (...) REFERENCES ...` - validated the same way + * [validateForeignKeyDefinitions] validates one at CREATE TABLE time (unknown column, referenced + * table/column existence, enforced uniqueness of the referenced column group), PLUS every + * existing row of [tableName] is checked against it via [ForeignKeyEnforcer.validateRow] before + * the constraint is installed - run against a throwaway [TableSchema] carrying [foreignKey] + * alongside every FOREIGN KEY [tableName] already has, so a self-referencing FOREIGN KEY + * (`referencedTable == tableName`) can find its own sibling rows the same way + * [validateForeignKeyDefinitions] does for a genuinely new table, except those "siblings" here + * are the table's own already-committed rows rather than an uncommitted INSERT batch. + */ + private fun executeAddForeignKeyConstraint(tableName: String, foreignKey: ForeignKeyConstraint) { + val schema = catalog.table(tableName).schema + foreignKey.name?.let { requireUniqueConstraintName(schema, it) } + if (foreignKey.columns.size != foreignKey.referencedColumns.size) { + throw KestrelException( + "FOREIGN KEY (${foreignKey.columns.joinToString(", ")}) column count does not match " + + "REFERENCES ${foreignKey.referencedTable}(${foreignKey.referencedColumns.joinToString(", ")})" + ) + } + for (columnName in foreignKey.columns) { + if (schema.columnIndex(columnName) == null) { + throw KestrelException("FOREIGN KEY refers to unknown column: $columnName") + } + } + val selfReference = foreignKey.referencedTable.equals(tableName, ignoreCase = true) + val referencedSchema = if (selfReference) { + schema + } else { + catalog.tableOrNull(foreignKey.referencedTable)?.schema + ?: throw KestrelException("FOREIGN KEY references unknown table: ${foreignKey.referencedTable}") + } + for (columnName in foreignKey.referencedColumns) { + if (referencedSchema.column(columnName) == null) { + throw KestrelException("FOREIGN KEY references unknown column: ${foreignKey.referencedTable}.$columnName") + } + } + if (!referencedSchema.hasEnforcedUniqueness(foreignKey.referencedColumns)) { + throw KestrelException( + "FOREIGN KEY must reference a PRIMARY KEY or UNIQUE column(s), but " + + "${foreignKey.referencedTable}(${foreignKey.referencedColumns.joinToString(", ")}) is not one" + ) + } + val tempSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys + foreignKey, schema.checks, + schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + for (row in catalog.table(tableName).rows) { + ForeignKeyEnforcer.validateRow(catalog, tempSchema, row.values) + } + catalog.addForeignKey(tableName, foreignKey) + } + + /** + * `ALTER TABLE t ALTER COLUMN col TYPE newtype [USING expression]` - deliberately RESTRICT-only + * (no CASCADE option, unlike [kestrel.catalog.Catalog.dropColumn]): a column that is part of + * the PRIMARY KEY, any FOREIGN KEY (this table's own, or another table's referencing it), a + * UNIQUE constraint, or a secondary index is rejected outright rather than threading a type + * change through every one of those structures too - drop the dependency first, exactly like + * `ALTER TABLE ADD COLUMN` already refuses to declare one of those inline + * ([kestrel.sql.parser.Parser.parseAlterAddColumnDefinition]). A GENERATED column is rejected + * for a related but distinct reason: its value is always freshly recomputed from its own + * expression, never carried over from an OLD value the way [ChangeColumnType.using] (or its + * implicit "recompute this column as its own old value" default) assumes - changing its type + * means dropping and re-adding it with a new expression instead. + */ + private fun executeChangeColumnType(tableName: String, action: ChangeColumnType) { + val schema = catalog.table(tableName).schema + val columnIndex = schema.columnIndex(action.columnName) + ?: throw KestrelException("no such column: ${action.columnName} in table $tableName") + val column = schema.columns[columnIndex] + if (column.generated != null) { + throw KestrelException( + "cannot change the type of GENERATED column ${action.columnName} in $tableName; drop and re-add it instead" + ) + } + if (schema.primaryKeyColumns.any { it.equals(action.columnName, ignoreCase = true) }) { + throw KestrelException( + "cannot change the type of column ${action.columnName} because it is part of the PRIMARY KEY of $tableName" + ) + } + if (schema.foreignKeys.any { fk -> fk.columns.any { it.equals(action.columnName, ignoreCase = true) } }) { + throw KestrelException( + "cannot change the type of column ${action.columnName} because it is used by one of ${tableName}'s own " + + "FOREIGN KEY constraints; drop the constraint first" + ) + } + val referencingTables = catalog.tableNames.filter { other -> + !other.equals(tableName, ignoreCase = true) && + catalog.tableOrNull(other)!!.schema.foreignKeys.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.any { it.equals(action.columnName, ignoreCase = true) } + } + } + if (referencingTables.isNotEmpty()) { + throw KestrelException( + "cannot change the type of column ${action.columnName} because table(s) ${referencingTables.joinToString(", ")} " + + "have a FOREIGN KEY referencing it; drop the constraint first" + ) + } + if (schema.uniqueConstraints.any { unique -> unique.columns.any { it.equals(action.columnName, ignoreCase = true) } }) { + throw KestrelException( + "cannot change the type of column ${action.columnName} because it is used by a UNIQUE constraint of $tableName; " + + "drop the constraint first" + ) + } + val dependentIndexes = catalog.indexesOn(tableName).filter { index -> + index.definition.columnNames.any { it.equals(action.columnName, ignoreCase = true) } + } + if (dependentIndexes.isNotEmpty()) { + throw KestrelException( + "cannot change the type of column ${action.columnName} because index(es) " + + "${dependentIndexes.joinToString(", ") { it.definition.name }} are defined on it; drop the index first" + ) + } + + val newColumn = column.copy(type = action.newType, precision = action.precision, scale = action.scale) + newColumn.default?.let { default -> + try { + coerceForColumn(Evaluator().evaluate(default), newColumn) + } catch (e: KestrelException) { + throw KestrelException( + "cannot change the type of column ${action.columnName}: its DEFAULT expression is not compatible " + + "with ${action.newType.displayWith(action.precision, action.scale)} (${e.message})" + ) + } + } + val newSchema = TableSchema( + schema.name, schema.columns.toMutableList().also { it[columnIndex] = newColumn }, schema.foreignKeys, + schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey, schema.temporary + ) + + // Every row's new value is computed and validated against newSchema BEFORE any of them is + // handed to the catalog, so a mid-batch NOT NULL/CHECK failure never leaves the table + // schema pointing at the new type while some rows still hold values coerced for the old + // one - either every row passes and the whole table is replaced atomically, or none of it + // is (see Catalog.changeColumnType's own doc on this contract). + val using = action.using ?: ColumnExpression(action.columnName) + val newRows = catalog.table(tableName).rows.map { row -> + val values = row.values.toMutableList() + values[columnIndex] = coerceForColumn(Evaluator(schema, row, subqueryRunner).evaluate(using), newColumn) + val newRow = Row(values) + validateNotNull(newSchema, newRow.values) + enforceChecks(newSchema, newRow) + newRow + } + catalog.changeColumnType(tableName, action.columnName, newColumn, newRows) + } + + /** A constraint name must be unique among every named CHECK/UNIQUE/FOREIGN KEY already on [schema]'s table (case-insensitive) - mirrors Postgres, which keeps one namespace of constraint names per table. */ + private fun requireUniqueConstraintName(schema: TableSchema, name: String) { + val taken = schema.foreignKeys.any { it.name?.equals(name, ignoreCase = true) == true } || + schema.checks.any { it.name?.equals(name, ignoreCase = true) == true } || + schema.uniqueConstraints.any { it.name?.equals(name, ignoreCase = true) == true } + if (taken) throw KestrelException("constraint '$name' already exists on table ${schema.name}") + } + + /** + * A NOT NULL column added to a table that already has rows must carry a DEFAULT (there is no + * value to backfill existing rows with otherwise) - an empty table has no such requirement, + * matching the same rule this engine already applies at the row level via [validateNotNull]. + * GENERATED ALWAYS AS sidesteps that requirement entirely (see [addGeneratedColumnValue]): it + * backfills by evaluating the expression fresh against each row's own other columns, so a NOT + * NULL GENERATED column can be added to a non-empty table with no DEFAULT at all - it only fails, + * per row, if that row's own expression happens to evaluate to NULL. + */ + private fun executeAddColumn(tableName: String, column: Column) { + val table = catalog.table(tableName) + if (column.generated != null) { + validateAddGeneratedColumn(table.schema, column) + catalog.addColumn(tableName, column) { row -> addGeneratedColumnValue(table.schema, column, row) } + return + } + column.default?.let { validateConstantDefault(it) } + // Unlike CREATE TABLE, ADD COLUMN backfills every existing row with ONE already-evaluated + // Value (Catalog.addColumn's contract - see its own doc), not a fresh per-row expression + // evaluation. A NEXTVAL DEFAULT would therefore hand every pre-existing row the exact same + // value instead of a growing sequence, silently defeating the one thing NEXTVAL is for - + // rejected outright rather than allowed to produce that surprising result. INSERT's DEFAULT + // has no such problem (see buildInsertRow, which evaluates fresh per row). + if (column.default is NextValueExpression) { + throw KestrelException( + "NEXTVAL cannot be used as an ADD COLUMN DEFAULT - it would give every existing row " + + "the same value; CREATE TABLE's DEFAULT evaluates fresh per row instead" + ) + } + if (column.notNull && column.default == null && table.rows.isNotEmpty()) { + throw KestrelException( + "cannot add NOT NULL column ${column.name} to table $tableName with existing rows without a DEFAULT value" + ) + } + val defaultValue = column.default?.let { coerceForColumn(Evaluator().evaluate(it), column) } ?: Value.NullValue + catalog.addColumn(tableName, column) { defaultValue } + } + + /** + * Validates a GENERATED ALWAYS AS expression being added via ALTER TABLE ADD COLUMN against the + * table's CURRENT [schema] (before the new column exists) - the same rules + * [validateGeneratedDefinitions] applies at CREATE TABLE time (no unknown column, no aggregate/ + * window function, no subquery, no reference to another GENERATED column), just checked one + * column at a time against an already-existing schema instead of a freshly-declared column list. + * A self-reference is rejected for free: the new column is not part of [schema] yet, so + * [checkColumns] already reports it as "no such column". + */ + private fun validateAddGeneratedColumn(schema: TableSchema, column: Column) { + val generated = column.generated!! + checkColumns(generated, schema) + requireNoSubquery(generated) + val generatedNames = schema.columns.filter { it.generated != null }.map { it.name.lowercase() }.toSet() + requireNoGeneratedColumnReference(generated, generatedNames, column.name) + } + + /** + * Backfills GENERATED ALWAYS AS for one pre-existing [row], evaluated against the table's OLD + * [schema] (the new column is not part of it yet, matching what [validateAddGeneratedColumn] + * already checked against). Enforces NOT NULL the same way INSERT/UPDATE do post- + * [applyGeneratedColumns] via [validateNotNull] - inlined here instead, since there is no + * full new-shape row to hand that function until after this value is appended. + */ + private fun addGeneratedColumnValue(schema: TableSchema, column: Column, row: Row): Value { + val value = coerceForColumn(Evaluator(schema, row).evaluate(column.generated!!), column) + if (column.notNull && value is Value.NullValue) { + throw KestrelException("NOT NULL constraint failed: ${schema.name}.${column.name}") + } + return value + } + private fun executeCreateIndex(statement: CreateIndexStatement): StatementResult { - catalog.createIndex(IndexDefinition(statement.indexName, statement.tableName, statement.columnName)) + catalog.createIndex( + IndexDefinition(statement.indexName, statement.tableName, statement.columnNames, statement.unique) + ) return CreateIndexResult(statement.indexName) } @@ -99,29 +955,577 @@ class Executor(private val catalog: Catalog) { return DropIndexResult(statement.indexName) } - private fun executeInsert(statement: InsertStatement): StatementResult { + private fun executeCreateView(statement: CreateViewStatement): StatementResult { + selectRunner.explain(statement.query) + catalog.createView(statement.viewName, statement.query, statement.orReplace) + return CreateViewResult(statement.viewName) + } + + private fun executeDropView(statement: DropViewStatement): StatementResult { + catalog.dropView(statement.viewName, cascade = statement.mode == DropMode.Cascade) + return DropViewResult(statement.viewName) + } + + /** + * Body statements are deliberately NOT deep-validated here (no eager check that every column + * referenced in a body statement's WHERE/SET/VALUES actually exists) - unlike CREATE VIEW's + * eager `selectRunner.explain`, a trigger body can target a different table per statement and + * SQLite itself defers exactly this kind of check to the moment the trigger actually fires. + * What IS checked eagerly, matching this codebase's usual "catch it before it's the user's + * runtime problem" bar for constraint-shaped input: the trigger's own table and every body + * statement's target table both exist right now, and the WHEN clause - evaluated directly by + * this method rather than through a nested statement executor, so nothing else will ever + * validate it - contains no subquery/aggregate and only ever references NEW/OLD columns that + * both exist on this table and are actually available for this trigger's event (NEW is not + * available on DELETE, OLD is not available on INSERT). + */ + private fun executeCreateTrigger(statement: CreateTriggerStatement): StatementResult { + val schema = catalog.table(statement.tableName).schema + statement.whenClause?.let { whenClause -> + requireNoSubquery(whenClause) + validateTriggerRowReferences(whenClause, statement.event, schema) + } + for (bodyStatement in statement.body) { + validateTriggerBodyStatementTarget(bodyStatement) + } + catalog.createTrigger( + TriggerDefinition(statement.triggerName, statement.timing, statement.event, statement.tableName, statement.whenClause, statement.body) + ) + return CreateTriggerResult(statement.triggerName) + } + + private fun validateTriggerBodyStatementTarget(statement: Statement) { + when (statement) { + is InsertStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + is UpdateStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + is DeleteStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + else -> throw KestrelException("trigger body only supports INSERT, UPDATE, or DELETE statements") + } + } + + /** A trigger body statement's own [runTriggerBody] discards whatever [StatementResult] it returns, so a RETURNING clause there would compute output nobody could ever see - rejected eagerly, matching [validateTriggerRowReferences]'s "structurally without sense" bar. */ + private fun rejectReturning(returning: Projection?) { + if (returning != null) { + throw KestrelException("RETURNING is not allowed in a trigger body statement - its result would never be visible to anything") + } + } + + private fun validateTriggerRowReferences(expression: Expression, event: TriggerEvent, schema: TableSchema) { + when (expression) { + is ColumnExpression -> { + val qualifier = expression.qualifier + ?: throw KestrelException("column '${expression.name}' in trigger WHEN clause must be qualified with NEW or OLD") + val isNew = qualifier.equals("new", ignoreCase = true) + val isOld = qualifier.equals("old", ignoreCase = true) + if (!isNew && !isOld) throw KestrelException("no such table or alias: $qualifier") + val availableForEvent = when (event) { + TriggerEvent.Insert -> isNew + TriggerEvent.Delete -> isOld + TriggerEvent.Update -> true + } + if (!availableForEvent) { + throw KestrelException("${if (isNew) "NEW" else "OLD"} is not available in a trigger ON ${event.name.uppercase()}") + } + if (schema.columnIndex(expression.name) == null) { + throw KestrelException("no such column: ${expression.name}") + } + } + is BinaryExpression -> { + validateTriggerRowReferences(expression.left, event, schema) + validateTriggerRowReferences(expression.right, event, schema) + } + is UnaryExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is CallExpression -> expression.arguments.forEach { validateTriggerRowReferences(it, event, schema) } + is IsNullExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is InExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + expression.values.forEach { validateTriggerRowReferences(it, event, schema) } + } + is BetweenExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + validateTriggerRowReferences(expression.low, event, schema) + validateTriggerRowReferences(expression.high, event, schema) + } + is LikeExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + validateTriggerRowReferences(expression.pattern, event, schema) + } + is CaseExpression -> { + expression.operand?.let { validateTriggerRowReferences(it, event, schema) } + expression.branches.forEach { + validateTriggerRowReferences(it.condition, event, schema) + validateTriggerRowReferences(it.result, event, schema) + } + expression.elseExpression?.let { validateTriggerRowReferences(it, event, schema) } + } + is CastExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is CollateExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is LiteralExpression -> Unit + is FunctionExpression, is WindowExpression, is ScalarSubqueryExpression, + is ExistsExpression, is InSubqueryExpression, + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> + error("unreachable: banned by requireNoSubquery") + } + } + + private fun executeDropTrigger(statement: DropTriggerStatement): StatementResult { + catalog.dropTrigger(statement.triggerName) + return DropTriggerResult(statement.triggerName) + } + + /** + * Resolves the parser's "not written" `null`s into Postgres' own ascending/descending + * defaults - MINVALUE 1/MAXVALUE `Long.MAX_VALUE`/START WITH MINVALUE for a positive + * INCREMENT BY (1 if not given either), the mirror image for a negative one - then validates + * the combination as a whole via [SequenceDefinition]'s own `init` plus the one check it can't + * make itself: START WITH must actually fall within [MINVALUE, MAXVALUE]. + */ + private fun executeCreateSequence(statement: CreateSequenceStatement): StatementResult { + val incrementBy = statement.incrementBy ?: 1L + if (incrementBy == 0L) throw KestrelException("sequence '${statement.sequenceName}' INCREMENT BY cannot be zero") + val ascending = incrementBy > 0 + val minValue = statement.minValue ?: if (ascending) 1L else Long.MIN_VALUE + val maxValue = statement.maxValue ?: if (ascending) Long.MAX_VALUE else -1L + if (minValue > maxValue) { + throw KestrelException("sequence '${statement.sequenceName}' MINVALUE ($minValue) must not exceed MAXVALUE ($maxValue)") + } + val startValue = statement.startValue ?: if (ascending) minValue else maxValue + if (startValue < minValue || startValue > maxValue) { + throw KestrelException( + "START WITH ($startValue) must be between MINVALUE ($minValue) and MAXVALUE ($maxValue)" + ) + } + catalog.createSequence(SequenceDefinition(statement.sequenceName, incrementBy, minValue, maxValue, startValue, statement.cycle)) + return CreateSequenceResult(statement.sequenceName) + } + + private fun executeDropSequence(statement: DropSequenceStatement): StatementResult { + catalog.dropSequence(statement.sequenceName) + return DropSequenceResult(statement.sequenceName) + } + + /** + * Unlike [executeCreateSequence], an option left `null`/`false` here means "keep whatever this + * sequence already has" rather than "apply a default" - only [AlterSequenceStatement.clearMinValue]/ + * [AlterSequenceStatement.clearMaxValue] (Postgres' `NO MINVALUE`/`NO MAXVALUE`) recompute a bound + * from [SequenceDefinition.incrementBy]'s sign the way a fresh CREATE SEQUENCE would. + * [SequenceDefinition.startValue] is never touched by this method (see its own doc) regardless + * of what RESTART is asked for. + */ + private fun executeAlterSequence(statement: AlterSequenceStatement): StatementResult { + val existing = catalog.sequence(statement.sequenceName).definition + val incrementBy = statement.incrementBy ?: existing.incrementBy + if (incrementBy == 0L) throw KestrelException("sequence '${statement.sequenceName}' INCREMENT BY cannot be zero") + val ascending = incrementBy > 0 + val minValue = when { + statement.clearMinValue -> if (ascending) 1L else Long.MIN_VALUE + statement.minValue != null -> statement.minValue + else -> existing.minValue + } + val maxValue = when { + statement.clearMaxValue -> if (ascending) Long.MAX_VALUE else -1L + statement.maxValue != null -> statement.maxValue + else -> existing.maxValue + } + if (minValue > maxValue) { + throw KestrelException("sequence '${statement.sequenceName}' MINVALUE ($minValue) must not exceed MAXVALUE ($maxValue)") + } + val cycle = statement.cycle ?: existing.cycle + val newDefinition = SequenceDefinition(statement.sequenceName, incrementBy, minValue, maxValue, existing.startValue, cycle) + val restartAt = if (statement.restart) (statement.restartWith ?: existing.startValue) else null + if (restartAt != null && (restartAt < minValue || restartAt > maxValue)) { + throw KestrelException("RESTART value ($restartAt) must be between MINVALUE ($minValue) and MAXVALUE ($maxValue)") + } + catalog.alterSequence(statement.sequenceName, newDefinition, restartAt) + return AlterSequenceResult(statement.sequenceName) + } + + /** + * Fires every trigger registered for ([schema], [timing], [event]) in creation order, once per + * (oldRow, newRow) pair in [rows] - INSERT passes `null` for oldRow, DELETE passes `null` for + * newRow, UPDATE passes both. A `WHEN` clause that evaluates to false or NULL skips only that + * trigger for that row, exactly like a plain WHERE clause would; it never skips other triggers + * on the same event, and never affects the firing statement itself. + */ + private fun fireRowTriggers(schema: TableSchema, timing: TriggerTiming, event: TriggerEvent, rows: List>) { + val triggers = catalog.triggersOn(schema.name, timing, event) + if (triggers.isEmpty()) return + for ((oldRow, newRow) in rows) { + val context = TriggerRowContext(schema, newRow, oldRow) + for (trigger in triggers) { + val whenClause = trigger.whenClause + if (whenClause != null) { + val evaluator = Evaluator(composeResolver(null, context), subqueryRunner) + if (!isTrue(evaluator.evaluate(whenClause))) continue + } + runTriggerBody(trigger, context) + } + } + } + + private fun runTriggerBody(trigger: TriggerDefinition, context: TriggerRowContext) { + if (triggerDepth >= MAX_TRIGGER_DEPTH) { + throw KestrelException( + "trigger recursion limit ($MAX_TRIGGER_DEPTH) exceeded while firing '${trigger.name}' - " + + "check for a trigger whose body ends up re-triggering itself" + ) + } + triggerDepth++ + try { + for (bodyStatement in trigger.body) { + when (bodyStatement) { + is InsertStatement -> executeInsert(bodyStatement, context) + is UpdateStatement -> executeUpdate(bodyStatement, context) + is DeleteStatement -> executeDelete(bodyStatement, context) + else -> error("unreachable: trigger bodies only ever contain INSERT/UPDATE/DELETE") + } + } + } finally { + triggerDepth-- + } + } + + /** + * Resolves `NEW.col`/`OLD.col` against [trigger] first (when both the qualifier and that side + * of the row are present); everything else - including a bare column with no qualifier at + * all - falls through to [base] exactly as it would with no trigger in play, so passing + * `trigger = null` (or a [TriggerRowContext] that never matches) reproduces today's behavior + * without change. + */ + private fun composeResolver(base: ColumnResolver?, trigger: TriggerRowContext?): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value = + if (trigger != null && trigger.matches(expression)) trigger.resolve(expression) + else base?.resolve(expression) ?: throw KestrelException("column '${expression.name}' is not allowed here") + + override fun declaredCollation(expression: ColumnExpression): Collation? = + if (trigger != null && trigger.matches(expression)) trigger.declaredCollation(expression) + else base?.declaredCollation(expression) + } + + /** Reproduces the (schema, row) convenience constructor on [Evaluator], as a [ColumnResolver] that [composeResolver] can layer a [TriggerRowContext] on top of. */ + private fun rowResolver(schema: TableSchema, row: Row): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val index = schema.columnIndex(expression.name) + ?: throw KestrelException("no such column: ${expression.name}") + return row.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = schema.column(expression.name)?.collation + } + + /** + * Evaluates an INSERT/UPDATE/DELETE statement's `RETURNING` clause against the rows it actually + * affected. [rows] is always the image RETURNING should see for that statement kind: the freshly + * inserted or updated row for INSERT/UPDATE (there is nothing else to show), the pre-deletion row + * for DELETE (it no longer exists in the table afterward). Column references resolve against + * [schema] alone - never NEW/OLD, since [rejectReturning] already keeps RETURNING out of every + * trigger body statement, so this is always the outermost, non-nested call. [checkColumns] runs + * over every projected expression up front (not just lazily via [rowResolver] while evaluating + * row by row) so a bad column name or a stray aggregate is reported even when zero rows matched - + * matching how every other schema-shaped clause in this engine is validated regardless of data. + */ + private fun evaluateReturning(schema: TableSchema, rows: List, projection: Projection): SelectResult { + val columns = when (projection) { + is StarProjection -> schema.columns.map { it.name } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> + checkColumns(item.expression, schema) + returningColumnLabel(item, index) + } + } + val resultRows = rows.map { row -> + when (projection) { + is StarProjection -> row.values.toList() + is ExpressionProjection -> { + val evaluator = Evaluator(rowResolver(schema, row), subqueryRunner) + projection.items.map { evaluator.evaluate(it.expression) } + } + } + } + return SelectResult(columns, resultRows) + } + + private fun returningColumnLabel(item: SelectItem, index: Int): String = + item.alias ?: when (val expression = item.expression) { + is ColumnExpression -> expression.name + is FunctionExpression -> expression.name.lowercase() + is CallExpression -> expression.name.lowercase() + is WindowExpression -> expression.name.lowercase() + else -> "column${index + 1}" + } + + /** + * Without this, a BEFORE trigger's own INSERT/UPDATE/DELETE would survive even when the + * statement that fired it goes on to fail its own constraint check (or an AFTER trigger fails + * after the row already landed) - a real database treats a whole statement plus everything its + * triggers do as one atomic unit, and this reuses the exact snapshot/restore machinery + * [executeRollback]/[executeRollbackToSavepoint] already rely on to do the same. Only the + * OUTERMOST statement in a trigger cascade takes the snapshot ([triggerContext] is null exactly + * there): [catalog.snapshot] captures the WHOLE catalog, not just [tableName], so restoring it + * once at the top undoes every table any nested trigger body touched, not only this one - a + * nested call skips wrapping itself again, and skips it entirely when [tableName] has no + * trigger at all, so the overwhelmingly common trigger-free INSERT/UPDATE/DELETE pays no + * snapshot cost. + */ + private fun withStatementAtomicity(tableName: String, triggerContext: TriggerRowContext?, block: () -> T): T { + if (triggerContext != null || !catalog.hasTriggers(tableName)) return block() + val snapshot = catalog.snapshot() + try { + return block() + } catch (exception: Throwable) { + catalog.restore(snapshot) + throw exception + } + } + + private companion object { + const val MAX_TRIGGER_DEPTH = 100 + } + + private fun executeInsert(statement: InsertStatement, triggerContext: TriggerRowContext? = null): StatementResult = + withStatementAtomicity(statement.tableName, triggerContext) { executeInsertInner(statement, triggerContext) } + + private fun executeInsertInner(statement: InsertStatement, triggerContext: TriggerRowContext?): StatementResult { val table = catalog.table(statement.tableName) val schema = table.schema val targetIndices = resolveInsertTargets(schema, statement.columns) - val evaluator = Evaluator() - val newRows = statement.rows.map { rawRow -> - if (rawRow.size != targetIndices.size) { - throw KestrelException( - "table ${schema.name} expects ${targetIndices.size} values but ${rawRow.size} were supplied" - ) + val evaluator = Evaluator(composeResolver(null, triggerContext), subqueryRunner, this) + val newRows = statement.rows.map { rawRow -> buildInsertRow(schema, targetIndices, rawRow, evaluator) } + val onConflict = statement.onConflict + if (onConflict == null) { + // Fires before any constraint is even checked, matching SQLite/Postgres: a BEFORE + // trigger can reject the whole statement (by having its body throw) before this + // engine's own NOT NULL/CHECK/FOREIGN KEY/UNIQUE validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, newRows.map { null to it }) + // Validated as a whole batch (not inline while building newRows above) so that a + // self-referencing FK can see sibling rows from the same multi-row INSERT regardless + // of their order, exactly like enforcePrimaryKeyUniqueness below already does. + for (row in newRows) { + ForeignKeyEnforcer.validateRow(catalog, schema, row.values, uncommittedSiblings = newRows) + enforceChecks(schema, row) } - val values = MutableList(schema.columns.size) { Value.NullValue } - for ((position, expression) in rawRow.withIndex()) { - val columnIndex = targetIndices[position] - values[columnIndex] = coerceForColumn(evaluator.evaluate(expression), schema.columns[columnIndex]) - } - validateNotNull(schema, values) - Row(values) + enforcePrimaryKeyUniqueness(schema, table.rows + newRows) + enforceUniqueConstraints(schema, table.rows + newRows) + enforceUniqueIndexes(schema, table.rows + newRows) + table.rows.addAll(newRows) + catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, newRows.map { null to it }) + val returning = statement.returning?.let { evaluateReturning(schema, newRows, it) } + return InsertResult(newRows.size, returning) } - enforcePrimaryKeyUniqueness(schema, table.rows + newRows) - table.rows.addAll(newRows) - catalog.reindexTable(schema.name) - return InsertResult(newRows.size) + return executeInsertWithConflictHandling(schema, table, newRows, onConflict, statement.returning, triggerContext) + } + + private fun buildInsertRow( + schema: TableSchema, + targetIndices: List, + rawRow: List, + evaluator: Evaluator + ): Row { + if (rawRow.size != targetIndices.size) { + throw KestrelException( + "table ${schema.name} expects ${targetIndices.size} values but ${rawRow.size} were supplied" + ) + } + val values = MutableList(schema.columns.size) { Value.NullValue } + val touched = BooleanArray(schema.columns.size) + for ((position, expression) in rawRow.withIndex()) { + val columnIndex = targetIndices[position] + // A literal DEFAULT in the VALUES list is left untouched here, exactly like a + // column missing from an explicit INSERT column list - both fall through to the + // same default-filling loop below, matching UPDATE ... SET col = DEFAULT semantics. + if (expression != null) { + rejectGeneratedAssignment(schema.columns[columnIndex], explicitValueGiven = true) + values[columnIndex] = coerceForColumn(evaluator.evaluate(expression), schema.columns[columnIndex]) + touched[columnIndex] = true + } + } + // A column omitted from an explicit INSERT column list falls back to its DEFAULT + // (evaluated fresh per row); one that was supplied - even an explicit NULL - never + // does, matching standard SQL DEFAULT semantics. + for ((index, column) in schema.columns.withIndex()) { + val default = column.default + if (!touched[index] && default != null) { + values[index] = coerceForColumn(evaluator.evaluate(default), column) + } + } + applyGeneratedColumns(schema, values) + validateNotNull(schema, values) + return Row(values) + } + + private class UniqueKey(val columnIndices: List, val columnNames: List) + + private fun uniqueKeysOf(schema: TableSchema): List { + val keys = mutableListOf() + val pkIndices = schema.primaryKeyIndices + if (pkIndices.isNotEmpty()) keys.add(UniqueKey(pkIndices, pkIndices.map { schema.columns[it].name })) + for (unique in schema.uniqueConstraints) { + val indices = unique.columns.map { schema.columnIndex(it)!! } + keys.add(UniqueKey(indices, unique.columns)) + } + for (definition in catalog.indexDefinitions()) { + if (!definition.unique || !definition.tableName.equals(schema.name, ignoreCase = true)) continue + val indices = definition.columnNames.mapNotNull { schema.columnIndex(it) } + if (indices.size != definition.columnNames.size) continue + keys.add(UniqueKey(indices, definition.columnNames)) + } + return keys + } + + /** Empty [targetColumns] (bare `ON CONFLICT DO ...`) matches against any of [keys]. */ + private fun resolveConflictTargetKeys(schema: TableSchema, targetColumns: List, keys: List): List { + if (targetColumns.isEmpty()) return keys + val targetIndices = targetColumns.map { name -> + schema.columnIndex(name) ?: throw KestrelException("no such column: $name") + } + val targetSet = targetIndices.toSet() + val matched = keys.firstOrNull { it.columnIndices.toSet() == targetSet } + ?: throw KestrelException( + "no unique or primary key constraint matching ON CONFLICT specification: " + + targetColumns.joinToString(", ") + ) + return listOf(matched) + } + + /** NULLs never collide with each other (standard SQL), mirroring enforceUniqueConstraints/enforceUniqueIndexes above. */ + private fun findConflictingRow(existingRows: List, candidate: List, keys: List): Row? { + for (existing in existingRows) { + for (key in keys) { + val candidateKey = key.columnIndices.map { candidate[it] } + if (candidateKey.any { it is Value.NullValue }) continue + val existingKey = key.columnIndices.map { existing.values[it] } + if (candidateKey == existingKey) return existing + } + } + return null + } + + /** + * Processes each candidate row as its own micro-insert against the table's CURRENT state + * (rather than batch-validating the whole VALUES list like the no-conflict path above), so + * a later row in the same multi-row INSERT correctly sees an earlier row's insert or update - + * exactly how Postgres/SQLite define ON CONFLICT for multi-row VALUES. A row landing with no + * conflict fires an INSERT trigger pair, exactly like the no-conflict path in [executeInsertInner]; + * a row resolved by DO UPDATE fires an UPDATE trigger pair instead, matching that the row's own + * data ends up changed by an UPDATE rather than a fresh INSERT (mirroring Postgres: the proposed + * row that lost the race is folded into an update of the pre-existing one). DO NOTHING fires no + * trigger at all, since nothing about the table actually changed for that row. When [schema] has + * at least one trigger, [catalog.reindexTable] runs right after each row's own mutation - before + * that row's AFTER trigger, rather than once for the whole statement like the batch paths do - so a + * trigger body's own query against this table never observes a stale index for a row this very + * statement already landed; a trigger-free table keeps the original single-reindex-at-the-end cost. + */ + private fun executeInsertWithConflictHandling( + schema: TableSchema, + table: Table, + candidateRows: List, + onConflict: OnConflictClause, + returning: Projection? = null, + triggerContext: TriggerRowContext? = null + ): StatementResult { + val keys = uniqueKeysOf(schema) + val targetKeys = resolveConflictTargetKeys(schema, onConflict.targetColumns, keys) + // Reindexing per row (rather than once for the whole statement like the batch INSERT/UPDATE + // paths) is only needed so a trigger body's own query against this table never sees a stale + // index for a row this very statement already landed - skipped entirely, preserving the + // original single-reindex-at-the-end cost, when this table has no trigger to fire at all. + val hasTriggers = catalog.hasTriggers(schema.name) + var affected = 0 + // DO NOTHING contributes no row (nothing about the table changed for it); a DO UPDATE row + // whose WHERE guard fails `continue`s before ever reaching the append below, same as DO + // NOTHING - both are absent from RETURNING exactly like Postgres. + val returnedRows = mutableListOf() + for (row in candidateRows) { + val conflicting = findConflictingRow(table.rows, row.values, targetKeys) + if (conflicting == null) { + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, listOf(null to row)) + ForeignKeyEnforcer.validateRow(catalog, schema, row.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, row) + enforcePrimaryKeyUniqueness(schema, table.rows + row) + enforceUniqueConstraints(schema, table.rows + row) + enforceUniqueIndexes(schema, table.rows + row) + table.rows.add(row) + affected++ + returnedRows.add(row) + if (hasTriggers) catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, listOf(null to row)) + continue + } + when (val action = onConflict.action) { + ConflictDoNothing -> Unit + is ConflictDoUpdate -> { + val resolver = composeResolver(excludedAwareResolver(schema, conflicting, row), triggerContext) + // NEXTVAL/CURRVAL are deliberately usable in the assignment values below but not + // in this WHERE-shaped guard - same "not in WHERE" rule as everywhere else, kept + // airtight here by using a second Evaluator instance rather than one shared one. + val conditionEvaluator = Evaluator(resolver, subqueryRunner) + if (action.where != null && !isTrue(conditionEvaluator.evaluate(action.where))) continue + val assignmentEvaluator = Evaluator(resolver, subqueryRunner, this) + val values = conflicting.values.toMutableList() + for (assignment in action.assignments) { + val index = schema.columnIndex(assignment.columnName) + ?: throw KestrelException("no such column: ${assignment.columnName}") + val column = schema.columns[index] + rejectGeneratedAssignment(column, explicitValueGiven = assignment.value != null) + values[index] = if (assignment.value != null) { + coerceForColumn(assignmentEvaluator.evaluate(assignment.value), column) + } else { + column.default?.let { coerceForColumn(assignmentEvaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values) + validateNotNull(schema, values) + val updated = Row(values) + // Fires before any constraint below is checked, same as executeUpdate's own BEFORE + // trigger - a BEFORE UPDATE trigger's body can reject this row's update before this + // engine's own validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, listOf(conflicting to updated)) + ForeignKeyEnforcer.validateRow(catalog, schema, updated.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, updated) + val others = table.rows.filter { it !== conflicting } + enforcePrimaryKeyUniqueness(schema, others + updated) + enforceUniqueConstraints(schema, others + updated) + enforceUniqueIndexes(schema, others + updated) + // Same as executeUpdate: any table with a FK pointing at one of this table's + // enforced-unique key groups needs a cascade/RESTRICT pass when that group's + // value actually changes, not just the PRIMARY KEY. + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val oldKey = columnIndices.map { conflicting.values[it] } + val newKey = columnIndices.map { updated.values[it] } + ForeignKeyEnforcer.cascadeUpdate(catalog, schema.name, group, listOf(oldKey to newKey)) + } + val position = table.rows.indexOfFirst { it === conflicting } + table.rows[position] = updated + affected++ + returnedRows.add(updated) + if (hasTriggers) catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, listOf(conflicting to updated)) + } + } + } + if (!hasTriggers) catalog.reindexTable(schema.name) + return InsertResult(affected, returning?.let { evaluateReturning(schema, returnedRows, it) }) + } + + /** + * Bare column names resolve against [target] (the existing conflicting row, matching plain + * UPDATE semantics), while `EXCLUDED.column` resolves against [excluded] (the row proposed by + * this INSERT that failed to land) - the same pseudo-table Postgres exposes in DO UPDATE. + */ + private fun excludedAwareResolver(schema: TableSchema, target: Row, excluded: Row): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + val useExcluded = qualifier != null && qualifier.equals("excluded", ignoreCase = true) + if (qualifier != null && !useExcluded && !qualifier.equals(schema.name, ignoreCase = true)) { + throw KestrelException("no such table or alias: $qualifier") + } + val index = schema.columnIndex(expression.name) + ?: throw KestrelException("no such column: ${expression.name}") + return if (useExcluded) excluded.values[index] else target.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = schema.column(expression.name)?.collation } private fun resolveInsertTargets(schema: TableSchema, columns: List?): List { @@ -132,60 +1536,501 @@ class Executor(private val catalog: Catalog) { } } - private fun executeDelete(statement: DeleteStatement): StatementResult { + private fun executeDelete(statement: DeleteStatement, triggerContext: TriggerRowContext? = null): StatementResult = + withStatementAtomicity(statement.tableName, triggerContext) { executeDeleteInner(statement, triggerContext) } + + private fun executeDeleteInner(statement: DeleteStatement, triggerContext: TriggerRowContext?): StatementResult { val table = catalog.table(statement.tableName) val schema = table.schema - if (statement.where == null) { - val removed = table.rows.size - table.rows.clear() - catalog.reindexTable(schema.name) - return DeleteResult(removed) - } - checkColumns(statement.where, schema) - val kept = mutableListOf() - var deleted = 0 - for (row in table.rows) { - if (isTrue(Evaluator(schema, row).evaluate(statement.where))) { - deleted++ - } else { - kept.add(row) + val deletedRows = if (statement.where == null) { + table.rows.toList() + } else { + checkColumns(statement.where, schema, triggerContext) + table.rows.filter { + isTrue(Evaluator(composeResolver(rowResolver(schema, it), triggerContext), subqueryRunner).evaluate(statement.where)) } } - table.rows.clear() - table.rows.addAll(kept) + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Delete, deletedRows.map { it to null }) + ForeignKeyEnforcer.cascadeDelete(catalog, schema.name, deletedRows) + table.rows.removeAll(deletedRows) catalog.reindexTable(schema.name) - return DeleteResult(deleted) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Delete, deletedRows.map { it to null }) + val returning = statement.returning?.let { evaluateReturning(schema, deletedRows, it) } + return DeleteResult(deletedRows.size, returning) } - private fun executeUpdate(statement: UpdateStatement): StatementResult { + private fun executeUpdate(statement: UpdateStatement, triggerContext: TriggerRowContext? = null): StatementResult = + withStatementAtomicity(statement.tableName, triggerContext) { executeUpdateInner(statement, triggerContext) } + + private fun executeUpdateInner(statement: UpdateStatement, triggerContext: TriggerRowContext?): StatementResult { val table = catalog.table(statement.tableName) val schema = table.schema val assignments = statement.assignments.map { assignment -> val index = schema.columnIndex(assignment.columnName) ?: throw KestrelException("no such column: ${assignment.columnName}") - checkColumns(assignment.value, schema) + assignment.value?.let { checkColumns(it, schema, triggerContext) } + rejectGeneratedAssignment(schema.columns[index], explicitValueGiven = assignment.value != null) index to assignment.value } - statement.where?.let { checkColumns(it, schema) } + statement.where?.let { checkColumns(it, schema, triggerContext) } var updated = 0 - val newRows = table.rows.map { row -> - val matches = statement.where == null || isTrue(Evaluator(schema, row).evaluate(statement.where)) + val oldRows = table.rows.toList() + // Every row actually matched by WHERE, paired old-to-new - fires an UPDATE trigger for + // every row WHERE selects, regardless of whether any assigned value actually differs from + // what was already there (matching standard SQL: WHERE selects the affected rows, not the + // changed ones). + val touchedRows = mutableListOf>() + val newRows = oldRows.map { row -> + val matches = statement.where == null || + isTrue(Evaluator(composeResolver(rowResolver(schema, row), triggerContext), subqueryRunner).evaluate(statement.where)) if (!matches) return@map row val values = row.values.toMutableList() - val evaluator = Evaluator(schema, row) + // NEXTVAL/CURRVAL are allowed in SET's assignment values (below), not in WHERE (above) - + // a separate Evaluator per clause, same reasoning as executeInsertWithConflictHandling. + val evaluator = Evaluator(composeResolver(rowResolver(schema, row), triggerContext), subqueryRunner, this) for ((index, expression) in assignments) { - values[index] = coerceForColumn(evaluator.evaluate(expression), schema.columns[index]) + val column = schema.columns[index] + values[index] = if (expression != null) { + coerceForColumn(evaluator.evaluate(expression), column) + } else { + column.default?.let { coerceForColumn(evaluator.evaluate(it), column) } ?: Value.NullValue + } } + applyGeneratedColumns(schema, values) validateNotNull(schema, values) updated++ - Row(values) + val newRow = Row(values) + touchedRows.add(row to newRow) + newRow + } + // Fires before any constraint below is checked, same as executeInsert's BEFORE trigger - + // a BEFORE UPDATE trigger's body can reject the whole statement before this engine's own + // validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, touchedRows) + // Same whole-batch validation as executeInsert: a self-referencing FK must see every row's + // final value for this statement, not just rows already committed to the table. + for (row in newRows) { + ForeignKeyEnforcer.validateRow(catalog, schema, row.values, uncommittedSiblings = newRows) + enforceChecks(schema, row) } enforcePrimaryKeyUniqueness(schema, newRows) + enforceUniqueConstraints(schema, newRows) + enforceUniqueIndexes(schema, newRows) + // Every enforced-unique key group (the PRIMARY KEY, single-column or composite, plus each + // UNIQUE constraint) is a valid FOREIGN KEY target, so each group needs its own + // cascade/RESTRICT pass on UPDATE - not just the PRIMARY KEY. + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val changedPairs = oldRows.indices.map { i -> + columnIndices.map { oldRows[i].values[it] } to columnIndices.map { newRows[i].values[it] } + } + ForeignKeyEnforcer.cascadeUpdate(catalog, schema.name, group, changedPairs) + } table.rows.clear() table.rows.addAll(newRows) catalog.reindexTable(schema.name) - return UpdateResult(updated) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, touchedRows) + val returning = statement.returning?.let { evaluateReturning(schema, touchedRows.map { pair -> pair.second }, it) } + return UpdateResult(updated, returning) + } + + private fun executeMerge(statement: MergeStatement): StatementResult = + withStatementAtomicity(statement.targetTable, null) { executeMergeInner(statement) } + + /** + * The `ON` join is matched against [originalTargetRows] - a snapshot of the target table taken + * once before any mutation - not the live, mutating `table.rows`: this is what makes "a target + * row matched by two different source rows" a well-defined, detectable condition at all (see + * below), and matches how a real MERGE's source/target join reads one consistent view of the + * target rather than re-joining against its own in-flight writes. A row this MERGE inserts is + * therefore never itself matched by a later source row (it never enters the snapshot), and a + * row this MERGE updates/deletes keeps its ORIGINAL identity for join purposes even after the + * live table's copy of it changes underneath - [applyMergeUpdate]/[applyMergeDelete] are handed + * that same original [Row] object and use `===` against it to find their spot in the live table. + * + * For each source row, every target row (from the snapshot) satisfying [MergeStatement.onCondition] + * is a MATCHED pair: the first [MergeMatchedUpdate]/[MergeMatchedDelete] clause (in + * [MergeStatement.whenClauses] written order - MATCHED and NOT MATCHED clauses can be + * interleaved, but a given pair only ever considers its own category) whose own `AND` condition + * also holds runs; if none holds, that pair is left untouched, matching Postgres. A target row + * already affected earlier in this same statement by a different matched pair throws rather than + * silently applying a second action - the same "MERGE command cannot affect a row a second time" + * guarantee Postgres gives, needed here because this engine has no snapshot isolation to make + * that outcome well-defined otherwise. A source row matched by zero target rows is NOT MATCHED: + * the first [MergeNotMatchedInsert] whose own condition holds (evaluated against the source row + * alone - there is no target row to reference) inserts one new row; [checkColumns]-equivalent + * validation for every clause happens up front via [checkMergeColumns], even when zero source + * rows exist, matching this engine's usual eager-validation bar for every other schema-shaped + * clause. + */ + private fun executeMergeInner(statement: MergeStatement): StatementResult { + val table = catalog.table(statement.targetTable) + val schema = table.schema + val targetLabel = statement.targetAlias ?: statement.targetTable + val (sourceSchema, sourceRows) = selectRunner.resolveMergeSource(statement.source) + val sourceLabel = selectRunner.mergeSourceLabel(statement.source) + if (targetLabel.equals(sourceLabel, ignoreCase = true)) { + throw KestrelException("MERGE target and source cannot share the same name or alias: $targetLabel") + } + checkMergeColumns(statement, schema, targetLabel, sourceSchema, sourceLabel) + + val originalTargetRows = table.rows.toList() + val hasTriggers = catalog.hasTriggers(schema.name) + val affectedTargets = mutableListOf() + val returnedRows = mutableListOf>() + var inserted = 0 + var updated = 0 + var deleted = 0 + + for (sourceRow in sourceRows) { + val matchedTargets = originalTargetRows.filter { targetRow -> + val resolver = mergeResolver(schema, targetLabel, targetRow, sourceSchema, sourceLabel, sourceRow) + isTrue(Evaluator(resolver, subqueryRunner).evaluate(statement.onCondition)) + } + if (matchedTargets.isNotEmpty()) { + for (targetRow in matchedTargets) { + val resolver = mergeResolver(schema, targetLabel, targetRow, sourceSchema, sourceLabel, sourceRow) + val clause = statement.whenClauses.firstOrNull { candidate -> + (candidate is MergeMatchedUpdate || candidate is MergeMatchedDelete) && + (candidate.condition == null || isTrue(Evaluator(resolver, subqueryRunner).evaluate(candidate.condition!!))) + } ?: continue + if (affectedTargets.any { it === targetRow }) { + throw KestrelException("MERGE command cannot affect a target row a second time") + } + affectedTargets.add(targetRow) + when (clause) { + is MergeMatchedUpdate -> { + val updatedRow = applyMergeUpdate(schema, table, targetRow, clause, resolver, hasTriggers) + returnedRows.add(updatedRow to "UPDATE") + updated++ + } + is MergeMatchedDelete -> { + val deletedRow = applyMergeDelete(schema, table, targetRow, hasTriggers) + returnedRows.add(deletedRow to "DELETE") + deleted++ + } + else -> error("unreachable: filtered to MATCHED clauses above") + } + } + continue + } + val resolver = notMatchedResolver(sourceSchema, sourceLabel, sourceRow, targetLabel) + val clause = statement.whenClauses.firstOrNull { candidate -> + candidate is MergeNotMatchedInsert && + (candidate.condition == null || isTrue(Evaluator(resolver, subqueryRunner).evaluate(candidate.condition!!))) + } as? MergeNotMatchedInsert ?: continue + val insertedRow = applyMergeInsert(schema, table, clause, resolver, hasTriggers) + returnedRows.add(insertedRow to "INSERT") + inserted++ + } + if (!hasTriggers) catalog.reindexTable(schema.name) + val returning = statement.returning?.let { evaluateMergeReturning(schema, returnedRows, it) } + return MergeResult(inserted, updated, deleted, returning) + } + + /** + * Evaluates a MERGE statement's RETURNING clause against every row an action of this MERGE + * actually touched - the post-image for an inserted/updated row, the pre-deletion image for a + * deleted one (same "nothing else to show" reasoning as [evaluateReturning]) - in the exact order + * [executeMergeInner] applied those actions in (source-row order, and within a single source row's + * multiple matched target rows, the order [executeMergeInner]'s own inner loop visits them). + * [actionedRow]'s second element is `"INSERT"`/`"UPDATE"`/`"DELETE"`, bound to that row's + * [Evaluator] as [MergeActionExpression]'s value so `MERGE_ACTION()` can appear in the projection - + * the one context where it is actually legal. Column references resolve against [schema] (the + * MERGE's target table) alone, same as [evaluateReturning] - RETURNING never sees the source row, + * only what ended up true of the target. + */ + private fun evaluateMergeReturning(schema: TableSchema, actionedRows: List>, projection: Projection): SelectResult { + val columns = when (projection) { + is StarProjection -> schema.columns.map { it.name } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> + checkColumns(item.expression, schema) + returningColumnLabel(item, index) + } + } + val resultRows = actionedRows.map { (row, action) -> + when (projection) { + is StarProjection -> row.values.toList() + is ExpressionProjection -> { + val evaluator = Evaluator(rowResolver(schema, row), subqueryRunner, mergeAction = action) + projection.items.map { evaluator.evaluate(it.expression) } + } + } + } + return SelectResult(columns, resultRows) + } + + private fun applyMergeUpdate( + schema: TableSchema, + table: Table, + targetRow: Row, + clause: MergeMatchedUpdate, + resolver: ColumnResolver, + hasTriggers: Boolean + ): Row { + val assignmentEvaluator = Evaluator(resolver, subqueryRunner, this) + val values = targetRow.values.toMutableList() + for (assignment in clause.assignments) { + val index = schema.columnIndex(assignment.columnName) + ?: throw KestrelException("no such column: ${assignment.columnName}") + val column = schema.columns[index] + rejectGeneratedAssignment(column, explicitValueGiven = assignment.value != null) + values[index] = if (assignment.value != null) { + coerceForColumn(assignmentEvaluator.evaluate(assignment.value), column) + } else { + column.default?.let { coerceForColumn(assignmentEvaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values) + validateNotNull(schema, values) + val updatedRow = Row(values) + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, listOf(targetRow to updatedRow)) + ForeignKeyEnforcer.validateRow(catalog, schema, updatedRow.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, updatedRow) + val others = table.rows.filter { it !== targetRow } + enforcePrimaryKeyUniqueness(schema, others + updatedRow) + enforceUniqueConstraints(schema, others + updatedRow) + enforceUniqueIndexes(schema, others + updatedRow) + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val oldKey = columnIndices.map { targetRow.values[it] } + val newKey = columnIndices.map { updatedRow.values[it] } + ForeignKeyEnforcer.cascadeUpdate(catalog, schema.name, group, listOf(oldKey to newKey)) + } + val position = table.rows.indexOfFirst { it === targetRow } + table.rows[position] = updatedRow + if (hasTriggers) catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, listOf(targetRow to updatedRow)) + return updatedRow + } + + /** Returns [targetRow] itself - the pre-deletion image, matching plain DELETE's own RETURNING (see [evaluateReturning]'s doc). */ + private fun applyMergeDelete(schema: TableSchema, table: Table, targetRow: Row, hasTriggers: Boolean): Row { + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Delete, listOf(targetRow to null)) + ForeignKeyEnforcer.cascadeDelete(catalog, schema.name, listOf(targetRow)) + table.rows.removeAll { it === targetRow } + if (hasTriggers) catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Delete, listOf(targetRow to null)) + return targetRow + } + + private fun applyMergeInsert( + schema: TableSchema, + table: Table, + clause: MergeNotMatchedInsert, + resolver: ColumnResolver, + hasTriggers: Boolean + ): Row { + val targetIndices = resolveInsertTargets(schema, clause.columns) + val evaluator = Evaluator(resolver, subqueryRunner, this) + val row = buildInsertRow(schema, targetIndices, clause.values, evaluator) + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, listOf(null to row)) + ForeignKeyEnforcer.validateRow(catalog, schema, row.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, row) + enforcePrimaryKeyUniqueness(schema, table.rows + row) + enforceUniqueConstraints(schema, table.rows + row) + enforceUniqueIndexes(schema, table.rows + row) + table.rows.add(row) + if (hasTriggers) catalog.reindexTable(schema.name) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, listOf(null to row)) + return row + } + + /** + * Resolves a MERGE `ON`/`WHEN MATCHED` column reference against whichever of [targetRow]/ + * [sourceRow] its qualifier names ([targetLabel]/[sourceLabel] - alias if given, bare name + * otherwise); an unqualified reference must exist on exactly one side, matching standard SQL's + * "ambiguous column" rule for a two-relation scope. + */ + private fun mergeResolver( + targetSchema: TableSchema, + targetLabel: String, + targetRow: Row, + sourceSchema: TableSchema, + sourceLabel: String, + sourceRow: Row + ): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + if (qualifier != null) { + return when { + qualifier.equals(sourceLabel, ignoreCase = true) -> + sourceRow.values[sourceSchema.columnIndex(expression.name) ?: throw KestrelException("no such column: ${expression.name}")] + qualifier.equals(targetLabel, ignoreCase = true) -> + targetRow.values[targetSchema.columnIndex(expression.name) ?: throw KestrelException("no such column: ${expression.name}")] + else -> throw KestrelException("no such table or alias: $qualifier") + } + } + val targetIndex = targetSchema.columnIndex(expression.name) + val sourceIndex = sourceSchema.columnIndex(expression.name) + if (targetIndex != null && sourceIndex != null) { + throw KestrelException("column reference '${expression.name}' is ambiguous between $targetLabel and $sourceLabel") + } + targetIndex?.let { return targetRow.values[it] } + sourceIndex?.let { return sourceRow.values[it] } + throw KestrelException("no such column: ${expression.name}") + } + + override fun declaredCollation(expression: ColumnExpression): Collation? { + val qualifier = expression.qualifier + if (qualifier != null) { + return when { + qualifier.equals(sourceLabel, ignoreCase = true) -> sourceSchema.column(expression.name)?.collation + qualifier.equals(targetLabel, ignoreCase = true) -> targetSchema.column(expression.name)?.collation + else -> null + } + } + val targetColumn = targetSchema.column(expression.name) + val sourceColumn = sourceSchema.column(expression.name) + return if (targetColumn != null && sourceColumn == null) targetColumn.collation + else if (sourceColumn != null && targetColumn == null) sourceColumn.collation + else null + } + } + + /** A `WHEN NOT MATCHED` clause has no target row to read - a qualifier naming the target throws rather than silently resolving to nothing. */ + private fun notMatchedResolver( + sourceSchema: TableSchema, + sourceLabel: String, + sourceRow: Row, + targetLabel: String + ): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + if (qualifier != null && qualifier.equals(targetLabel, ignoreCase = true)) { + throw KestrelException("no target row is available in a WHEN NOT MATCHED clause: $targetLabel.${expression.name}") + } + if (qualifier != null && !qualifier.equals(sourceLabel, ignoreCase = true)) { + throw KestrelException("no such table or alias: $qualifier") + } + val index = sourceSchema.columnIndex(expression.name) ?: throw KestrelException("no such column: ${expression.name}") + return sourceRow.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = sourceSchema.column(expression.name)?.collation + } + + /** Upfront validation for every clause of a MERGE statement, run even when the source produces zero rows - matching [evaluateReturning]'s own reasoning for why RETURNING is checked before knowing how many rows will actually match. */ + private fun checkMergeColumns( + statement: MergeStatement, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String + ) { + checkMergeExpressionColumns(statement.onCondition, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) + for (clause in statement.whenClauses) { + when (clause) { + is MergeMatchedUpdate -> { + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + for (assignment in clause.assignments) { + if (targetSchema.columnIndex(assignment.columnName) == null) { + throw KestrelException("no such column: ${assignment.columnName}") + } + assignment.value?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + } + } + is MergeMatchedDelete -> + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + is MergeNotMatchedInsert -> { + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = false) } + val targetIndices = resolveInsertTargets(targetSchema, clause.columns) + if (clause.values.size != targetIndices.size) { + throw KestrelException( + "table ${targetSchema.name} expects ${targetIndices.size} values but ${clause.values.size} were supplied" + ) + } + clause.values.forEach { value -> + value?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = false) } + } + } + } + } + } + + private fun checkMergeExpressionColumns( + expression: Expression, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String, + allowTarget: Boolean + ) { + fun recurse(inner: Expression) = + checkMergeExpressionColumns(inner, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget) + when (expression) { + is ColumnExpression -> checkMergeColumnReference(expression, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget) + is BinaryExpression -> { recurse(expression.left); recurse(expression.right) } + is UnaryExpression -> recurse(expression.operand) + is FunctionExpression -> throw KestrelException("aggregate functions are not allowed here") + is WindowExpression -> throw KestrelException("window functions are not allowed here") + is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { recurse(it) } + is IsNullExpression -> recurse(expression.operand) + is InExpression -> { + recurse(expression.operand) + expression.values.forEach { recurse(it) } + } + is BetweenExpression -> { + recurse(expression.operand) + recurse(expression.low) + recurse(expression.high) + } + is LikeExpression -> { + recurse(expression.operand) + recurse(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { recurse(it) } + expression.branches.forEach { recurse(it.condition); recurse(it.result) } + expression.elseExpression?.let { recurse(it) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> recurse(expression.operand) + is CastExpression -> recurse(expression.operand) + is CollateExpression -> recurse(expression.operand) + // Structurally harmless here too (no column reference to check) - none of ON/WHEN's + // AND-condition/assignments/INSERT values are where MERGE_ACTION() is ever legal (only + // this statement's own RETURNING is, see evaluateMergeReturning), and it is never handed + // a bound action there, so evaluation itself throws - same division of labor as + // checkColumns's own NEXTVAL/CURRVAL branch. A parameter reference IS legitimately allowed + // here (MERGE is one of PreparableStatement's own kinds) - by the time this runs it has + // already been substituted for a literal by resolveExecute, so this branch only ever sees + // one directly typed outside PREPARE, which is just as harmless structurally. + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> Unit + } + } + + private fun checkMergeColumnReference( + expression: ColumnExpression, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String, + allowTarget: Boolean + ) { + val qualifier = expression.qualifier + if (qualifier != null) { + val matchesSource = qualifier.equals(sourceLabel, ignoreCase = true) + val matchesTargetLabel = qualifier.equals(targetLabel, ignoreCase = true) + if (matchesTargetLabel && !allowTarget) { + throw KestrelException("no target row is available in a WHEN NOT MATCHED clause: $targetLabel.${expression.name}") + } + if (!matchesSource && !matchesTargetLabel) { + throw KestrelException("no such table or alias: $qualifier") + } + val schema = if (matchesSource) sourceSchema else targetSchema + if (schema.columnIndex(expression.name) == null) throw KestrelException("no such column: ${expression.name}") + return + } + val inTarget = allowTarget && targetSchema.columnIndex(expression.name) != null + val inSource = sourceSchema.columnIndex(expression.name) != null + if (inTarget && inSource) { + throw KestrelException("column reference '${expression.name}' is ambiguous between $targetLabel and $sourceLabel") + } + if (!inTarget && !inSource) throw KestrelException("no such column: ${expression.name}") } private fun coerceForColumn(value: Value, column: Column): Value { @@ -199,6 +2044,16 @@ class Executor(private val catalog: Catalog) { is Value.IntegerValue -> Value.RealValue(value.value.toDouble()) else -> throw typeMismatch(column, value) } + ColumnType.Decimal -> { + val raw = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> throw typeMismatch(column, value) + } + Value.DecimalValue(DecimalColumns.normalize(raw, column.precision, column.scale, "column ${column.name}")) + } + ColumnType.Date -> value as? Value.DateValue ?: throw typeMismatch(column, value) } } @@ -214,28 +2069,130 @@ class Executor(private val catalog: Catalog) { } private fun enforcePrimaryKeyUniqueness(schema: TableSchema, rows: List) { - val primaryKeyIndex = schema.primaryKeyIndex ?: return - val seen = HashSet() + val indices = schema.primaryKeyIndices + if (indices.isEmpty()) return + val seen = HashSet>() for (row in rows) { - val key = row.values[primaryKeyIndex] + val key = collationFoldedUniquenessKey(schema, indices, indices.map { row.values[it] }) if (!seen.add(key)) { - throw KestrelException("UNIQUE constraint failed: ${schema.name}.${schema.columns[primaryKeyIndex].name}") + val columnList = indices.joinToString(", ") { schema.columns[it].name } + throw KestrelException("UNIQUE constraint failed: ${schema.name}($columnList)") } } } - private fun checkColumns(expression: Expression, schema: TableSchema) { + /** + * A row where any constrained column is NULL is exempt (standard SQL: NULLs never collide + * with each other), so such rows are simply skipped rather than added to [seen]. + */ + private fun enforceUniqueConstraints(schema: TableSchema, rows: List) { + for (unique in schema.uniqueConstraints) { + val columnIndices = unique.columns.map { schema.columnIndex(it)!! } + val seen = HashSet>() + for (row in rows) { + val rawKey = columnIndices.map { row.values[it] } + if (rawKey.any { it is Value.NullValue }) continue + if (!seen.add(collationFoldedUniquenessKey(schema, columnIndices, rawKey))) { + val label = unique.name?.let { " $it" } ?: "" + val columnList = unique.columns.joinToString(", ") + throw KestrelException("UNIQUE constraint$label failed: ${schema.name}($columnList)") + } + } + } + } + + /** + * A `CREATE UNIQUE INDEX` is a way to attach a uniqueness guarantee to a table AFTER it + * already exists (the engine has no ALTER TABLE), so it is checked the same way as a + * declarative UNIQUE constraint - against the whole candidate row set, before any row is + * written - rather than relying on `SortedIndex.add` to reject a duplicate during the + * post-write `catalog.reindexTable` rebuild, which would leave the table already mutated. + * This key is folded through [collationFoldedUniquenessKey] for the same reason + * [enforcePrimaryKeyUniqueness]/[enforceUniqueConstraints] are: since `SortedIndex`'s own + * comparator is *also* collation-aware (see [kestrel.catalog.IndexKeyOrdering]), skipping the + * fold here would not merely miss the duplicate - `table.rows.addAll`/`.add` would already have + * run by the time `reindexTable` calls `SortedIndex.add` and throws on the same duplicate, and + * for a trigger-free table [withStatementAtomicity] takes no snapshot, so that later throw would + * leave the just-inserted duplicate permanently stuck in `table.rows`, indexed nowhere - a real, + * reproduced-during-development corruption mode, not a hypothetical one. + */ + private fun enforceUniqueIndexes(schema: TableSchema, rows: List) { + for (definition in catalog.indexDefinitions()) { + if (!definition.unique || !definition.tableName.equals(schema.name, ignoreCase = true)) continue + val columnIndices = definition.columnNames.mapNotNull { schema.columnIndex(it) } + if (columnIndices.size != definition.columnNames.size) continue + val seen = HashSet>() + for (row in rows) { + val rawKey = columnIndices.map { row.values[it] } + if (rawKey.any { it is Value.NullValue }) continue + if (!seen.add(collationFoldedUniquenessKey(schema, columnIndices, rawKey))) { + throw KestrelException("UNIQUE constraint failed: ${schema.name}(${definition.columnDisplayName})") + } + } + } + } + + /** + * `triggerContext` lets a `NEW.col`/`OLD.col` reference inside a trigger body statement pass + * validation against the trigger's OWN table (via [TriggerRowContext.checkColumnExists]) + * instead of being checked - wrongly - against [schema], the body statement's own target + * table, which is often a different table entirely. + */ + private fun checkColumns(expression: Expression, schema: TableSchema, triggerContext: TriggerRowContext? = null) { when (expression) { - is ColumnExpression -> if (schema.columnIndex(expression.name) == null) { + is ColumnExpression -> if (triggerContext != null && triggerContext.matches(expression)) { + triggerContext.checkColumnExists(expression) + } else if (schema.columnIndex(expression.name) == null) { throw KestrelException("no such column: ${expression.name}") } is BinaryExpression -> { - checkColumns(expression.left, schema) - checkColumns(expression.right, schema) + checkColumns(expression.left, schema, triggerContext) + checkColumns(expression.right, schema, triggerContext) } - is UnaryExpression -> checkColumns(expression.operand, schema) + is UnaryExpression -> checkColumns(expression.operand, schema, triggerContext) is FunctionExpression -> throw KestrelException("aggregate functions are not allowed here") + is WindowExpression -> throw KestrelException("window functions are not allowed here") is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { checkColumns(it, schema, triggerContext) } + is IsNullExpression -> checkColumns(expression.operand, schema, triggerContext) + is InExpression -> { + checkColumns(expression.operand, schema, triggerContext) + expression.values.forEach { checkColumns(it, schema, triggerContext) } + } + is BetweenExpression -> { + checkColumns(expression.operand, schema, triggerContext) + checkColumns(expression.low, schema, triggerContext) + checkColumns(expression.high, schema, triggerContext) + } + is LikeExpression -> { + checkColumns(expression.operand, schema, triggerContext) + checkColumns(expression.pattern, schema, triggerContext) + } + is CaseExpression -> { + expression.operand?.let { checkColumns(it, schema, triggerContext) } + expression.branches.forEach { + checkColumns(it.condition, schema, triggerContext) + checkColumns(it.result, schema, triggerContext) + } + expression.elseExpression?.let { checkColumns(it, schema, triggerContext) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> checkColumns(expression.operand, schema, triggerContext) + is CastExpression -> checkColumns(expression.operand, schema, triggerContext) + is CollateExpression -> checkColumns(expression.operand, schema, triggerContext) + // Structurally harmless either way (no column reference to check) - whether NEXTVAL/ + // CURRVAL is actually legal at this particular call site (SET assignments: yes; WHERE: + // no, see Evaluator's SequenceAccess doc) is enforced by whether the Evaluator this + // feeds into was actually given a SequenceAccess, not by this column-existence pass. + // MERGE_ACTION() is the same story: this generic checkColumns also backs plain INSERT/ + // UPDATE/DELETE's own RETURNING (evaluateReturning), where it is never legal - but that is + // enforced by evaluateReturning's Evaluator never being handed a mergeAction, exactly like + // evaluateMergeReturning is the one call site that does. A parameter reference is + // legitimately allowed anywhere this generic pass runs (INSERT/UPDATE/DELETE/SELECT are + // all PreparableStatement kinds) - already substituted for a literal by the time a real + // EXECUTE reaches here. + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> Unit } } diff --git a/src/main/kotlin/kestrel/engine/ForeignKeyEnforcer.kt b/src/main/kotlin/kestrel/engine/ForeignKeyEnforcer.kt new file mode 100644 index 0000000..3e9f89a --- /dev/null +++ b/src/main/kotlin/kestrel/engine/ForeignKeyEnforcer.kt @@ -0,0 +1,207 @@ +package kestrel.engine + +import kestrel.KestrelException +import kestrel.catalog.Catalog +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.TableSchema +import kestrel.storage.Row +import kestrel.storage.Table +import kestrel.storage.Value + +/** + * Runtime enforcement for FOREIGN KEY constraints: existence checks on INSERT/UPDATE of the + * referencing (child) table, and ON DELETE/ON UPDATE actions on the referenced (parent) table. + * `referencedColumns` is always a column group with an enforced uniqueness guarantee in the + * parent (its PRIMARY KEY, single-column or composite, or a UNIQUE constraint), so cascades track + * a whole key tuple (`List`, one entry per column, positionally paired with `columns`) at + * a time rather than a single value - this is what lets a single-column FK and a multi-column FK + * share every code path below unchanged. + * + * Cascades are computed as a plan first and only mutate afterwards (see [CascadeStep]) so that a + * RESTRICT violation discovered in one child table can never leave an earlier, unrelated child + * table half-cascaded - either the whole cascade applies, or nothing does. + */ +object ForeignKeyEnforcer { + + /** + * [uncommittedSiblings] covers a self-referencing FK (`referencedTable == schema.name`, e.g. an + * employee/manager hierarchy) inside a multi-row INSERT/UPDATE statement: those sibling rows + * are being validated as part of the very same statement and aren't in the catalog's table yet, + * so without this a later row couldn't reference an earlier row from the same batch. + * + * Follows MATCH SIMPLE (the SQL default): if *any* column of the FK is NULL in this row, the + * whole constraint is skipped for that row - not just the null column. + */ + fun validateRow( + catalog: Catalog, + schema: TableSchema, + values: List, + uncommittedSiblings: List = emptyList() + ) { + for (fk in schema.foreignKeys) { + val columnIndices = fk.columns.map { schema.columnIndex(it)!! } + val key = columnIndices.map { values[it] } + if (key.any { it is Value.NullValue }) continue + val referenced = catalog.tableOrNull(fk.referencedTable) + ?: throw KestrelException( + "FOREIGN KEY constraint failed: table '${fk.referencedTable}' referenced by " + + "${schema.name}(${fk.columns.joinToString(", ")}) does not exist" + ) + val refIndices = fk.referencedColumns.map { referenced.schema.columnIndex(it)!! } + val siblings = if (fk.referencedTable.equals(schema.name, ignoreCase = true)) uncommittedSiblings else emptyList() + val exists = referenced.rows.any { refIndices.map { idx -> it.values[idx] } == key } || + siblings.any { refIndices.map { idx -> it.values[idx] } == key } + if (!exists) { + throw KestrelException( + "FOREIGN KEY constraint failed: ${schema.name}(${fk.columns.joinToString(", ")}) has no " + + "matching row in ${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")})" + ) + } + } + } + + private sealed interface CascadeStep { + data class Delete(val table: Table, val rows: List, val next: List) : CascadeStep + data class SetToNull(val table: Table, val columnIndices: List, val rows: List) : CascadeStep + data class SetTo(val table: Table, val columnIndices: List, val rows: List, val values: List) : CascadeStep + } + + fun cascadeDelete(catalog: Catalog, parentTableName: String, deletedRows: List) { + applySteps(catalog, planDelete(catalog, parentTableName, deletedRows)) + } + + /** + * [referencedColumns] is the full column group whose values changed together (a PRIMARY KEY or + * UNIQUE key group) and [changedPairs] holds the old/new key tuple for every row of the parent + * table, positionally aligned with [referencedColumns] - a matching FK may declare its own + * `referencedColumns` in a different order, so [planUpdate] re-projects each tuple by name. + */ + fun cascadeUpdate( + catalog: Catalog, + parentTableName: String, + referencedColumns: List, + changedPairs: List, List>> + ) { + applySteps(catalog, planUpdate(catalog, parentTableName, referencedColumns, changedPairs)) + } + + private fun planDelete(catalog: Catalog, parentTableName: String, deletedRows: List): List { + if (deletedRows.isEmpty()) return emptyList() + val parent = catalog.table(parentTableName) + val steps = mutableListOf() + for (childName in catalog.tableNames) { + val child = catalog.table(childName) + for (fk in child.schema.foreignKeys) { + if (!fk.referencedTable.equals(parentTableName, ignoreCase = true)) continue + val refIndices = fk.referencedColumns.map { parent.schema.columnIndex(it)!! } + val deletedKeys = deletedRows.map { row -> refIndices.map { row.values[it] } }.toHashSet() + val childColumnIndices = fk.columns.map { child.schema.columnIndex(it)!! } + val matches = child.rows.filter { row -> childColumnIndices.map { row.values[it] } in deletedKeys } + if (matches.isEmpty()) continue + when (fk.onDelete) { + ForeignKeyAction.Cascade -> + steps.add(CascadeStep.Delete(child, matches, planDelete(catalog, child.schema.name, matches))) + ForeignKeyAction.SetNull -> { + requireNullable(child.schema, childColumnIndices, "ON DELETE SET NULL") + steps.add(CascadeStep.SetToNull(child, childColumnIndices, matches)) + } + ForeignKeyAction.Restrict, ForeignKeyAction.NoAction -> throw KestrelException( + "FOREIGN KEY constraint failed: ${matches.size} row(s) in ${child.schema.name} still " + + "reference $parentTableName (use ON DELETE CASCADE or remove them first)" + ) + } + } + } + return steps + } + + private fun planUpdate( + catalog: Catalog, + parentTableName: String, + referencedColumns: List, + changedPairs: List, List>> + ): List { + val changes = changedPairs.filter { (old, new) -> old != new } + if (changes.isEmpty()) return emptyList() + val steps = mutableListOf() + for (childName in catalog.tableNames) { + val child = catalog.table(childName) + for (fk in child.schema.foreignKeys) { + if (!fk.referencedTable.equals(parentTableName, ignoreCase = true)) continue + if (fk.referencedColumns.size != referencedColumns.size) continue + // Re-project the tuple by name: this FK's referencedColumns may list the same key + // group in a different order than the one changedPairs was built from. + val positions = fk.referencedColumns.map { refCol -> + referencedColumns.indexOfFirst { it.equals(refCol, ignoreCase = true) } + } + if (positions.any { it == -1 }) continue + val childColumnIndices = fk.columns.map { child.schema.columnIndex(it)!! } + for ((oldTuple, newTuple) in changes) { + val oldKey = positions.map { oldTuple[it] } + val newKey = positions.map { newTuple[it] } + if (oldKey == newKey) continue + val matches = child.rows.filter { row -> childColumnIndices.map { row.values[it] } == oldKey } + if (matches.isEmpty()) continue + when (fk.onUpdate) { + ForeignKeyAction.Cascade -> + steps.add(CascadeStep.SetTo(child, childColumnIndices, matches, newKey)) + ForeignKeyAction.SetNull -> { + requireNullable(child.schema, childColumnIndices, "ON UPDATE SET NULL") + steps.add(CascadeStep.SetToNull(child, childColumnIndices, matches)) + } + ForeignKeyAction.Restrict, ForeignKeyAction.NoAction -> throw KestrelException( + "FOREIGN KEY constraint failed: ${matches.size} row(s) in ${child.schema.name} still " + + "reference $parentTableName (use ON UPDATE CASCADE)" + ) + } + } + } + } + return steps + } + + private fun applySteps(catalog: Catalog, steps: List) { + for (step in steps) { + when (step) { + is CascadeStep.Delete -> { + step.table.rows.removeAll(step.rows) + catalog.reindexTable(step.table.schema.name) + applySteps(catalog, step.next) + } + is CascadeStep.SetToNull -> { + val nulls = step.columnIndices.map { Value.NullValue } + replaceRows(step.table, step.rows) { row -> withColumns(row, step.columnIndices, nulls) } + catalog.reindexTable(step.table.schema.name) + } + is CascadeStep.SetTo -> { + replaceRows(step.table, step.rows) { row -> withColumns(row, step.columnIndices, step.values) } + catalog.reindexTable(step.table.schema.name) + } + } + } + } + + private fun requireNullable(schema: TableSchema, columnIndices: List, action: String) { + for (columnIndex in columnIndices) { + val column = schema.columns[columnIndex] + if (column.notNull) { + throw KestrelException( + "FOREIGN KEY constraint failed: cannot $action on NOT NULL column ${schema.name}.${column.name}" + ) + } + } + } + + private fun withColumns(row: Row, columnIndices: List, values: List): Row { + val result = row.values.toMutableList() + columnIndices.forEachIndexed { position, columnIndex -> result[columnIndex] = values[position] } + return Row(result) + } + + private fun replaceRows(table: Table, targets: List, transform: (Row) -> Row) { + val targetSet = targets.toHashSet() + val updated = table.rows.map { row -> if (row in targetSet) transform(row) else row } + table.rows.clear() + table.rows.addAll(updated) + } +} diff --git a/src/main/kotlin/kestrel/engine/GroupAggregation.kt b/src/main/kotlin/kestrel/engine/GroupAggregation.kt index 0a72735..b140114 100644 --- a/src/main/kotlin/kestrel/engine/GroupAggregation.kt +++ b/src/main/kotlin/kestrel/engine/GroupAggregation.kt @@ -1,54 +1,188 @@ package kestrel.engine import kestrel.KestrelException +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseBranch +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression import kestrel.sql.ast.Expression import kestrel.sql.ast.ExpressionProjection import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.OrderKey import kestrel.sql.ast.SelectItem import kestrel.sql.ast.SelectStatement import kestrel.sql.ast.UnaryExpression +import kestrel.storage.Collation import kestrel.storage.Value -import kestrel.storage.ValueOrdering -import kestrel.storage.typeName class GroupAggregation( private val resolveIndex: (kestrel.sql.ast.ColumnExpression) -> Int, private val evaluateRow: (Expression, List) -> Value, - private val label: (SelectItem, Int) -> String + private val label: (SelectItem, Int) -> String, + private val subqueries: SubqueryRunner? = null, + /** A bare grouping column's declared `COLLATE` (see [kestrel.catalog.Column.collation]) - defaults to reporting none, matching every [ColumnResolver] built before this existed. */ + private val declaredCollationOf: (kestrel.sql.ast.ColumnExpression) -> Collation? = { null } ) { - fun aggregate(statement: SelectStatement, orderExpression: Expression?, combined: List>): SelectResult { - val groups = LinkedHashMap, MutableList>>() - for (row in combined) { - val key = statement.groupBy.map { evaluateRow(it, row) } - groups.getOrPut(key) { mutableListOf() }.add(row) - } - var groupRows: List>> = - if (statement.groupBy.isEmpty() && groups.isEmpty()) listOf(emptyList()) else groups.values.toList() + /** + * One row of the eventual result, before projection: the underlying source rows that fell into + * it, and the grouping-set expressions that were "active" (present, not rolled up away) for it. + * For an ordinary `GROUP BY` (no [kestrel.sql.ast.GroupingSpec]) there is exactly one grouping set + * per query - [activeExpressions] is always the full `GROUP BY` list, so every column reads + * normally and `GROUPING(...)` (see [substituteGroupingAwareCalls]) always answers 0, matching + * plain SQL. [rows] is empty only for the grand-total group of a set with zero expressions over a + * zero-row input - the same "ungrouped aggregate over nothing" case a bare `SELECT COUNT(*)` with + * no rows at all already has to handle. + */ + private data class LogicalGroup(val rows: List>, val activeExpressions: List) + + fun aggregate(statement: SelectStatement, orderKeys: List, combined: List>): SelectResult { + val allGroupingColumns = statement.groupBy + val sets = GroupingSetExpansion.setsFor(statement.groupingSpec, statement.groupBy) + var logicalGroups = sets.flatMap { groupsForSet(it, combined) } statement.having?.let { having -> - groupRows = groupRows.filter { isTrue(evaluateInGroup(having, it)) } + logicalGroups = logicalGroups.filter { isTrue(evaluateInGroup(having, it, allGroupingColumns)) } } - orderExpression?.let { expression -> - val comparator = Comparator>> { first, second -> - ValueOrdering.compare(evaluateInGroup(expression, first), evaluateInGroup(expression, second)) + if (orderKeys.isNotEmpty()) { + val comparator = Comparator { first, second -> + var result = 0 + for (key in orderKeys) { + val comparison = compareOrderKeyValues( + evaluateInGroup(key.expression, first, allGroupingColumns), + evaluateInGroup(key.expression, second, allGroupingColumns), + key, + declaredCollationOf + ) + if (comparison != 0) { + result = comparison + break + } + } + result } - groupRows = groupRows.sortedWith(if (statement.orderBy!!.descending) comparator.reversed() else comparator) + logicalGroups = logicalGroups.sortedWith(comparator) } - statement.limit?.let { - if (it < 0) throw KestrelException("LIMIT must not be negative") - groupRows = groupRows.take(it) + statement.offset?.let { if (it < 0) throw KestrelException("OFFSET must not be negative") } + statement.limit?.let { if (it < 0) throw KestrelException("LIMIT must not be negative") } + // DISTINCT can still collapse two separate groups into the same output row (e.g. + // `SELECT DISTINCT dept FROM t GROUP BY dept, sub_dept`) - LIMIT/OFFSET are deferred past + // dedup below so they count distinct output rows, not pre-dedup groups. + if (!statement.distinct && (statement.offset != null || statement.limit != null)) { + logicalGroups = logicalGroups.drop(statement.offset ?: 0) + statement.limit?.let { logicalGroups = logicalGroups.take(it) } } val projection = statement.projection as ExpressionProjection val labels = projection.items.mapIndexed { index, item -> label(item, index) } - val projected = groupRows.map { group -> projection.items.map { evaluateInGroup(it.expression, group) } } + var projected = logicalGroups.map { group -> projection.items.map { evaluateInGroup(it.expression, group, allGroupingColumns) } } + if (statement.distinct) { + val itemExpressions = projection.items.map { it.expression } + projected = projected.distinctBy { row -> collationFoldedKey(row, itemExpressions, declaredCollationOf) } + if (statement.offset != null || statement.limit != null) { + projected = projected.drop(statement.offset ?: 0) + statement.limit?.let { projected = projected.take(it) } + } + } return SelectResult(labels, projected) } - private fun evaluateInGroup(expression: Expression, group: List>): Value { - val substituted = substituteAggregates(expression, group) - val resolver = ColumnResolver { columnExpression -> group.first()[resolveIndex(columnExpression)] } - return Evaluator(resolver).evaluate(substituted) + /** + * Groups [combined] by [setExpressions] alone (not the query's full grouping-column union) - + * exactly the single-set logic this class always had, just parameterized per set instead of + * always using `statement.groupBy`. A set with zero expressions over zero source rows still + * produces the one grand-total group (`combined.isEmpty()` means the inner loop never ran, so + * `buckets` is genuinely empty rather than holding a single `emptyList()`-keyed bucket) - every + * *other* set, or this same set over a non-empty input, groups normally. + */ + private fun groupsForSet(setExpressions: List, combined: List>): List { + val buckets = LinkedHashMap, MutableList>>() + for (row in combined) { + val key = setExpressions.map { evaluateRow(it, row) } + val bucketKey = collationFoldedKey(key, setExpressions, declaredCollationOf) + buckets.getOrPut(bucketKey) { mutableListOf() }.add(row) + } + return if (setExpressions.isEmpty() && buckets.isEmpty()) { + listOf(LogicalGroup(emptyList(), setExpressions)) + } else { + buckets.values.map { LogicalGroup(it, setExpressions) } + } + } + + private fun evaluateInGroup(expression: Expression, group: LogicalGroup, allGroupingColumns: List): Value { + val nulled = substituteInactiveGroupingColumns(expression, group.activeExpressions, allGroupingColumns) + val substituted = substituteAggregates(nulled, group.rows) + // Only reached for a bare reference to one of `group.activeExpressions` (anything else was + // just replaced with NULL above), and a group with any active expression at all is never + // empty - it was built from at least one matching source row - so `.first()` cannot throw. + val resolver = object : ColumnResolver { + override fun resolve(columnExpression: kestrel.sql.ast.ColumnExpression): Value = + group.rows.first()[resolveIndex(columnExpression)] + + override fun declaredCollation(columnExpression: kestrel.sql.ast.ColumnExpression): Collation? = + declaredCollationOf(columnExpression) + } + return Evaluator(resolver, subqueries).evaluate(substituted) + } + + /** + * Replaces every grouping-column expression not in [activeExpressions] with a NULL literal (SQL's + * rule for a ROLLUP/CUBE/GROUPING SETS super-aggregate row: a collapsed column reads as NULL + * wherever it is used outside an aggregate's own argument) and every `GROUPING(...)` call with its + * literal bitmask. Stops at a [FunctionExpression] without recursing into its argument - an + * aggregate always sees real per-row values regardless of which columns this particular row + * collapsed, which is exactly what makes `SUM(amount)` correct on the grand-total row too. + */ + private fun substituteInactiveGroupingColumns( + expression: Expression, + activeExpressions: List, + allGroupingColumns: List + ): Expression { + if (expression in allGroupingColumns && expression !in activeExpressions) { + return LiteralExpression(Value.NullValue) + } + fun recurse(child: Expression) = substituteInactiveGroupingColumns(child, activeExpressions, allGroupingColumns) + return when (expression) { + is BinaryExpression -> BinaryExpression(expression.operator, recurse(expression.left), recurse(expression.right)) + is UnaryExpression -> UnaryExpression(expression.operator, recurse(expression.operand)) + is CallExpression -> + if (expression.name.equals("GROUPING", ignoreCase = true)) { + LiteralExpression(computeGrouping(expression, activeExpressions)) + } else { + CallExpression(expression.name, expression.arguments.map(::recurse)) + } + is IsNullExpression -> IsNullExpression(recurse(expression.operand), expression.negated) + is InExpression -> InExpression(recurse(expression.operand), expression.values.map(::recurse), expression.negated) + is BetweenExpression -> BetweenExpression( + recurse(expression.operand), recurse(expression.low), recurse(expression.high), expression.negated + ) + is LikeExpression -> LikeExpression(recurse(expression.operand), recurse(expression.pattern), expression.negated) + is CaseExpression -> CaseExpression( + expression.operand?.let(::recurse), + expression.branches.map { CaseBranch(recurse(it.condition), recurse(it.result)) }, + expression.elseExpression?.let(::recurse) + ) + is CastExpression -> CastExpression(recurse(expression.operand), expression.targetType, expression.precision, expression.scale) + else -> expression + } + } + + /** + * `GROUPING(e1, ..., en)` per the SQL standard: an integer whose bits, most significant first, are + * 1 for each `ei` collapsed away in this row's grouping set and 0 for each one still present - + * `computeAggregate`'s own arguments never reach this ([FunctionExpression] is not + * [CallExpression]), so there is no ambiguity with a real aggregate named similarly. + */ + private fun computeGrouping(call: CallExpression, activeExpressions: List): Value { + if (call.arguments.isEmpty()) throw KestrelException("GROUPING requires at least one argument") + var bits = 0L + for (argument in call.arguments) { + bits = (bits shl 1) or if (argument in activeExpressions) 0L else 1L + } + return Value.IntegerValue(bits) } private fun substituteAggregates(expression: Expression, group: List>): Expression = when (expression) { @@ -59,46 +193,52 @@ class GroupAggregation( substituteAggregates(expression.right, group) ) is UnaryExpression -> UnaryExpression(expression.operator, substituteAggregates(expression.operand, group)) + is CallExpression -> CallExpression(expression.name, expression.arguments.map { substituteAggregates(it, group) }) + is IsNullExpression -> IsNullExpression(substituteAggregates(expression.operand, group), expression.negated) + is InExpression -> InExpression( + substituteAggregates(expression.operand, group), + expression.values.map { substituteAggregates(it, group) }, + expression.negated + ) + is BetweenExpression -> BetweenExpression( + substituteAggregates(expression.operand, group), + substituteAggregates(expression.low, group), + substituteAggregates(expression.high, group), + expression.negated + ) + is LikeExpression -> LikeExpression( + substituteAggregates(expression.operand, group), + substituteAggregates(expression.pattern, group), + expression.negated + ) + is CaseExpression -> CaseExpression( + expression.operand?.let { substituteAggregates(it, group) }, + expression.branches.map { + CaseBranch(substituteAggregates(it.condition, group), substituteAggregates(it.result, group)) + }, + expression.elseExpression?.let { substituteAggregates(it, group) } + ) + is CastExpression -> CastExpression( + substituteAggregates(expression.operand, group), + expression.targetType, + expression.precision, + expression.scale + ) else -> expression } + /** + * `FILTER (WHERE ...)` narrows the rows this one aggregate call sees to those matching its own + * predicate, evaluated per source row exactly like a `WHERE` clause would be - independent of + * every other aggregate in the same SELECT list, which each see [group] unfiltered (or filtered by + * their own, different, FILTER clause) rather than this one's rows removed for everybody. + */ private fun computeAggregate(function: FunctionExpression, group: List>): Value { - val name = function.name.uppercase() - if (name == "COUNT") { - if (function.star) return Value.IntegerValue(group.size.toLong()) - val argument = function.argument ?: return Value.IntegerValue(group.size.toLong()) - val count = group.count { evaluateRow(argument, it) !is Value.NullValue } - return Value.IntegerValue(count.toLong()) - } - val argument = function.argument - ?: throw KestrelException("aggregate function ${function.name} requires an argument") - val values = group.map { evaluateRow(argument, it) }.filter { it !is Value.NullValue } - return when (name) { - "SUM" -> sum(values) - "AVG" -> average(values) - "MIN" -> values.minWithOrNull(ValueOrdering) ?: Value.NullValue - "MAX" -> values.maxWithOrNull(ValueOrdering) ?: Value.NullValue - else -> throw KestrelException("unknown function: ${function.name}") - } - } - - private fun sum(values: List): Value { - if (values.isEmpty()) return Value.NullValue - if (values.all { it is Value.IntegerValue }) { - return Value.IntegerValue(values.sumOf { (it as Value.IntegerValue).value }) - } - return Value.RealValue(values.sumOf { numeric(it) }) - } - - private fun average(values: List): Value { - if (values.isEmpty()) return Value.NullValue - return Value.RealValue(values.sumOf { numeric(it) } / values.size) - } - - private fun numeric(value: Value): Double = when (value) { - is Value.IntegerValue -> value.value.toDouble() - is Value.RealValue -> value.value - else -> throw KestrelException("aggregate requires numeric operands but got ${value.typeName}") + val filtered = function.filter?.let { predicate -> group.filter { isTrue(evaluateRow(predicate, it)) } } ?: group + return AggregateFunctions.compute( + function.name, function.argument, function.star, filtered, evaluateRow, + function.separator, function.orderBy, function.distinct, declaredCollationOf + ) } private fun isTrue(value: Value): Boolean = value is Value.BooleanValue && value.value diff --git a/src/main/kotlin/kestrel/engine/GroupingSetExpansion.kt b/src/main/kotlin/kestrel/engine/GroupingSetExpansion.kt new file mode 100644 index 0000000..89c2653 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/GroupingSetExpansion.kt @@ -0,0 +1,42 @@ +package kestrel.engine + +import kestrel.KestrelException +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.Expression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.GroupingSpec +import kestrel.sql.ast.RollupSpec + +/** + * Turns a [GroupingSpec] into the concrete list of grouping sets [GroupAggregation] groups by - one + * full pass over the source rows per set, later unioned together. `null` (an ordinary `GROUP BY a, b` + * or no `GROUP BY` at all) is exactly one implicit set, [groupBy] itself - this is what lets + * [GroupAggregation] treat the plain case as a special case of the general one instead of a separate + * code path. + */ +object GroupingSetExpansion { + /** CUBE(e1..en) is 2^n sets; capped well short of overflow or a runaway allocation from a typo'd column list. */ + private const val MAX_CUBE_COLUMNS = 16 + + fun setsFor(spec: GroupingSpec?, groupBy: List): List> = when (spec) { + null -> listOf(groupBy) + is RollupSpec -> rollupSets(spec.expressions) + is CubeSpec -> cubeSets(spec.expressions) + is GroupingSetsSpec -> spec.sets + } + + /** ROLLUP(e1, e2, e3) -> {e1,e2,e3}, {e1,e2}, {e1}, {} - each prefix of the declared order, longest first. */ + private fun rollupSets(expressions: List): List> = + (expressions.size downTo 0).map { expressions.subList(0, it) } + + /** CUBE(e1, e2, e3) -> every one of the 2^3 subsets, largest first; membership of expression i is bit i of a descending bitmask. */ + private fun cubeSets(expressions: List): List> { + if (expressions.size > MAX_CUBE_COLUMNS) { + throw KestrelException("CUBE supports at most $MAX_CUBE_COLUMNS expressions") + } + val subsetCount = 1 shl expressions.size + return (subsetCount - 1 downTo 0).map { mask -> + expressions.filterIndexed { index, _ -> (mask shr index) and 1 == 1 } + } + } +} diff --git a/src/main/kotlin/kestrel/engine/IndexAccessPlanner.kt b/src/main/kotlin/kestrel/engine/IndexAccessPlanner.kt index dacd328..afbc8d1 100644 --- a/src/main/kotlin/kestrel/engine/IndexAccessPlanner.kt +++ b/src/main/kotlin/kestrel/engine/IndexAccessPlanner.kt @@ -1,6 +1,7 @@ package kestrel.engine import kestrel.catalog.Catalog +import kestrel.catalog.SortedIndex import kestrel.catalog.TableSchema import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator @@ -14,6 +15,9 @@ sealed class PrimaryAccess { class IndexScan(val indexName: String, val description: String, val rowIndices: List) : PrimaryAccess() } +/** An [IndexScan] candidate together with how many of the index's columns it actually pinned down, used to pick the most selective option among several usable indexes. */ +private class ScoredScan(val boundColumns: Int, val scan: PrimaryAccess.IndexScan) + class IndexAccessPlanner(private val catalog: Catalog) { fun plan(tableName: String, alias: String, schema: TableSchema, predicate: Expression?): PrimaryAccess { if (predicate == null) return PrimaryAccess.FullScan @@ -22,32 +26,75 @@ class IndexAccessPlanner(private val catalog: Catalog) { val simple = asColumnPredicate(conjunct, alias, schema) ?: continue byColumn.getOrPut(simple.first) { mutableListOf() }.add(simple.second to simple.third) } - for ((columnName, predicates) in byColumn) { - val index = catalog.indexOn(tableName, columnName) ?: continue - val equal = predicates.firstOrNull { it.first == BinaryOperator.Equal } - if (equal != null) { - val candidates = index.equalTo(equal.second).sorted() - return PrimaryAccess.IndexScan(index.definition.name, "$columnName = ${literalText(equal.second)}", candidates) - } - var lower: Value? = null - var lowerInclusive = false - var upper: Value? = null - var upperInclusive = false - val parts = mutableListOf() - for ((operator, literal) in predicates) { - when (operator) { - BinaryOperator.Greater -> { lower = literal; lowerInclusive = false; parts.add("$columnName > ${literalText(literal)}") } - BinaryOperator.GreaterEqual -> { lower = literal; lowerInclusive = true; parts.add("$columnName >= ${literalText(literal)}") } - BinaryOperator.Less -> { upper = literal; upperInclusive = false; parts.add("$columnName < ${literalText(literal)}") } - BinaryOperator.LessEqual -> { upper = literal; upperInclusive = true; parts.add("$columnName <= ${literalText(literal)}") } - else -> Unit - } - } - if (lower == null && upper == null) continue - val candidates = index.rangeOf(lower, lowerInclusive, upper, upperInclusive).sorted() - return PrimaryAccess.IndexScan(index.definition.name, parts.joinToString(" AND "), candidates) + return catalog.indexesOn(tableName) + .mapNotNull { index -> planIndex(index, byColumn) } + .maxByOrNull { it.boundColumns } + ?.scan + ?: PrimaryAccess.FullScan + } + + /** + * Tries to serve [byColumn] from a single [index], preferring (in order): an exact match + * that binds every column of the index by equality; a range scan (<, <=, >, >=) on the + * column right after the longest leading equality run (works even with an empty run, i.e. + * a plain range on the index's very first column); or a pure equality prefix scan that + * binds a leading run of columns with the rest left entirely free. A composite index whose + * leading columns aren't bound at all (e.g. only its second column appears in the WHERE + * clause) is skipped - it cannot narrow the scan, since the index is sorted by its first + * column first. + */ + private fun planIndex(index: SortedIndex, byColumn: Map>>): ScoredScan? { + val definition = index.definition + val columns = definition.columnNames + val equalityValues = mutableListOf() + val descriptionParts = mutableListOf() + for (columnName in columns) { + val equal = byColumn[columnName]?.firstOrNull { it.first == BinaryOperator.Equal } ?: break + equalityValues.add(equal.second) + descriptionParts.add("$columnName = ${literalText(equal.second)}") } - return PrimaryAccess.FullScan + if (equalityValues.size == columns.size) { + val candidates = index.equalTo(equalityValues).sorted() + return ScoredScan(columns.size, PrimaryAccess.IndexScan(definition.name, descriptionParts.joinToString(" AND "), candidates)) + } + val nextColumn = columns[equalityValues.size] + val rangePredicates = byColumn[nextColumn]?.filter { it.first != BinaryOperator.Equal }.orEmpty() + if (rangePredicates.isNotEmpty()) { + val scan = rangeScan(index, nextColumn, rangePredicates, prefix = equalityValues.toList(), descriptionParts) + if (scan != null) return scan + } + if (equalityValues.isEmpty()) return null + // A pure equality prefix: every bound column matched by "=", nothing further constrained. + val candidates = index.rangeOf(equalityValues, true, equalityValues, true).sorted() + return ScoredScan(equalityValues.size, PrimaryAccess.IndexScan(definition.name, descriptionParts.joinToString(" AND "), candidates)) + } + + private fun rangeScan( + index: SortedIndex, + columnName: String, + predicates: List>, + prefix: List, + descriptionPrefix: List + ): ScoredScan? { + var lowerValue: Value? = null + var lowerInclusive = false + var upperValue: Value? = null + var upperInclusive = false + val parts = descriptionPrefix.toMutableList() + for ((operator, literal) in predicates) { + when (operator) { + BinaryOperator.Greater -> { lowerValue = literal; lowerInclusive = false; parts.add("$columnName > ${literalText(literal)}") } + BinaryOperator.GreaterEqual -> { lowerValue = literal; lowerInclusive = true; parts.add("$columnName >= ${literalText(literal)}") } + BinaryOperator.Less -> { upperValue = literal; upperInclusive = false; parts.add("$columnName < ${literalText(literal)}") } + BinaryOperator.LessEqual -> { upperValue = literal; upperInclusive = true; parts.add("$columnName <= ${literalText(literal)}") } + else -> Unit + } + } + if (lowerValue == null && upperValue == null) return null + val lower = lowerValue?.let { prefix + it } + val upper = upperValue?.let { prefix + it } + val candidates = index.rangeOf(lower, lowerInclusive, upper, upperInclusive).sorted() + return ScoredScan(prefix.size + 1, PrimaryAccess.IndexScan(index.definition.name, parts.joinToString(" AND "), candidates)) } private fun asColumnPredicate( @@ -95,6 +142,8 @@ class IndexAccessPlanner(private val catalog: Catalog) { is Value.RealValue -> value.value.toString() is Value.TextValue -> "'${value.value}'" is Value.BooleanValue -> value.value.toString() + is Value.DecimalValue -> value.value.toPlainString() + is Value.DateValue -> value.value.toString() is Value.NullValue -> "NULL" } diff --git a/src/main/kotlin/kestrel/engine/OrderKeyOrdering.kt b/src/main/kotlin/kestrel/engine/OrderKeyOrdering.kt new file mode 100644 index 0000000..d3ea7c6 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/OrderKeyOrdering.kt @@ -0,0 +1,49 @@ +package kestrel.engine + +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.OrderKey +import kestrel.storage.Collation +import kestrel.storage.Value +import kestrel.storage.ValueOrdering + +/** + * Compares two already-evaluated ORDER BY key values, shared by every sort in both the in-memory + * and disk engines (top-level ORDER BY, GROUP BY's own ORDER BY, STRING_AGG's inner ORDER BY, and + * a window function's `OVER (... ORDER BY ...)`) so `NULLS FIRST`/`NULLS LAST` behaves identically + * everywhere an [OrderKey] is honored. + * + * `NULLS FIRST`/`NULLS LAST` is an explicit override: when either side is NULL, [OrderKey.descending] + * plays no part - only [OrderKey.nullsFirst] decides, matching the SQL standard (unlike a plain + * value comparison, negating the whole result for DESC would also flip which end NULLs land on, + * which is not what an explicit NULLS clause means). When [OrderKey.nullsFirst] is unset (`null`, + * meaning the query never wrote `NULLS FIRST`/`LAST`), it defaults to `!descending` - the exact + * behavior every ORDER BY in this engine had before this clause existed, since [ValueOrdering] + * ranks NULL as the smallest value and the old code just negated that for DESC. + * + * [key]'s own [OrderKey.expression] also decides which [Collation] two [Value.TextValue]s are + * compared with, in the same two-tier precedence [kestrel.engine.Evaluator] uses for a binary + * comparison: (1) an explicit `COLLATE` - `ORDER BY name COLLATE NOCASE` reaches here as a plain + * [OrderKey] whose expression is a [CollateExpression]; (2) failing that, a bare [ColumnExpression] + * key's own declared `COLLATE` (see [kestrel.catalog.Column.collation]), reported by + * [declaredCollationOf] - `{ null }` by default so every pre-existing call site (one with no + * schema to consult, such as a UNION/INTERSECT/EXCEPT's synthetic output row) keeps behaving + * exactly as before; (3) [Collation.Binary] otherwise. + */ +fun compareOrderKeyValues( + left: Value, + right: Value, + key: OrderKey, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } +): Int { + val leftNull = left is Value.NullValue + val rightNull = right is Value.NullValue + if (leftNull || rightNull) { + if (leftNull && rightNull) return 0 + val nullsFirst = key.nullsFirst ?: !key.descending + return if (leftNull == nullsFirst) -1 else 1 + } + val collation = resolveCollation(key.expression, declaredCollationOf) + val comparison = ValueOrdering.compare(left, right, collation) + return if (key.descending) -comparison else comparison +} diff --git a/src/main/kotlin/kestrel/engine/ParameterSubstitution.kt b/src/main/kotlin/kestrel/engine/ParameterSubstitution.kt new file mode 100644 index 0000000..d0ce4c2 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/ParameterSubstitution.kt @@ -0,0 +1,371 @@ +package kestrel.engine + +import kestrel.KestrelException +import kestrel.sql.ast.Assignment +import kestrel.sql.ast.BetweenExpression +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.ConflictAction +import kestrel.sql.ast.ConflictDoNothing +import kestrel.sql.ast.ConflictDoUpdate +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FromItem +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.GroupingSpec +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.MergeWhenClause +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.OnConflictClause +import kestrel.sql.ast.OrderBy +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.PreparableStatement +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.RollupSpec +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.StarProjection +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WithQuery +import kestrel.storage.Value + +/** + * Resolves `$1`, `$2`, ... [ParameterExpression] placeholders inside a [PreparableStatement] - + * the only place they are ever actually turned into real values. Walks exactly the shape + * [kestrel.catalog.RelationRename]/[kestrel.catalog.DependencyAnalysis] already walk for a view's + * defining query (FROM/JOIN targets, derived tables, CTE bodies, set-operation branches, every + * subquery nested in WHERE/HAVING/projection/GROUP BY/ORDER BY/window frames), plus the parts only + * a top-level DML statement has that a view's body never does: INSERT's VALUES rows and ON CONFLICT + * clause, UPDATE's SET assignments, and MERGE's USING source/ON condition/WHEN clauses - since a + * prepared statement can be any [PreparableStatement] kind, not just a SELECT. + * [kestrel.engine.Executor.resolveExecute] is the only caller. + */ +internal object ParameterSubstitution { + + /** Highest 1-based parameter index anywhere in [statement] - 0 if it has none. EXECUTE must supply exactly this many arguments. */ + fun parameterCount(statement: PreparableStatement): Int = when (statement) { + is QueryStatement -> maxIndexInQuery(statement) + is InsertStatement -> maxOf( + statement.rows.maxOfOrNull { row -> row.maxOfOrNull { it?.let(::maxIndexInExpression) ?: 0 } ?: 0 } ?: 0, + statement.onConflict?.let(::maxIndexInOnConflict) ?: 0, + statement.returning?.let(::maxIndexInProjection) ?: 0 + ) + is UpdateStatement -> maxOf( + statement.assignments.maxOfOrNull { it.value?.let(::maxIndexInExpression) ?: 0 } ?: 0, + statement.where?.let(::maxIndexInExpression) ?: 0, + statement.returning?.let(::maxIndexInProjection) ?: 0 + ) + is DeleteStatement -> maxOf( + statement.where?.let(::maxIndexInExpression) ?: 0, + statement.returning?.let(::maxIndexInProjection) ?: 0 + ) + is MergeStatement -> maxOf( + maxIndexInFromItem(statement.source), + maxIndexInExpression(statement.onCondition), + statement.whenClauses.maxOfOrNull(::maxIndexInWhenClause) ?: 0, + statement.returning?.let(::maxIndexInProjection) ?: 0 + ) + } + + fun substitute(statement: PreparableStatement, boundValues: List): PreparableStatement = when (statement) { + is QueryStatement -> substituteQuery(statement, boundValues) + is InsertStatement -> statement.copy( + rows = statement.rows.map { row -> row.map { it?.let { expression -> substituteExpression(expression, boundValues) } } }, + onConflict = statement.onConflict?.let { substituteOnConflict(it, boundValues) }, + returning = statement.returning?.let { substituteProjection(it, boundValues) } + ) + is UpdateStatement -> statement.copy( + assignments = substituteAssignments(statement.assignments, boundValues), + where = statement.where?.let { substituteExpression(it, boundValues) }, + returning = statement.returning?.let { substituteProjection(it, boundValues) } + ) + is DeleteStatement -> statement.copy( + where = statement.where?.let { substituteExpression(it, boundValues) }, + returning = statement.returning?.let { substituteProjection(it, boundValues) } + ) + is MergeStatement -> statement.copy( + source = substituteFromItem(statement.source, boundValues), + onCondition = substituteExpression(statement.onCondition, boundValues), + whenClauses = statement.whenClauses.map { substituteWhenClause(it, boundValues) }, + returning = statement.returning?.let { substituteProjection(it, boundValues) } + ) + } + + private fun substituteAssignments(assignments: List, boundValues: List): List = + assignments.map { it.copy(value = it.value?.let { value -> substituteExpression(value, boundValues) }) } + + private fun substituteOnConflict(clause: OnConflictClause, boundValues: List): OnConflictClause = + clause.copy(action = substituteConflictAction(clause.action, boundValues)) + + private fun substituteConflictAction(action: ConflictAction, boundValues: List): ConflictAction = when (action) { + ConflictDoNothing -> action + is ConflictDoUpdate -> action.copy( + assignments = substituteAssignments(action.assignments, boundValues), + where = action.where?.let { substituteExpression(it, boundValues) } + ) + } + + private fun substituteWhenClause(clause: MergeWhenClause, boundValues: List): MergeWhenClause = when (clause) { + is MergeMatchedUpdate -> clause.copy( + condition = clause.condition?.let { substituteExpression(it, boundValues) }, + assignments = substituteAssignments(clause.assignments, boundValues) + ) + is MergeMatchedDelete -> clause.copy(condition = clause.condition?.let { substituteExpression(it, boundValues) }) + is MergeNotMatchedInsert -> clause.copy( + condition = clause.condition?.let { substituteExpression(it, boundValues) }, + values = clause.values.map { it?.let { expression -> substituteExpression(expression, boundValues) } } + ) + } + + private fun substituteQuery(query: QueryStatement, boundValues: List): QueryStatement = when (query) { + is SelectStatement -> substituteSelect(query, boundValues) + is SetOperationStatement -> query.copy( + left = substituteQuery(query.left, boundValues), + right = substituteQuery(query.right, boundValues), + orderBy = query.orderBy?.let { substituteOrderBy(it, boundValues) } + ) + is WithQuery -> query.copy( + ctes = query.ctes.map { it.copy(query = substituteQuery(it.query, boundValues)) }, + body = substituteQuery(query.body, boundValues) + ) + } + + private fun substituteSelect(select: SelectStatement, boundValues: List): SelectStatement = select.copy( + from = substituteFromItem(select.from, boundValues), + joins = select.joins.map { join -> + join.copy( + from = substituteFromItem(join.from, boundValues), + on = join.on?.let { substituteExpression(it, boundValues) } + ) + }, + where = select.where?.let { substituteExpression(it, boundValues) }, + groupBy = select.groupBy.map { substituteExpression(it, boundValues) }, + groupingSpec = select.groupingSpec?.let { substituteGroupingSpec(it, boundValues) }, + having = select.having?.let { substituteExpression(it, boundValues) }, + projection = substituteProjection(select.projection, boundValues), + orderBy = select.orderBy?.let { substituteOrderBy(it, boundValues) }, + distinctOn = select.distinctOn.map { substituteExpression(it, boundValues) } + ) + + private fun substituteGroupingSpec(spec: GroupingSpec, boundValues: List): GroupingSpec = when (spec) { + is RollupSpec -> spec.copy(expressions = spec.expressions.map { substituteExpression(it, boundValues) }) + is CubeSpec -> spec.copy(expressions = spec.expressions.map { substituteExpression(it, boundValues) }) + is GroupingSetsSpec -> spec.copy(sets = spec.sets.map { set -> set.map { substituteExpression(it, boundValues) } }) + } + + private fun substituteFromItem(item: FromItem, boundValues: List): FromItem = when (item) { + is TableSource -> item + is SubquerySource -> item.copy(select = substituteQuery(item.select, boundValues)) + } + + private fun substituteProjection(projection: Projection, boundValues: List): Projection = when (projection) { + StarProjection -> projection + is ExpressionProjection -> + projection.copy(items = projection.items.map { it.copy(expression = substituteExpression(it.expression, boundValues)) }) + } + + private fun substituteOrderBy(orderBy: OrderBy, boundValues: List): OrderBy = + orderBy.copy(keys = orderBy.keys.map { it.copy(expression = substituteExpression(it.expression, boundValues)) }) + + private fun substituteExpression(expression: Expression, boundValues: List): Expression = when (expression) { + is ParameterExpression -> LiteralExpression(boundValueAt(expression.index, boundValues)) + is ScalarSubqueryExpression -> expression.copy(select = substituteSelect(expression.select, boundValues)) + is ExistsExpression -> expression.copy(select = substituteSelect(expression.select, boundValues)) + is InSubqueryExpression -> expression.copy( + operand = substituteExpression(expression.operand, boundValues), + select = substituteSelect(expression.select, boundValues) + ) + is BinaryExpression -> expression.copy( + left = substituteExpression(expression.left, boundValues), + right = substituteExpression(expression.right, boundValues) + ) + is UnaryExpression -> expression.copy(operand = substituteExpression(expression.operand, boundValues)) + is CallExpression -> expression.copy(arguments = expression.arguments.map { substituteExpression(it, boundValues) }) + is IsNullExpression -> expression.copy(operand = substituteExpression(expression.operand, boundValues)) + is InExpression -> expression.copy( + operand = substituteExpression(expression.operand, boundValues), + values = expression.values.map { substituteExpression(it, boundValues) } + ) + is BetweenExpression -> expression.copy( + operand = substituteExpression(expression.operand, boundValues), + low = substituteExpression(expression.low, boundValues), + high = substituteExpression(expression.high, boundValues) + ) + is LikeExpression -> expression.copy( + operand = substituteExpression(expression.operand, boundValues), + pattern = substituteExpression(expression.pattern, boundValues) + ) + is CaseExpression -> expression.copy( + operand = expression.operand?.let { substituteExpression(it, boundValues) }, + branches = expression.branches.map { + it.copy(condition = substituteExpression(it.condition, boundValues), result = substituteExpression(it.result, boundValues)) + }, + elseExpression = expression.elseExpression?.let { substituteExpression(it, boundValues) } + ) + is CastExpression -> expression.copy(operand = substituteExpression(expression.operand, boundValues)) + is CollateExpression -> expression.copy(operand = substituteExpression(expression.operand, boundValues)) + is FunctionExpression -> expression.copy( + argument = expression.argument?.let { substituteExpression(it, boundValues) }, + separator = expression.separator?.let { substituteExpression(it, boundValues) }, + orderBy = expression.orderBy.map { it.copy(expression = substituteExpression(it.expression, boundValues)) }, + filter = expression.filter?.let { substituteExpression(it, boundValues) } + ) + is WindowExpression -> expression.copy( + argument = expression.argument?.let { substituteExpression(it, boundValues) }, + partitionBy = expression.partitionBy.map { substituteExpression(it, boundValues) }, + orderBy = expression.orderBy.map { it.copy(expression = substituteExpression(it.expression, boundValues)) }, + offset = expression.offset?.let { substituteExpression(it, boundValues) }, + defaultValue = expression.defaultValue?.let { substituteExpression(it, boundValues) }, + frame = expression.frame?.let { frame -> + frame.copy( + start = substituteFrameBound(frame.start, boundValues), + end = substituteFrameBound(frame.end, boundValues) + ) + } + ) + is ColumnExpression, is LiteralExpression, is NextValueExpression, is CurrentValueExpression, + is MergeActionExpression -> expression + } + + private fun substituteFrameBound(bound: FrameBound, boundValues: List): FrameBound = + bound.copy(offset = bound.offset?.let { substituteExpression(it, boundValues) }) + + private fun boundValueAt(index: Int, boundValues: List): Value = + boundValues.getOrNull(index - 1) ?: throw KestrelException("no value supplied for parameter \$$index") + + // --- parameterCount's twin walk: same shape as substitute above, but only tracks the max index seen. --- + + private fun maxIndexInAssignments(assignments: List): Int = + assignments.maxOfOrNull { it.value?.let(::maxIndexInExpression) ?: 0 } ?: 0 + + private fun maxIndexInOnConflict(clause: OnConflictClause): Int = when (val action = clause.action) { + ConflictDoNothing -> 0 + is ConflictDoUpdate -> maxOf(maxIndexInAssignments(action.assignments), action.where?.let(::maxIndexInExpression) ?: 0) + } + + private fun maxIndexInWhenClause(clause: MergeWhenClause): Int = when (clause) { + is MergeMatchedUpdate -> maxOf( + clause.condition?.let(::maxIndexInExpression) ?: 0, + maxIndexInAssignments(clause.assignments) + ) + is MergeMatchedDelete -> clause.condition?.let(::maxIndexInExpression) ?: 0 + is MergeNotMatchedInsert -> maxOf( + clause.condition?.let(::maxIndexInExpression) ?: 0, + clause.values.maxOfOrNull { it?.let(::maxIndexInExpression) ?: 0 } ?: 0 + ) + } + + private fun maxIndexInQuery(query: QueryStatement): Int = when (query) { + is SelectStatement -> maxIndexInSelect(query) + is SetOperationStatement -> maxOf( + maxIndexInQuery(query.left), + maxIndexInQuery(query.right), + query.orderBy?.let(::maxIndexInOrderBy) ?: 0 + ) + is WithQuery -> maxOf( + query.ctes.maxOfOrNull { maxIndexInQuery(it.query) } ?: 0, + maxIndexInQuery(query.body) + ) + } + + private fun maxIndexInSelect(select: SelectStatement): Int = maxOf( + maxIndexInFromItem(select.from), + select.joins.maxOfOrNull { join -> maxOf(maxIndexInFromItem(join.from), join.on?.let(::maxIndexInExpression) ?: 0) } ?: 0, + select.where?.let(::maxIndexInExpression) ?: 0, + select.groupBy.maxOfOrNull(::maxIndexInExpression) ?: 0, + select.groupingSpec?.let(::maxIndexInGroupingSpec) ?: 0, + select.having?.let(::maxIndexInExpression) ?: 0, + maxIndexInProjection(select.projection), + select.orderBy?.let(::maxIndexInOrderBy) ?: 0, + select.distinctOn.maxOfOrNull(::maxIndexInExpression) ?: 0 + ) + + private fun maxIndexInGroupingSpec(spec: GroupingSpec): Int = when (spec) { + is RollupSpec -> spec.expressions.maxOfOrNull(::maxIndexInExpression) ?: 0 + is CubeSpec -> spec.expressions.maxOfOrNull(::maxIndexInExpression) ?: 0 + is GroupingSetsSpec -> spec.sets.maxOfOrNull { set -> set.maxOfOrNull(::maxIndexInExpression) ?: 0 } ?: 0 + } + + private fun maxIndexInFromItem(item: FromItem): Int = when (item) { + is TableSource -> 0 + is SubquerySource -> maxIndexInQuery(item.select) + } + + private fun maxIndexInProjection(projection: Projection): Int = when (projection) { + StarProjection -> 0 + is ExpressionProjection -> projection.items.maxOfOrNull { maxIndexInExpression(it.expression) } ?: 0 + } + + private fun maxIndexInOrderBy(orderBy: OrderBy): Int = orderBy.keys.maxOfOrNull { maxIndexInExpression(it.expression) } ?: 0 + + private fun maxIndexInExpression(expression: Expression): Int = when (expression) { + is ParameterExpression -> expression.index + is ScalarSubqueryExpression -> maxIndexInSelect(expression.select) + is ExistsExpression -> maxIndexInSelect(expression.select) + is InSubqueryExpression -> maxOf(maxIndexInExpression(expression.operand), maxIndexInSelect(expression.select)) + is BinaryExpression -> maxOf(maxIndexInExpression(expression.left), maxIndexInExpression(expression.right)) + is UnaryExpression -> maxIndexInExpression(expression.operand) + is CallExpression -> expression.arguments.maxOfOrNull(::maxIndexInExpression) ?: 0 + is IsNullExpression -> maxIndexInExpression(expression.operand) + is InExpression -> maxOf( + maxIndexInExpression(expression.operand), + expression.values.maxOfOrNull(::maxIndexInExpression) ?: 0 + ) + is BetweenExpression -> maxOf( + maxIndexInExpression(expression.operand), + maxIndexInExpression(expression.low), + maxIndexInExpression(expression.high) + ) + is LikeExpression -> maxOf(maxIndexInExpression(expression.operand), maxIndexInExpression(expression.pattern)) + is CaseExpression -> maxOf( + expression.operand?.let(::maxIndexInExpression) ?: 0, + expression.branches.maxOfOrNull { maxOf(maxIndexInExpression(it.condition), maxIndexInExpression(it.result)) } ?: 0, + expression.elseExpression?.let(::maxIndexInExpression) ?: 0 + ) + is CastExpression -> maxIndexInExpression(expression.operand) + is CollateExpression -> maxIndexInExpression(expression.operand) + is FunctionExpression -> maxOf( + expression.argument?.let(::maxIndexInExpression) ?: 0, + expression.separator?.let(::maxIndexInExpression) ?: 0, + expression.orderBy.maxOfOrNull { maxIndexInExpression(it.expression) } ?: 0, + expression.filter?.let(::maxIndexInExpression) ?: 0 + ) + is WindowExpression -> maxOf( + expression.argument?.let(::maxIndexInExpression) ?: 0, + expression.partitionBy.maxOfOrNull(::maxIndexInExpression) ?: 0, + expression.orderBy.maxOfOrNull { maxIndexInExpression(it.expression) } ?: 0, + expression.offset?.let(::maxIndexInExpression) ?: 0, + expression.defaultValue?.let(::maxIndexInExpression) ?: 0, + expression.frame?.start?.offset?.let(::maxIndexInExpression) ?: 0, + expression.frame?.end?.offset?.let(::maxIndexInExpression) ?: 0 + ) + is ColumnExpression, is LiteralExpression, is NextValueExpression, is CurrentValueExpression, + is MergeActionExpression -> 0 + } +} diff --git a/src/main/kotlin/kestrel/engine/SelectRunner.kt b/src/main/kotlin/kestrel/engine/SelectRunner.kt index 6ce7ac9..a592c98 100644 --- a/src/main/kotlin/kestrel/engine/SelectRunner.kt +++ b/src/main/kotlin/kestrel/engine/SelectRunner.kt @@ -2,38 +2,379 @@ package kestrel.engine import kestrel.KestrelException import kestrel.catalog.Catalog +import kestrel.catalog.Column import kestrel.catalog.TableSchema +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CommonTableExpression +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.ExistsExpression import kestrel.sql.ast.Expression import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FrameBoundType +import kestrel.sql.ast.FrameExclude +import kestrel.sql.ast.FrameUnit +import kestrel.sql.ast.FromItem import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.IsNullExpression import kestrel.sql.ast.JoinClause import kestrel.sql.ast.JoinType +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.OrderKey +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.ScalarSubqueryExpression import kestrel.sql.ast.SelectItem import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.SetOperator import kestrel.sql.ast.StarProjection +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WindowFrame +import kestrel.sql.ast.WindowFunctionKind +import kestrel.sql.ast.WithQuery +import kestrel.storage.Collation +import kestrel.storage.ColumnType import kestrel.storage.Row import kestrel.storage.Value -import kestrel.storage.ValueOrdering +import kestrel.storage.displayWith +import java.math.BigDecimal class SelectRunner(private val catalog: Catalog) { private val indexPlanner = IndexAccessPlanner(catalog) + private val subqueryRunner = SubqueryRunner { select, outer -> run(select, outer) } + private val expandingViews = mutableSetOf() + private val activeCtes = mutableListOf() - fun run(statement: SelectStatement): SelectResult { - val plan = buildPlan(statement) + fun run(statement: SelectStatement, outer: ColumnResolver? = null): SelectResult { + val plan = buildPlan(statement, outer) val combined = plan.materialize() return if (plan.aggregated) plan.aggregate(combined) else plan.projectRows(combined) } - fun explain(statement: SelectStatement): List = buildPlan(statement).describe() + fun runQuery(query: QueryStatement, outer: ColumnResolver? = null): SelectResult = when (query) { + is SelectStatement -> run(query, outer) + is SetOperationStatement -> runSetOperation(query, outer) + is WithQuery -> runWith(query, outer) + } - private fun buildPlan(statement: SelectStatement): Plan { + fun explain(query: QueryStatement): List = when (query) { + is SelectStatement -> buildPlan(query, null).describe() + is SetOperationStatement -> explainSetOperation(query) + is WithQuery -> explainWith(query) + } + + /** + * Resolves a single [FromItem] to its schema and row snapshot - the same lookup a `FROM`/`JOIN` + * source uses ([resolveSource]), exposed for [Executor.executeMerge]'s `USING` clause so a MERGE + * source can be either a real table or a derived-table subquery without duplicating view/CTE/ + * subquery resolution. + */ + fun resolveMergeSource(item: FromItem): Pair> = resolveSource(item) + + /** The name a MERGE `ON`/`WHEN` condition would qualify a column from [item] with - its alias if given, its bare table name otherwise (a subquery source always has one). */ + fun mergeSourceLabel(item: FromItem): String = sourceLabel(item) + + private fun explainSetOperation(statement: SetOperationStatement): List { + val header = setOperatorKeyword(statement.operator) + val left = explain(statement.left).map { " -> $it" } + val right = explain(statement.right).map { " -> $it" } + var lines = listOf(header) + left + right + statement.orderBy?.let { lines = listOf("Sort") + lines.map { line -> " $line" } } + statement.offset?.let { lines = listOf("Offset ${statement.offset}") + lines.map { line -> " $line" } } + statement.limit?.let { lines = listOf("Limit ${statement.limit}") + lines.map { line -> " $line" } } + return lines + } + + private fun explainWith(query: WithQuery): List { + var pushed = 0 + try { + for (cte in query.ctes) { + activeCtes.add(buildCteScope(query.recursive, cte)) + pushed++ + } + val cteLines = query.ctes.flatMap { cte -> + listOf("CTE ${cte.name}") + explain(cte.query).map { " -> $it" } + } + return cteLines + explain(query.body) + } finally { + repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) } + } + } + + private fun setOperatorKeyword(operator: SetOperator): String = when (operator) { + SetOperator.Union -> "Union" + SetOperator.UnionAll -> "Union All" + SetOperator.Intersect -> "Intersect" + SetOperator.IntersectAll -> "Intersect All" + SetOperator.Except -> "Except" + SetOperator.ExceptAll -> "Except All" + } + + private fun runSetOperation(statement: SetOperationStatement, outer: ColumnResolver?): SelectResult { + val left = runQuery(statement.left, outer) + val right = runQuery(statement.right, outer) + if (left.columns.size != right.columns.size) { + throw KestrelException( + "${setOperatorKeyword(statement.operator)} queries must have the same number of columns " + + "but got ${left.columns.size} and ${right.columns.size}" + ) + } + val declaredCollationOf = declaredSetOperationCollation(statement, left.columns) + val outputExpressions = left.columns.map { ColumnExpression(it) } + fun foldedKey(row: List) = collationFoldedKey(row, outputExpressions, declaredCollationOf) + var rows = when (statement.operator) { + SetOperator.UnionAll -> left.rows + right.rows + SetOperator.Union -> (left.rows + right.rows).distinctBy(::foldedKey) + SetOperator.Intersect -> { + val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey) + left.rows.filter { foldedKey(it) in rightKeys }.distinctBy(::foldedKey) + } + SetOperator.Except -> { + val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey) + left.rows.filter { foldedKey(it) !in rightKeys }.distinctBy(::foldedKey) + } + SetOperator.IntersectAll -> multisetFilter(left.rows, right.rows, keepOnMatch = true, keyOf = ::foldedKey) + SetOperator.ExceptAll -> multisetFilter(left.rows, right.rows, keepOnMatch = false, keyOf = ::foldedKey) + } + val orderKeys = statement.orderBy?.keys.orEmpty() + if (orderKeys.isNotEmpty()) { + rows = rows.sortedWith( + multiKeyComparator( + orderKeys, + evaluate = { expression, row -> evaluateOutputRow(left.columns, expression, row) }, + declaredCollationOf = declaredCollationOf + ) + ) + } + statement.offset?.let { if (it < 0) throw KestrelException("OFFSET must not be negative") } + statement.limit?.let { if (it < 0) throw KestrelException("LIMIT must not be negative") } + if (statement.offset != null || statement.limit != null) { + rows = rows.drop(statement.offset ?: 0) + statement.limit?.let { rows = rows.take(it) } + } + return SelectResult(left.columns, rows) + } + + /** + * The declared `COLLATE` of [statement]'s combined output column named by an ORDER BY key + * expression (an unqualified [ColumnExpression] - the only kind [evaluateOutputRow] resolves + * for a set operation's own ORDER BY, since it rejects any qualifier), traced back to a real + * table column in [statement]'s LEFT branch only - the branch [SelectResult.columns] itself + * always takes column identity from (`runSetOperation` returns `SelectResult(left.columns, + * ...)`, never consulting the right side's names). Resolves only when that branch is a plain + * `SELECT ... FROM realTable [JOIN realTable ...]` (no CTE, view, or derived table anywhere in + * its FROM clause) referencing the column bare - going further would mean inheriting collation + * through a view/CTE/derived-table boundary, the still-open scope-cut already left by #65/#66 + * for a plain SELECT's own WHERE/ORDER BY over such a source (see kestrel-stan.md), not + * extended here. + */ + private fun declaredSetOperationCollation(statement: SetOperationStatement, columns: List): (ColumnExpression) -> Collation? = + (statement.left as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null } + + /** + * The same "plain `SELECT ... FROM realTable [JOIN realTable ...]`, no CTE/view/derived table" + * structural resolution [declaredSetOperationCollation] does for a set operation's LEFT branch, + * factored out so [declaredRecursiveCteCollation] can apply the identical rule to a recursive + * CTE's own anchor (base) term - the branch that owns [CteScope.schema]/column identity there, + * exactly like the LEFT branch owns [SelectResult.columns] for a plain set operation. + */ + private fun declaredCollationForSelectOutput(select: SelectStatement, columns: List): (ColumnExpression) -> Collation? { + val items = listOf(select.from) + select.joins.map { it.from } + val sources = items.map { item -> + (item as? TableSource)?.let { source -> catalog.tableOrNull(source.name)?.schema?.let { (source.alias ?: source.name) to it } } + } + if (sources.any { it == null }) return { null } + val resolvedSources = sources.filterNotNull() + fun columnCollation(expression: ColumnExpression): Collation? { + val matches = resolvedSources + .filter { (alias, _) -> expression.qualifier == null || alias.equals(expression.qualifier, ignoreCase = true) } + .mapNotNull { (_, schema) -> schema.column(expression.name) } + return matches.singleOrNull()?.collation + } + return when (val projection = select.projection) { + is ExpressionProjection -> outputExpr@{ outputExpr -> + val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) } + val itemExpression = index.takeIf { it in projection.items.indices }?.let { projection.items[it].expression } + (itemExpression as? ColumnExpression)?.let(::columnCollation) + } + is StarProjection -> outputExpr@{ outputExpr -> + val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) } + if (index < 0) return@outputExpr null + var remaining = index + for ((_, schema) in resolvedSources) { + if (remaining < schema.columns.size) return@outputExpr schema.columns[remaining].collation + remaining -= schema.columns.size + } + null + } + } + } + + /** Mirrors [declaredSetOperationCollation] but resolves against a recursive CTE's anchor (`base`) term. */ + private fun declaredRecursiveCteCollation(base: QueryStatement, columns: List): (ColumnExpression) -> Collation? = + (base as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null } + + private fun multisetFilter( + left: List>, + right: List>, + keepOnMatch: Boolean, + keyOf: (List) -> List + ): List> { + val remainingInRight = right.groupingBy(keyOf).eachCount().toMutableMap() + return left.filter { row -> + val key = keyOf(row) + val remaining = remainingInRight[key] ?: 0 + if (remaining > 0) { + remainingInRight[key] = remaining - 1 + keepOnMatch + } else { + !keepOnMatch + } + } + } + + private fun multiKeyComparator( + keys: List, + evaluate: (Expression, List) -> Value, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } + ): Comparator> = Comparator { first, second -> + var result = 0 + for (key in keys) { + val comparison = compareOrderKeyValues( + evaluate(key.expression, first), evaluate(key.expression, second), key, declaredCollationOf + ) + if (comparison != 0) { + result = comparison + break + } + } + result + } + + private fun evaluateOutputRow(columns: List, expression: Expression, row: List): Value { + val resolver = kestrel.engine.ColumnResolver { column -> + if (column.qualifier != null) throw KestrelException("no such column: ${column.qualifier}.${column.name}") + val index = columns.indexOfFirst { it.equals(column.name, ignoreCase = true) } + if (index == -1) throw KestrelException("no such column: ${column.name}") + row[index] + } + return Evaluator(resolver, subqueryRunner).evaluate(expression) + } + + private fun runWith(query: WithQuery, outer: ColumnResolver?): SelectResult { + var pushed = 0 + try { + for (cte in query.ctes) { + activeCtes.add(buildCteScope(query.recursive, cte)) + pushed++ + } + return runQuery(query.body, outer) + } finally { + repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) } + } + } + + private fun buildCteScope(recursive: Boolean, cte: CommonTableExpression): CteScope { + val recursivePlan = if (recursive) splitRecursiveCte(cte) else null + if (recursivePlan == null) { + val result = runQuery(cte.query, null) + return CteScope(cte.name, cteSchema(cte, result), result.rows.map { Row(it) }) + } + val (base, operator, recursiveTerm) = recursivePlan + val baseResult = runQuery(base, null) + val schema = cteSchema(cte, baseResult) + val scope = CteScope(cte.name, schema, baseResult.rows.map { Row(it) }) + val declaredCollationOf = declaredRecursiveCteCollation(base, baseResult.columns) + val outputExpressions = baseResult.columns.map { ColumnExpression(it) } + val seenKeys = if (operator == SetOperator.Union) + baseResult.rows.mapTo(HashSet()) { collationFoldedKey(it, outputExpressions, declaredCollationOf) } + else null + activeCtes.add(scope) + try { + val accumulated = mutableListOf>() + accumulated.addAll(baseResult.rows) + var working = baseResult.rows + var iterations = 0 + while (working.isNotEmpty()) { + iterations++ + if (iterations > MAX_RECURSION_ITERATIONS) { + throw KestrelException( + "recursive common table expression '${cte.name}' did not terminate within $MAX_RECURSION_ITERATIONS iterations" + ) + } + scope.rows = working.map { Row(it) } + var newRows = runQuery(recursiveTerm, null).rows + if (operator == SetOperator.Union) { + newRows = newRows.filter { candidate -> seenKeys!!.add(collationFoldedKey(candidate, outputExpressions, declaredCollationOf)) } + } + if (newRows.isEmpty()) break + accumulated.addAll(newRows) + working = newRows + } + scope.rows = accumulated.map { Row(it) } + } finally { + activeCtes.removeAt(activeCtes.size - 1) + } + return scope + } + + private data class RecursiveCtePlan(val base: QueryStatement, val operator: SetOperator, val recursiveTerm: QueryStatement) + + private fun splitRecursiveCte(cte: CommonTableExpression): RecursiveCtePlan? { + val query = cte.query + if (query !is SetOperationStatement) return null + if (query.operator != SetOperator.Union && query.operator != SetOperator.UnionAll) return null + val leftRefs = referencesName(query.left, cte.name) + val rightRefs = referencesName(query.right, cte.name) + return when { + leftRefs && rightRefs -> + throw KestrelException("recursive common table expression '${cte.name}' must not reference itself on both sides of ${query.operator}") + rightRefs -> RecursiveCtePlan(query.left, query.operator, query.right) + leftRefs -> RecursiveCtePlan(query.right, query.operator, query.left) + else -> null + } + } + + private fun referencesName(query: QueryStatement, name: String): Boolean = when (query) { + is SelectStatement -> referencesName(query.from, name) || query.joins.any { referencesName(it.from, name) } + is SetOperationStatement -> referencesName(query.left, name) || referencesName(query.right, name) + is WithQuery -> query.ctes.none { it.name.equals(name, ignoreCase = true) } && referencesName(query.body, name) + } + + private fun referencesName(item: FromItem, name: String): Boolean = when (item) { + is TableSource -> item.name.equals(name, ignoreCase = true) + is SubquerySource -> referencesName(item.select, name) + } + + private fun cteSchema(cte: CommonTableExpression, result: SelectResult): TableSchema { + val labels = cte.columns ?: result.columns + if (labels.size != result.columns.size) { + throw KestrelException( + "common table expression '${cte.name}' expects ${labels.size} columns but its query produces ${result.columns.size}" + ) + } + return derivedSchema(cte.name, result, labels) + } + + private fun buildPlan(statement: SelectStatement, outer: ColumnResolver?): Plan { val sources = buildSources(statement) - val plan = Plan(statement, sources) + val plan = Plan(statement, sources, outer) plan.validate() return plan } @@ -41,23 +382,117 @@ class SelectRunner(private val catalog: Catalog) { private fun buildSources(statement: SelectStatement): List { val sources = mutableListOf() var offset = 0 - fun add(tableName: String, alias: String?) { - val table = catalog.table(tableName) - val chosen = alias ?: tableName + fun add(item: FromItem) { + // A LATERAL derived table's rows depend on the row of an earlier FROM item it + // correlates with, so they cannot be resolved once up front like every other source - + // only its column *names* are needed here (for offset bookkeeping and validating + // references to its alias elsewhere in the query), computed structurally from the + // subquery's own AST without running it. Real rows are produced per left row in + // [executeJoin]. + val (schema, rows) = if (item is SubquerySource && item.lateral) { + lateralSchema(item) to emptyList() + } else { + resolveSource(item) + } + val chosen = sourceLabel(item) if (sources.any { it.alias.equals(chosen, ignoreCase = true) }) { throw KestrelException("duplicate table alias: $chosen") } - sources.add(Source(chosen, table.schema.name, table.schema, table.rows, offset)) - offset += table.schema.columns.size + sources.add(Source(chosen, schema.name, schema, rows, offset)) + offset += schema.columns.size } - add(statement.tableName, statement.tableAlias) - for (join in statement.joins) add(join.tableName, join.alias) + add(statement.from) + for (join in statement.joins) add(join.from) return sources } - private inner class Plan(val statement: SelectStatement, val sources: List) { + private fun lateralSchema(item: SubquerySource): TableSchema { + val labels = structuralColumnNames(item.select) + val columns = labels.map { Column(it, ColumnType.Text, notNull = false, primaryKey = false) } + return TableSchema(item.alias, columns) + } + + /** + * The output column names of [query], derived purely from its AST (no execution, no catalog + * row data) - needed to give a LATERAL derived table a stable schema before any row of the + * source it correlates with is known. Column names never depend on data (aliases and bare + * column references are syntactic, and a table/view/CTE's shape is fixed), so this is safe + * even though the real rows can only be produced later, per outer row. + */ + private fun structuralColumnNames(query: QueryStatement, localCtes: Map> = emptyMap()): List = + when (query) { + is SelectStatement -> when (val projection = query.projection) { + is StarProjection -> + structuralSourceColumnNames(query.from, localCtes) + + query.joins.flatMap { structuralSourceColumnNames(it.from, localCtes) } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> columnLabel(item, index) } + } + is SetOperationStatement -> structuralColumnNames(query.left, localCtes) + is WithQuery -> { + val extended = localCtes.toMutableMap() + for (cte in query.ctes) { + extended[cte.name.lowercase()] = cte.columns ?: structuralColumnNames(cte.query, extended) + } + structuralColumnNames(query.body, extended) + } + } + + private fun structuralSourceColumnNames(item: FromItem, localCtes: Map>): List = + when (item) { + is TableSource -> { + val key = item.name.lowercase() + localCtes[key] + ?: activeCtes.lastOrNull { it.name.equals(item.name, ignoreCase = true) }?.schema?.columnNames + ?: catalog.tableOrNull(item.name)?.schema?.columnNames + ?: catalog.viewOrNull(item.name)?.let { structuralColumnNames(it.query, localCtes) } + ?: throw KestrelException("no such table: ${item.name}") + } + is SubquerySource -> structuralColumnNames(item.select, localCtes) + } + + private fun sourceLabel(item: FromItem): String = when (item) { + is TableSource -> item.alias ?: item.name + is SubquerySource -> item.alias + } + + private fun resolveSource(item: FromItem): Pair> = when (item) { + is TableSource -> resolveNamedSource(item.name) + is SubquerySource -> { + val result = runQuery(item.select) + derivedSchema(item.alias, result) to result.rows.map { Row(it) } + } + } + + private fun resolveNamedSource(name: String): Pair> { + activeCtes.lastOrNull { it.name.equals(name, ignoreCase = true) }?.let { return it.schema to it.rows } + catalog.tableOrNull(name)?.let { return it.schema to it.rows } + val view = catalog.viewOrNull(name) ?: throw KestrelException("no such table: $name") + val key = name.lowercase() + if (!expandingViews.add(key)) { + throw KestrelException("recursive view definition: $name") + } + try { + val result = runQuery(view.query) + return derivedSchema(view.name, result) to result.rows.map { Row(it) } + } finally { + expandingViews.remove(key) + } + } + + private fun derivedSchema(name: String, result: SelectResult, labels: List = result.columns): TableSchema { + val columns = labels.mapIndexed { index, columnName -> + val type = result.rows.asSequence().map { it[index] }.firstOrNull { !it.isNull }?.type ?: ColumnType.Text + Column(columnName, type, notNull = false, primaryKey = false) + } + return TableSchema(name, columns) + } + + private class CteScope(val name: String, val schema: TableSchema, var rows: List) + + private inner class Plan(val statement: SelectStatement, val sources: List, val outer: ColumnResolver?) { val aggregated: Boolean = statement.groupBy.isNotEmpty() || + statement.groupingSpec != null || containsAggregate(statement.projection) || (statement.having != null && containsAggregate(statement.having)) @@ -68,7 +503,8 @@ class SelectRunner(private val catalog: Catalog) { ?.mapNotNull { item -> item.alias?.let { it.lowercase() to item.expression } } ?.toMap() ?: emptyMap() - private val orderExpression: Expression? = statement.orderBy?.let { resolveOrderAlias(it.expression) } + private val orderKeys: List = + statement.orderBy?.keys.orEmpty().map { OrderKey(resolveOrderAlias(it.expression), it.descending, it.nullsFirst) } private fun resolveOrderAlias(expression: Expression): Expression { if (expression is ColumnExpression && expression.qualifier == null) { @@ -77,7 +513,19 @@ class SelectRunner(private val catalog: Catalog) { return expression } - fun resolveIndex(expression: ColumnExpression): Int { + /** + * Every distinct window-function call in the SELECT list or (post alias-resolution) ORDER + * BY - the only two clauses where one is allowed. Computed once per [Plan] and reused both + * to append the values ([appendWindowColumns]) and to substitute them back in ([evaluateRow]). + */ + private val windowExpressions: List = run { + val projectionWindows = (statement.projection as? ExpressionProjection)?.items + ?.flatMap { collectWindowExpressions(it.expression) } ?: emptyList() + val orderWindows = orderKeys.flatMap { collectWindowExpressions(it.expression) } + (projectionWindows + orderWindows).distinct() + } + + fun resolveIndexOrNull(expression: ColumnExpression): Int? { val qualifier = expression.qualifier val matches = mutableListOf() for (source in sources) { @@ -86,12 +534,32 @@ class SelectRunner(private val catalog: Catalog) { matches.add(source.offset + local) } return when { - matches.isEmpty() -> throw KestrelException("no such column: ${describe(expression)}") + matches.isEmpty() -> null matches.size > 1 -> throw KestrelException("ambiguous column reference: ${describe(expression)}") else -> matches.single() } } + fun resolveIndex(expression: ColumnExpression): Int = + resolveIndexOrNull(expression) ?: throw KestrelException("no such column: ${describe(expression)}") + + /** + * [expression]'s declared `COLLATE` (its [kestrel.catalog.Column.collation]) from whichever + * [sources] entry it resolves against - `null` when it names no column here (ambiguous + * across sources, same as [resolveIndexOrNull], resolves to nothing at all, or the matching + * column simply declared no COLLATE of its own). + */ + fun declaredCollationOrNull(expression: ColumnExpression): Collation? { + val qualifier = expression.qualifier + val matches = mutableListOf() + for (source in sources) { + if (qualifier != null && !source.alias.equals(qualifier, ignoreCase = true)) continue + val column = source.schema.column(expression.name) ?: continue + matches.add(column.collation) + } + return matches.singleOrNull() + } + private fun sourceOfGlobal(global: Int): Int = sources.indexOfLast { global >= it.offset } @@ -99,25 +567,89 @@ class SelectRunner(private val catalog: Catalog) { collectColumns(expression).map { sourceOfGlobal(resolveIndex(it)) }.toSet() fun validate() { - statement.where?.let { validateColumns(it) } - orderExpression?.let { validateColumns(it) } - statement.groupBy.forEach { validateColumns(it) } - statement.having?.let { validateColumns(it) } + statement.where?.let { validateColumns(it); rejectWindowFunction(it, "WHERE") } + orderKeys.forEach { validateColumns(it.expression) } + statement.groupBy.forEach { validateColumns(it); rejectWindowFunction(it, "GROUP BY") } + statement.having?.let { validateColumns(it); rejectWindowFunction(it, "HAVING") } if (statement.projection is ExpressionProjection) { statement.projection.items.forEach { validateColumns(it.expression) } } statement.joins.forEachIndexed { position, join -> - validateColumns(join.on) + // CROSS JOIN has no ON condition at all (its `on` is null) - nothing to validate. + val on = join.on ?: return@forEachIndexed + validateColumns(on) + rejectWindowFunction(on, "an ON condition") val allowed = position + 1 - if (referencedSources(join.on).any { it > allowed }) { - throw KestrelException("ON condition for ${join.tableName} references a table joined later") + if (referencedSources(on).any { it > allowed }) { + throw KestrelException("ON condition for ${sourceLabel(join.from)} references a table joined later") } } if (aggregated) validateAggregation() + if (windowExpressions.isNotEmpty()) validateWindows() + if (statement.distinctOn.isNotEmpty()) validateDistinctOn() + } + + /** + * `DISTINCT ON (...)` has no unsurprising meaning once rows are already collapsed into + * aggregate groups - rejected outright rather than guessing which pre-aggregate row a group + * should be "first" for. The leading-`ORDER BY` requirement mirrors Postgres: `DISTINCT ON` + * exists to pick a deterministic row per group, which only means something if `ORDER BY` + * (when present at all) agrees on what "first" means for those same expressions. + */ + private fun validateDistinctOn() { + statement.distinctOn.forEach { validateColumns(it); rejectWindowFunction(it, "DISTINCT ON") } + if (aggregated) { + throw KestrelException("DISTINCT ON cannot be combined with GROUP BY or aggregate functions") + } + if (statement.orderBy != null) { + val leading = orderKeys.take(statement.distinctOn.size).map { it.expression } + if (leading != statement.distinctOn) { + throw KestrelException("SELECT DISTINCT ON expressions must match initial ORDER BY expressions") + } + } + } + + private fun rejectWindowFunction(expression: Expression, clause: String) { + if (containsWindowFunction(expression)) { + throw KestrelException("window functions are not allowed in $clause") + } + } + + /** + * Window functions are rejected outright alongside GROUP BY/aggregates ([aggregated]) - + * windowing over an already-grouped result is valid SQL but requires running GroupAggregation + * first and then a second pass over its output rows, which this engine does not implement. + * RANK/DENSE_RANK need an explicit tie-break (unlike ROW_NUMBER, which falls back to this + * partition's row arrival order when OVER has no ORDER BY) and nesting one window function + * inside another's argument/PARTITION BY/ORDER BY is invalid SQL in the first place. + */ + private fun validateWindows() { + if (aggregated) { + throw KestrelException("window functions cannot be combined with GROUP BY or aggregate functions in the same query") + } + for (window in windowExpressions) { + if ((window.kind == WindowFunctionKind.Rank || window.kind == WindowFunctionKind.DenseRank) && window.orderBy.isEmpty()) { + throw KestrelException("${window.name}() OVER (...) requires an ORDER BY clause") + } + if (window.frame != null && window.kind !in FRAME_CAPABLE_KINDS) { + throw KestrelException("${window.name}() cannot be used with a ROWS/RANGE/GROUPS frame clause") + } + val nested = listOfNotNull( + window.argument, window.offset, window.defaultValue, + window.frame?.start?.offset, window.frame?.end?.offset + ) + window.partitionBy + window.orderBy.map { it.expression } + if (nested.any { containsWindowFunction(it) }) { + throw KestrelException("window functions cannot be nested") + } + } } private fun validateColumns(expression: Expression) { - collectColumns(expression).forEach { resolveIndex(it) } + collectColumns(expression).forEach { column -> + if (resolveIndexOrNull(column) == null && outer == null) { + throw KestrelException("no such column: ${describe(column)}") + } + } } private fun validateAggregation() { @@ -130,10 +662,11 @@ class SelectRunner(private val catalog: Catalog) { statement.projection.items.forEach { requireGrouped(it.expression, groupColumns) } } statement.having?.let { requireGrouped(it, groupColumns) } - orderExpression?.let { requireGrouped(it, groupColumns) } + orderKeys.forEach { requireGrouped(it.expression, groupColumns) } } private fun requireGrouped(expression: Expression, groupColumns: Set) { + if (expression in statement.groupBy) return when (expression) { is FunctionExpression -> Unit is ColumnExpression -> if (resolveIndex(expression) !in groupColumns) { @@ -147,14 +680,107 @@ class SelectRunner(private val catalog: Catalog) { } is UnaryExpression -> requireGrouped(expression.operand, groupColumns) is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { requireGrouped(it, groupColumns) } + is IsNullExpression -> requireGrouped(expression.operand, groupColumns) + is InExpression -> { + requireGrouped(expression.operand, groupColumns) + expression.values.forEach { requireGrouped(it, groupColumns) } + } + is BetweenExpression -> { + requireGrouped(expression.operand, groupColumns) + requireGrouped(expression.low, groupColumns) + requireGrouped(expression.high, groupColumns) + } + is LikeExpression -> { + requireGrouped(expression.operand, groupColumns) + requireGrouped(expression.pattern, groupColumns) + } + is CaseExpression -> { + expression.operand?.let { requireGrouped(it, groupColumns) } + expression.branches.forEach { + requireGrouped(it.condition, groupColumns) + requireGrouped(it.result, groupColumns) + } + expression.elseExpression?.let { requireGrouped(it, groupColumns) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> requireGrouped(expression.operand, groupColumns) + is CastExpression -> requireGrouped(expression.operand, groupColumns) + is CollateExpression -> requireGrouped(expression.operand, groupColumns) + // Unreachable: validateWindows() rejects mixing window functions with GROUP BY/ + // aggregates before requireGrouped ever runs, so this branch only keeps `when` exhaustive. + is WindowExpression -> Unit + // Unreachable in practice: NEXTVAL/CURRVAL/MERGE_ACTION() are never legal in a SELECT + // at all (MERGE's RETURNING is evaluated by Executor.evaluateMergeReturning, an + // entirely separate projection path from this one) - kept here only to keep `when` + // exhaustive. + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression -> Unit + // A parameter reference behaves like any other constant here - `SELECT $1, SUM(x) + // FROM t GROUP BY y` needs $1 in GROUP BY no more than a literal would. + is ParameterExpression -> Unit } } - private fun rowResolver(row: List) = - kestrel.engine.ColumnResolver { expression -> row[resolveIndex(expression)] } + private fun rowResolver(row: List) = object : kestrel.engine.ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val index = resolveIndexOrNull(expression) + return if (index != null) { + row[index] + } else { + (outer ?: throw KestrelException("no such column: ${describe(expression)}")).resolve(expression) + } + } - private fun evaluateRow(expression: Expression, row: List): Value = - Evaluator(rowResolver(row)).evaluate(expression) + override fun declaredCollation(expression: ColumnExpression): Collation? = + declaredCollationOrNull(expression) ?: outer?.declaredCollation(expression) + } + + /** + * When [windowExpressions] is non-empty, a [row] handed in here may carry extra trailing + * values appended by [appendWindowColumns] (one per entry of [windowExpressions], in order, + * right after the real columns) - [substituteWindowValues] rewrites any window-function node + * in [expression] into a [LiteralExpression] reading straight from that trailing slot before + * the plain [Evaluator] ever sees it, the same trick [GroupAggregation] uses for aggregates. + */ + private fun evaluateRow(expression: Expression, row: List): Value { + val prepared = if (windowExpressions.isEmpty()) expression else substituteWindowValues(expression, row) + return Evaluator(rowResolver(row), subqueryRunner).evaluate(prepared) + } + + private fun substituteWindowValues(expression: Expression, row: List): Expression = when (expression) { + is WindowExpression -> LiteralExpression(row[columnNames.size + windowExpressions.indexOf(expression)]) + is BinaryExpression -> expression.copy( + left = substituteWindowValues(expression.left, row), + right = substituteWindowValues(expression.right, row) + ) + is UnaryExpression -> expression.copy(operand = substituteWindowValues(expression.operand, row)) + is CallExpression -> expression.copy(arguments = expression.arguments.map { substituteWindowValues(it, row) }) + is IsNullExpression -> expression.copy(operand = substituteWindowValues(expression.operand, row)) + is InExpression -> expression.copy( + operand = substituteWindowValues(expression.operand, row), + values = expression.values.map { substituteWindowValues(it, row) } + ) + is BetweenExpression -> expression.copy( + operand = substituteWindowValues(expression.operand, row), + low = substituteWindowValues(expression.low, row), + high = substituteWindowValues(expression.high, row) + ) + is LikeExpression -> expression.copy( + operand = substituteWindowValues(expression.operand, row), + pattern = substituteWindowValues(expression.pattern, row) + ) + is CaseExpression -> expression.copy( + operand = expression.operand?.let { substituteWindowValues(it, row) }, + branches = expression.branches.map { + it.copy(condition = substituteWindowValues(it.condition, row), result = substituteWindowValues(it.result, row)) + }, + elseExpression = expression.elseExpression?.let { substituteWindowValues(it, row) } + ) + is CastExpression -> expression.copy(operand = substituteWindowValues(expression.operand, row)) + is CollateExpression -> expression.copy(operand = substituteWindowValues(expression.operand, row)) + else -> expression + } fun materialize(): List> { var combined = primaryRows().map { it.values } @@ -164,7 +790,412 @@ class SelectRunner(private val catalog: Catalog) { statement.where?.let { predicate -> combined = combined.filter { isTrue(evaluateRow(predicate, it)) } } - return combined + return appendWindowColumns(combined) + } + + /** + * Runs after FROM/JOIN/WHERE and (per [validateWindows]) never alongside GROUP BY, matching + * standard SQL's logical order of window-function evaluation. Each entry of + * [windowExpressions] contributes one trailing value per row, computed once over the whole + * (already filtered) row set - [evaluateRow]/[substituteWindowValues] read them back out by + * position for the final projection and this query's own ORDER BY. + */ + private fun appendWindowColumns(combined: List>): List> { + if (windowExpressions.isEmpty()) return combined + val perExpression = windowExpressions.map { computeWindowValues(it, combined) } + return combined.mapIndexed { rowIndex, row -> row + windowExpressions.indices.map { perExpression[it][rowIndex] } } + } + + private fun computeWindowValues(window: WindowExpression, combined: List>): List { + val values = arrayOfNulls(combined.size) + val partitions = combined.indices.groupBy { index -> window.partitionBy.map { evaluateRow(it, combined[index]) } } + for (memberIndices in partitions.values) { + val ordered = if (window.orderBy.isEmpty()) { + memberIndices + } else { + memberIndices.sortedWith(Comparator { first, second -> + var result = 0 + for (key in window.orderBy) { + val comparison = compareOrderKeyValues( + evaluateRow(key.expression, combined[first]), + evaluateRow(key.expression, combined[second]), + key, + ::declaredCollationOrNull + ) + if (comparison != 0) { + result = comparison + break + } + } + result + }) + } + assignWindowValues(window, ordered, combined, values) + } + return values.map { it ?: error("window value not computed for every row") } + } + + private fun assignWindowValues(window: WindowExpression, ordered: List, combined: List>, out: Array) { + when (window.kind) { + WindowFunctionKind.RowNumber -> + ordered.forEachIndexed { position, index -> out[index] = Value.IntegerValue((position + 1).toLong()) } + WindowFunctionKind.Rank, WindowFunctionKind.DenseRank -> { + var rank = 0 + var denseRank = 0 + var previousKey: List? = null + ordered.forEachIndexed { position, index -> + val key = window.orderBy.map { evaluateRow(it.expression, combined[index]) } + if (previousKey == null || key != previousKey) { + rank = position + 1 + denseRank++ + previousKey = key + } + out[index] = Value.IntegerValue((if (window.kind == WindowFunctionKind.Rank) rank else denseRank).toLong()) + } + } + WindowFunctionKind.Aggregate -> assignAggregateWindowValues(window, ordered, combined, out) + WindowFunctionKind.Lag -> assignOffsetWindowValues(window, ordered, combined, out, direction = -1) + WindowFunctionKind.Lead -> assignOffsetWindowValues(window, ordered, combined, out, direction = 1) + WindowFunctionKind.FirstValue -> { + val frame = window.frame + if (frame == null) { + val firstValue = evaluateRow(window.argument!!, combined[ordered.first()]) + ordered.forEach { index -> out[index] = firstValue } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined) + out[ordered[position]] = + if (members.isEmpty()) Value.NullValue else evaluateRow(window.argument!!, combined[ordered[members.first()]]) + } + } + } + WindowFunctionKind.LastValue -> { + val frame = window.frame + if (frame == null) { + for (range in peerGroupRanges(window.orderBy, ordered, combined)) { + val value = evaluateRow(window.argument!!, combined[ordered[range.last]]) + for (position in range) out[ordered[position]] = value + } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined) + out[ordered[position]] = + if (members.isEmpty()) Value.NullValue else evaluateRow(window.argument!!, combined[ordered[members.last()]]) + } + } + } + WindowFunctionKind.Ntile -> assignNtileWindowValues(window, ordered, combined, out) + } + } + + /** + * With no ORDER BY in OVER, the whole partition is one frame (every row gets the same + * value). With an ORDER BY, the default SQL frame is `RANGE BETWEEN UNBOUNDED PRECEDING AND + * CURRENT ROW`: a running aggregate where rows tied on the order key (a "peer group") share + * one value - the aggregate through the end of their peer group, not through their own row + * alone. A single-column running total (`ORDER BY` with no ties) falls out of this as the + * special case where every peer group has size 1. + */ + private fun assignAggregateWindowValues(window: WindowExpression, ordered: List, combined: List>, out: Array) { + val frame = window.frame + if (frame == null) { + for (range in peerGroupRanges(window.orderBy, ordered, combined)) { + val value = aggregateWindowValue(window, (0..range.last).map { combined[ordered[it]] }) + for (position in range) out[ordered[position]] = value + } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined) + out[ordered[position]] = aggregateWindowValue(window, members.map { combined[ordered[it]] }) + } + } + } + + /** + * The frame's base [frameRange], narrowed by its [FrameExclude] mode (default [FrameExclude.NoOthers], + * which leaves the range untouched). Excluding rows can split a contiguous range into two + * pieces (e.g. `EXCLUDE CURRENT ROW` on a frame that spans the current row), so this returns + * the actual member positions rather than an [IntRange]. + */ + private fun frameMembers(window: WindowExpression, position: Int, ordered: List, combined: List>): List { + val range = frameRange(window, position, ordered, combined) + if (range.isEmpty()) return emptyList() + return when (window.frame!!.exclude) { + FrameExclude.NoOthers -> range.toList() + FrameExclude.CurrentRow -> range.filter { it != position } + FrameExclude.Group -> { + val peer = peerBoundary(window.orderBy, position, ordered, combined, isStart = true).. + peerBoundary(window.orderBy, position, ordered, combined, isStart = false) + range.filter { it !in peer } + } + FrameExclude.Ties -> { + val peer = peerBoundary(window.orderBy, position, ordered, combined, isStart = true).. + peerBoundary(window.orderBy, position, ordered, combined, isStart = false) + range.filter { it == position || it !in peer } + } + } + } + + /** + * Resolves an explicit `ROWS|RANGE BETWEEN ... AND ...` [WindowFrame] to a range of positions + * into [ordered] for the row at [position] - clamped to the partition's edges rather than + * erroring, exactly like the implicit default frame already does. A frame that falls entirely + * off one edge (e.g. `2 FOLLOWING AND 3 FOLLOWING` on the partition's last row) yields an empty + * range, which [aggregateWindowValue]/FIRST_VALUE/LAST_VALUE each already know how to treat as + * "no rows" (NULL, or 0 for COUNT). + */ + private fun frameRange(window: WindowExpression, position: Int, ordered: List, combined: List>): IntRange { + val frame = window.frame!! + if (frame.unit == FrameUnit.Groups) { + return groupsFrameRange(frame, window.orderBy, position, ordered, combined) + } + val size = ordered.size + val start = resolveFrameBound(frame, frame.start, window, position, ordered, combined, isStart = true) + val end = resolveFrameBound(frame, frame.end, window, position, ordered, combined, isStart = false) + val clampedStart = start.coerceAtLeast(0) + val clampedEnd = end.coerceAtMost(size - 1) + return if (clampedStart > clampedEnd) IntRange.EMPTY else clampedStart..clampedEnd + } + + private fun resolveFrameBound( + frame: WindowFrame, + bound: FrameBound, + window: WindowExpression, + position: Int, + ordered: List, + combined: List>, + isStart: Boolean + ): Int = when (bound.type) { + FrameBoundType.UnboundedPreceding -> 0 + FrameBoundType.UnboundedFollowing -> ordered.size - 1 + FrameBoundType.CurrentRow -> + if (frame.unit == FrameUnit.Rows) position else peerBoundary(window.orderBy, position, ordered, combined, isStart) + FrameBoundType.Preceding -> + if (frame.unit == FrameUnit.Rows) position - resolveIntegerFrameOffset(bound, combined[ordered[position]], "ROWS") + else rangeBoundary(bound, window.orderBy, position, ordered, combined, isStart, subtract = true) + FrameBoundType.Following -> + if (frame.unit == FrameUnit.Rows) position + resolveIntegerFrameOffset(bound, combined[ordered[position]], "ROWS") + else rangeBoundary(bound, window.orderBy, position, ordered, combined, isStart, subtract = false) + } + + private fun resolveIntegerFrameOffset(bound: FrameBound, row: List, unit: String): Int { + val value = evaluateRow(bound.offset!!, row) + val amount = (value as? Value.IntegerValue)?.value + ?: throw KestrelException("$unit frame offset must be an integer") + if (amount < 0) throw KestrelException("$unit frame offset must not be negative") + return amount.toInt() + } + + /** + * `GROUPS BETWEEN AND ` counts whole peer groups (rows tied on the OVER's + * ORDER BY) rather than physical rows ([FrameUnit.Rows]) or distance in a value + * ([FrameUnit.Range]) - `1 PRECEDING` means "the peer group immediately before the current + * row's own peer group", however many rows that group has. Resolved entirely in group-index + * space (mirroring exactly how [frameRange] resolves ROWS/RANGE in row-index space, including + * the same one-sided-clamp trick for correctly detecting an out-of-partition frame as empty + * rather than accidentally clamping it back into range), then translated to a row [IntRange] + * only once both bounds land on a valid group. Unlike [rangeBoundary], this places no + * restriction on the OVER's ORDER BY - zero, one, or many columns of any type all define peer + * groups equally well, since [peerGroupRanges] only ever compares them for equality. + */ + private fun groupsFrameRange( + frame: WindowFrame, + orderBy: List, + position: Int, + ordered: List, + combined: List> + ): IntRange { + val groups = peerGroupRanges(orderBy, ordered, combined) + val currentGroup = groups.indexOfFirst { position in it } + val row = combined[ordered[position]] + val startGroup = resolveGroupBound(frame.start, currentGroup, groups.size, row).coerceAtLeast(0) + val endGroup = resolveGroupBound(frame.end, currentGroup, groups.size, row).coerceAtMost(groups.size - 1) + return if (startGroup > endGroup) IntRange.EMPTY else groups[startGroup].first..groups[endGroup].last + } + + private fun resolveGroupBound(bound: FrameBound, currentGroup: Int, groupCount: Int, row: List): Int = when (bound.type) { + FrameBoundType.UnboundedPreceding -> 0 + FrameBoundType.UnboundedFollowing -> groupCount - 1 + FrameBoundType.CurrentRow -> currentGroup + FrameBoundType.Preceding -> currentGroup - resolveIntegerFrameOffset(bound, row, "GROUPS") + FrameBoundType.Following -> currentGroup + resolveIntegerFrameOffset(bound, row, "GROUPS") + } + + /** + * `CURRENT ROW` in a RANGE frame does not mean "just this row" (that is what ROWS is for) - it + * means the boundary of the current row's peer group (every row tied with it on the OVER's + * ORDER BY, or the whole partition when there is no ORDER BY). [isStart] picks which edge of + * that peer group to return. + */ + private fun peerBoundary(orderBy: List, position: Int, ordered: List, combined: List>, isStart: Boolean): Int { + if (orderBy.isEmpty()) return if (isStart) 0 else ordered.size - 1 + fun keyAt(index: Int) = orderBy.map { evaluateRow(it.expression, combined[ordered[index]]) } + val key = keyAt(position) + var boundary = position + if (isStart) { + while (boundary > 0 && keyAt(boundary - 1) == key) boundary-- + } else { + while (boundary < ordered.size - 1 && keyAt(boundary + 1) == key) boundary++ + } + return boundary + } + + /** + * ` PRECEDING`/` FOLLOWING` in a RANGE frame measure distance in the value of a + * single numeric or DATE ORDER BY column rather than a row count, so the frame can span a + * different number of physical rows for every current row (e.g. `RANGE BETWEEN 5 PRECEDING AND + * 5 FOLLOWING` on `ORDER BY amount` widens or narrows with how densely `amount` is populated + * nearby). Binary search works because [ordered] is already sorted on this column - ascending + * in "logical" terms via [rangeLogicalValue], which folds DESC into a sign flip so the same + * `>=`/`<=` comparisons apply regardless of sort direction. + */ + private fun rangeBoundary( + bound: FrameBound, + orderBy: List, + position: Int, + ordered: List, + combined: List>, + isStart: Boolean, + subtract: Boolean + ): Int { + if (orderBy.size != 1) { + throw KestrelException("a RANGE frame with a PRECEDING/FOLLOWING bound requires exactly one ORDER BY column") + } + val key = orderBy.single() + val current = rangeLogicalValue(key, combined[ordered[position]]) + val offset = resolveRangeFrameOffset(bound, combined[ordered[position]]) + val threshold = if (subtract) current - offset else current + offset + return if (isStart) { + rangeLowerBound(ordered, combined, key, threshold) + } else { + rangeUpperBound(ordered, combined, key, threshold) + } + } + + private fun rangeLowerBound(ordered: List, combined: List>, key: OrderKey, threshold: BigDecimal): Int { + var low = 0 + var high = ordered.size + while (low < high) { + val mid = (low + high) / 2 + if (rangeLogicalValue(key, combined[ordered[mid]]) >= threshold) high = mid else low = mid + 1 + } + return low + } + + private fun rangeUpperBound(ordered: List, combined: List>, key: OrderKey, threshold: BigDecimal): Int { + var low = 0 + var high = ordered.size + while (low < high) { + val mid = (low + high) / 2 + if (rangeLogicalValue(key, combined[ordered[mid]]) <= threshold) low = mid + 1 else high = mid + } + return low - 1 + } + + private fun rangeLogicalValue(key: OrderKey, row: List): BigDecimal { + val magnitude = rangeMagnitude(evaluateRow(key.expression, row)) + return if (key.descending) magnitude.negate() else magnitude + } + + private fun rangeMagnitude(value: Value): BigDecimal = when (value) { + is Value.IntegerValue -> BigDecimal(value.value) + is Value.DecimalValue -> value.value + is Value.RealValue -> BigDecimal.valueOf(value.value) + is Value.DateValue -> BigDecimal(value.value.toEpochDay()) + else -> throw KestrelException("a RANGE frame requires the ORDER BY column to be numeric or DATE") + } + + private fun resolveRangeFrameOffset(bound: FrameBound, row: List): BigDecimal { + val offset = when (val value = evaluateRow(bound.offset!!, row)) { + is Value.IntegerValue -> BigDecimal(value.value) + is Value.DecimalValue -> value.value + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> throw KestrelException("RANGE frame offset must be numeric") + } + if (offset.signum() < 0) throw KestrelException("RANGE frame offset must not be negative") + return offset + } + + /** + * Splits [ordered] (positions into [combined], already sorted by the OVER's ORDER BY) into + * contiguous "peer group" ranges of rows tied on the order key - or one range spanning the + * whole partition when there is no ORDER BY, matching the default SQL frame. Shared by + * [assignAggregateWindowValues] (running aggregate through the end of a peer group) and + * [WindowFunctionKind.LastValue] (last row of a peer group). + */ + private fun peerGroupRanges(orderBy: List, ordered: List, combined: List>): List { + if (orderBy.isEmpty()) return listOf(ordered.indices) + val keys = ordered.map { index -> orderBy.map { key -> evaluateRow(key.expression, combined[index]) } } + val ranges = mutableListOf() + var i = 0 + while (i < ordered.size) { + var j = i + while (j < ordered.size && keys[j] == keys[i]) j++ + ranges.add(i until j) + i = j + } + return ranges + } + + private fun aggregateWindowValue(window: WindowExpression, rows: List>): Value = + AggregateFunctions.compute(window.name, window.argument, window.star, rows, ::evaluateRow) + + /** + * LAG (direction -1) / LEAD (direction +1): reads [WindowExpression.argument] from the row + * [WindowExpression.offset] positions before/after the current one in [ordered] (default 1, + * evaluated per row so it may itself be a computed expression, though in practice it is + * almost always a literal). Past either edge of the partition, falls back to + * [WindowExpression.defaultValue] (evaluated against the *current* row) or NULL. + */ + private fun assignOffsetWindowValues( + window: WindowExpression, + ordered: List, + combined: List>, + out: Array, + direction: Int + ) { + ordered.forEachIndexed { position, index -> + val offset = resolveOffset(window, combined[index]) + val targetPosition = position + direction * offset + out[index] = if (targetPosition in ordered.indices) { + evaluateRow(window.argument!!, combined[ordered[targetPosition]]) + } else { + window.defaultValue?.let { evaluateRow(it, combined[index]) } ?: Value.NullValue + } + } + } + + private fun resolveOffset(window: WindowExpression, row: List): Int { + val expression = window.offset ?: return 1 + val value = evaluateRow(expression, row) + val amount = (value as? Value.IntegerValue)?.value + ?: throw KestrelException("${window.name}() offset must be an integer") + if (amount < 0) throw KestrelException("${window.name}() offset must not be negative") + return amount.toInt() + } + + /** + * NTILE(n): splits the partition into `n` buckets as evenly as possible - when the partition + * size does not divide evenly, the first `size % n` buckets get one extra row, exactly like + * every other SQL engine's NTILE. `n` larger than the partition just leaves the highest + * bucket numbers unused, rather than being an error. + */ + private fun assignNtileWindowValues(window: WindowExpression, ordered: List, combined: List>, out: Array) { + val bucketValue = evaluateRow(window.argument!!, combined[ordered.first()]) + val buckets = (bucketValue as? Value.IntegerValue)?.value + ?: throw KestrelException("${window.name}() bucket count must be an integer") + if (buckets <= 0) throw KestrelException("${window.name}() bucket count must be positive") + val bucketCount = buckets.toInt() + val base = ordered.size / bucketCount + val remainder = ordered.size % bucketCount + var position = 0 + for (bucket in 1..bucketCount) { + val size = base + if (bucket <= remainder) 1 else 0 + repeat(size) { + out[ordered[position]] = Value.IntegerValue(bucket.toLong()) + position++ + } + } } private fun primaryRows(): List { @@ -179,30 +1210,82 @@ class SelectRunner(private val catalog: Catalog) { private fun executeJoin(left: List>, join: JoinClause, rightIndex: Int): List> { val right = sources[rightIndex] val nullRight = List(right.schema.columns.size) { Value.NullValue } + // Width of everything already combined to the left of this join - exactly what a RIGHT/ + // FULL JOIN needs to pad an unmatched right row with, since `right.offset` is defined as + // the running column count of every source before it. + val nullLeft = List(right.offset) { Value.NullValue } + val outerRight = join.type == JoinType.Right || join.type == JoinType.Full + val outerLeft = join.type == JoinType.Left || join.type == JoinType.Full + val lateralSelect = lateralSelectOf(join.from) + if (lateralSelect != null) return executeLateralJoin(left, join, lateralSelect, nullRight) val hashKeys = hashJoinKeys(join, rightIndex) val result = mutableListOf>() if (hashKeys != null) { - val buckets = HashMap>() - for (rightRow in right.rows) { - val key = evaluateOnRight(hashKeys.buildExpression, right, rightRow) - if (key !is Value.NullValue) buckets.getOrPut(key) { mutableListOf() }.add(rightRow) + val buckets = HashMap>>() + for (indexedRow in right.rows.withIndex()) { + val key = foldHashJoinKey(evaluateOnRight(hashKeys.buildExpression, right, indexedRow.value), hashKeys.collation) + if (key !is Value.NullValue) buckets.getOrPut(key) { mutableListOf() }.add(indexedRow) } + val matchedRight = BooleanArray(right.rows.size) for (leftRow in left) { - val probe = evaluateRow(hashKeys.probeExpression, leftRow) + val probe = foldHashJoinKey(evaluateRow(hashKeys.probeExpression, leftRow), hashKeys.collation) val matches = if (probe is Value.NullValue) emptyList() else buckets[probe].orEmpty() if (matches.isNotEmpty()) { - matches.forEach { result.add(leftRow + it.values) } - } else if (join.type == JoinType.Left) { + matches.forEach { (index, row) -> result.add(leftRow + row.values); matchedRight[index] = true } + } else if (outerLeft) { result.add(leftRow + nullRight) } } + if (outerRight) { + right.rows.forEachIndexed { index, row -> if (!matchedRight[index]) result.add(nullLeft + row.values) } + } return result } + val condition = join.on + val matchedRight = BooleanArray(right.rows.size) for (leftRow in left) { var matched = false - for (rightRow in right.rows) { + right.rows.forEachIndexed { index, rightRow -> val candidate = leftRow + rightRow.values - if (isTrue(evaluateRow(join.on, candidate))) { + if (condition == null || isTrue(evaluateRow(condition, candidate))) { + result.add(candidate) + matched = true + matchedRight[index] = true + } + } + if (!matched && outerLeft) result.add(leftRow + nullRight) + } + if (outerRight) { + right.rows.forEachIndexed { index, row -> if (!matchedRight[index]) result.add(nullLeft + row.values) } + } + return result + } + + private fun lateralSelectOf(item: FromItem): QueryStatement? = + (item as? SubquerySource)?.takeIf { it.lateral }?.select + + /** + * Unlike every other join, a LATERAL derived table has no fixed row set: it is + * re-evaluated once per row already accumulated on the left, correlated to that row via + * [lateralOuterResolver] exactly like a correlated subquery in WHERE/SELECT (same + * [SubqueryRunner] plumbing, just invoked from FROM instead). Hash-joining is therefore + * meaningless here (there is no static right-hand row set to build buckets from) - always + * nested loop, matching how real engines execute LATERAL. + */ + private fun executeLateralJoin( + left: List>, + join: JoinClause, + lateralSelect: QueryStatement, + nullRight: List + ): List> { + val condition = join.on + val result = mutableListOf>() + for (leftRow in left) { + val rightRows = runQuery(lateralSelect, lateralOuterResolver(leftRow)).rows + var matched = false + for (rightRow in rightRows) { + val candidate = leftRow + rightRow + if (condition == null || isTrue(evaluateRow(condition, candidate))) { result.add(candidate) matched = true } @@ -212,6 +1295,27 @@ class SelectRunner(private val catalog: Catalog) { return result } + /** + * Resolves a column against the sources already joined by the time a LATERAL subquery + * runs (everything to its left in the FROM clause) - [leftRow] holds exactly their values, + * so a resolved global index landing past its end means the expression names a source that + * has not been joined yet (or is the LATERAL item itself), which SQL forbids. + */ + private fun lateralOuterResolver(leftRow: List): kestrel.engine.ColumnResolver = object : kestrel.engine.ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val index = resolveIndex(expression) + if (index >= leftRow.size) { + throw KestrelException( + "LATERAL subquery cannot reference ${describe(expression)}: not joined yet at this point in the FROM clause" + ) + } + return leftRow[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = + resolveIndexOrNull(expression)?.takeIf { it < leftRow.size }?.let { declaredCollationOrNull(expression) } + } + private fun evaluateOnRight(expression: Expression, right: Source, rightRow: Row): Value { val resolver = kestrel.engine.ColumnResolver { columnExpression -> rightRow.values[resolveIndex(columnExpression) - right.offset] @@ -219,17 +1323,35 @@ class SelectRunner(private val catalog: Catalog) { return Evaluator(resolver).evaluate(expression) } + /** + * A [Value.TextValue] hash-join key under any collation but [Collation.Binary] must land in + * the same bucket as every other spelling the declared/explicit `COLLATE` treats as equal + * (e.g. `'Bob'`/`'bob'` under `NOCASE`) - otherwise the fast path silently disagrees with + * the nested-loop `Evaluator` path it is an optimization of, which already resolves this via + * [pairCollation] for every non-hashable equality condition. [Collation.Binary]'s + * [Collation.canonicalize] is the identity, so a key from an equality with no COLLATE + * anywhere in play keeps its exact original `equals`/`hashCode`. + */ + private fun foldHashJoinKey(value: Value, collation: Collation): Value = + if (value is Value.TextValue) Value.TextValue(collation.canonicalize(value.value)) else value + private fun hashJoinKeys(join: JoinClause, rightIndex: Int): HashKeys? { + if (lateralSelectOf(join.from) != null) return null val condition = join.on if (condition !is BinaryExpression || condition.operator != BinaryOperator.Equal) return null val built = (0 until rightIndex).toSet() val leftRefs = referencedSources(condition.left) val rightRefs = referencedSources(condition.right) + // Resolved from the ORIGINAL (unswapped) left/right of `a = b`, not buildExpression/ + // probeExpression below, so `pairCollation`'s left-first tie-break matches exactly what + // evaluating this same `=` through Evaluator would have decided regardless of which side + // ends up building the hash buckets. + val collation = pairCollation(condition.left, condition.right, ::declaredCollationOrNull) return when { rightRefs == setOf(rightIndex) && leftRefs.all { it in built } -> - HashKeys(buildExpression = condition.right, probeExpression = condition.left) + HashKeys(buildExpression = condition.right, probeExpression = condition.left, collation = collation) leftRefs == setOf(rightIndex) && rightRefs.all { it in built } -> - HashKeys(buildExpression = condition.left, probeExpression = condition.right) + HashKeys(buildExpression = condition.left, probeExpression = condition.right, collation = collation) else -> null } } @@ -241,40 +1363,84 @@ class SelectRunner(private val catalog: Catalog) { } fun projectRows(combined: List>): SelectResult { + statement.limit?.let { if (it < 0) throw KestrelException("LIMIT must not be negative") } + statement.offset?.let { if (it < 0) throw KestrelException("OFFSET must not be negative") } var rows = combined - orderExpression?.let { expression -> - val comparator = Comparator> { first, second -> - ValueOrdering.compare(evaluateRow(expression, first), evaluateRow(expression, second)) + if (orderKeys.isNotEmpty()) { + rows = rows.sortedWith(multiKeyComparator(orderKeys, ::evaluateRow, ::declaredCollationOrNull)) + } + if (statement.distinctOn.isNotEmpty()) { + // Kept in the order already established above (scan order with no ORDER BY, sorted + // order otherwise) - `LinkedHashMap.putIfAbsent` keeps the first row per key and + // never disturbs where that first row sits relative to the others kept. + val kept = LinkedHashMap, List>() + for (row in rows) { + val key = statement.distinctOn.map { evaluateRow(it, row) } + kept.putIfAbsent(collationFoldedKey(key, statement.distinctOn, ::declaredCollationOrNull), row) } - rows = rows.sortedWith(if (statement.orderBy!!.descending) comparator.reversed() else comparator) + rows = kept.values.toList() } - statement.limit?.let { - if (it < 0) throw KestrelException("LIMIT must not be negative") - rows = rows.take(it) + if (!statement.distinct) { + if (statement.offset != null || statement.limit != null) { + rows = rows.drop(statement.offset ?: 0) + statement.limit?.let { rows = rows.take(it) } + } } - return when (val projection = statement.projection) { - is StarProjection -> SelectResult(columnNames, rows) + val (labels, projected) = when (val projection = statement.projection) { + // rows may carry trailing window-function values appended by appendWindowColumns + // (only possible here if ORDER BY names one directly, since `SELECT *` cannot spell + // an OVER clause itself) - trimmed back off so the row width matches columnNames. + is StarProjection -> columnNames to rows.map { it.subList(0, columnNames.size) } is ExpressionProjection -> { val labels = projection.items.mapIndexed { index, item -> columnLabel(item, index) } - val projected = rows.map { row -> projection.items.map { evaluateRow(it.expression, row) } } - SelectResult(labels, projected) + labels to rows.map { row -> projection.items.map { evaluateRow(it.expression, row) } } } } + var resultRows = projected + if (statement.distinct) { + // Plain DISTINCT dedups the *output* row (post-projection); DISTINCT ON already + // collapsed to one row per group above, on the pre-projection row, so re-deduping the + // projected columns here would incorrectly merge distinct groups that happen to + // project to the same visible values. + if (statement.distinctOn.isEmpty()) { + val itemExpressions = when (val projection = statement.projection) { + is ExpressionProjection -> projection.items.map { it.expression } + is StarProjection -> columnNames.map { ColumnExpression(it) } + } + resultRows = resultRows.distinctBy { row -> collationFoldedKey(row, itemExpressions, ::declaredCollationOrNull) } + } + if (statement.offset != null || statement.limit != null) { + resultRows = resultRows.drop(statement.offset ?: 0) + statement.limit?.let { resultRows = resultRows.take(it) } + } + } + return SelectResult(labels, resultRows) } fun aggregate(combined: List>): SelectResult = - GroupAggregation(::resolveIndex, ::evaluateRow, this@SelectRunner::columnLabel) - .aggregate(statement, orderExpression, combined) + GroupAggregation(::resolveIndex, ::evaluateRow, this@SelectRunner::columnLabel, subqueryRunner, ::declaredCollationOrNull) + .aggregate(statement, orderKeys, combined) fun describe(): List { val lines = mutableListOf() var body = joinLines() if (aggregated) { val keys = statement.groupBy.map { describeExpression(it) } - val header = if (keys.isEmpty()) "Aggregate" else "GroupAggregate keys: ${keys.joinToString(", ")}" + val header = if (keys.isEmpty()) { + "Aggregate" + } else { + val setCount = GroupingSetExpansion.setsFor(statement.groupingSpec, statement.groupBy).size + val suffix = if (setCount > 1) " ($setCount grouping sets)" else "" + "GroupAggregate keys: ${keys.joinToString(", ")}$suffix" + } body = listOf(header) + body.map { " $it" } } statement.orderBy?.let { body = listOf("Sort") + body.map { " $it" } } + if (statement.distinct) { + val label = if (statement.distinctOn.isNotEmpty()) "Unique (DISTINCT ON)" else "Unique" + body = listOf(label) + body.map { " $it" } + } + statement.offset?.let { body = listOf("Offset ${statement.offset}") + body.map { " $it" } } statement.limit?.let { body = listOf("Limit ${statement.limit}") + body.map { " $it" } } lines.addAll(body) return lines @@ -284,9 +1450,14 @@ class SelectRunner(private val catalog: Catalog) { var lines = listOf(scanLine(sources.first(), primaryAccess())) statement.joins.forEachIndexed { position, join -> val right = sources[position + 1] - val hash = hashJoinKeys(join, position + 1) != null - val header = if (hash) "Hash Join ${describeExpression(join.on)}" else "Nested Loop ${joinLabel(join)}" - lines = listOf(header) + lines.map { " -> $it" } + listOf(" -> Seq Scan on ${right.tableName}") + val lateral = lateralSelectOf(join.from) != null + val hash = !lateral && hashJoinKeys(join, position + 1) != null + val label = joinLabel(join) + if (lateral) " Lateral" else "" + // `hash` is only true when hashJoinKeys() matched a binary equality condition, which + // means join.on is guaranteed non-null here (CROSS JOIN's null `on` never hashes). + val header = if (hash) "Hash Join ${describeExpression(join.on!!)}" else "Nested Loop $label" + val rightScan = if (lateral) "Lateral Subquery Scan on ${right.tableName}" else "Seq Scan on ${right.tableName}" + lines = listOf(header) + lines.map { " -> $it" } + listOf(" -> $rightScan") } return lines } @@ -297,14 +1468,21 @@ class SelectRunner(private val catalog: Catalog) { "Index Scan using ${access.indexName} on ${source.tableName} (${access.description})" } - private fun joinLabel(join: JoinClause): String = - if (join.type == JoinType.Left) "Left" else "Inner" + private fun joinLabel(join: JoinClause): String = when (join.type) { + JoinType.Left -> "Left" + JoinType.Right -> "Right" + JoinType.Full -> "Full" + JoinType.Inner -> "Inner" + JoinType.Cross -> "Cross" + } } private fun columnLabel(item: SelectItem, index: Int): String = item.alias ?: when (val expression = item.expression) { is ColumnExpression -> expression.name is FunctionExpression -> expression.name.lowercase() + is CallExpression -> expression.name.lowercase() + is WindowExpression -> expression.name.lowercase() else -> "column${index + 1}" } @@ -312,8 +1490,34 @@ class SelectRunner(private val catalog: Catalog) { is ColumnExpression -> listOf(expression) is BinaryExpression -> collectColumns(expression.left) + collectColumns(expression.right) is UnaryExpression -> collectColumns(expression.operand) - is FunctionExpression -> expression.argument?.let { collectColumns(it) } ?: emptyList() + is FunctionExpression -> + (expression.argument?.let { collectColumns(it) } ?: emptyList()) + + (expression.separator?.let { collectColumns(it) } ?: emptyList()) + + expression.orderBy.flatMap { collectColumns(it.expression) } + + (expression.filter?.let { collectColumns(it) } ?: emptyList()) is LiteralExpression -> emptyList() + is CallExpression -> expression.arguments.flatMap { collectColumns(it) } + is IsNullExpression -> collectColumns(expression.operand) + is InExpression -> collectColumns(expression.operand) + expression.values.flatMap { collectColumns(it) } + is BetweenExpression -> collectColumns(expression.operand) + collectColumns(expression.low) + collectColumns(expression.high) + is LikeExpression -> collectColumns(expression.operand) + collectColumns(expression.pattern) + is CaseExpression -> (expression.operand?.let { collectColumns(it) } ?: emptyList()) + + expression.branches.flatMap { collectColumns(it.condition) + collectColumns(it.result) } + + (expression.elseExpression?.let { collectColumns(it) } ?: emptyList()) + is ScalarSubqueryExpression -> emptyList() + is ExistsExpression -> emptyList() + is InSubqueryExpression -> collectColumns(expression.operand) + is CastExpression -> collectColumns(expression.operand) + is CollateExpression -> collectColumns(expression.operand) + is WindowExpression -> + (expression.argument?.let { collectColumns(it) } ?: emptyList()) + + (expression.offset?.let { collectColumns(it) } ?: emptyList()) + + (expression.defaultValue?.let { collectColumns(it) } ?: emptyList()) + + (expression.frame?.start?.offset?.let { collectColumns(it) } ?: emptyList()) + + (expression.frame?.end?.offset?.let { collectColumns(it) } ?: emptyList()) + + expression.partitionBy.flatMap { collectColumns(it) } + + expression.orderBy.flatMap { collectColumns(it.expression) } + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> emptyList() } private fun containsAggregate(projection: kestrel.sql.ast.Projection): Boolean = when (projection) { @@ -325,9 +1529,66 @@ class SelectRunner(private val catalog: Catalog) { is FunctionExpression -> true is BinaryExpression -> containsAggregate(expression.left) || containsAggregate(expression.right) is UnaryExpression -> containsAggregate(expression.operand) + is CallExpression -> expression.arguments.any { containsAggregate(it) } + is IsNullExpression -> containsAggregate(expression.operand) + is InExpression -> containsAggregate(expression.operand) || expression.values.any { containsAggregate(it) } + is BetweenExpression -> + containsAggregate(expression.operand) || containsAggregate(expression.low) || containsAggregate(expression.high) + is LikeExpression -> containsAggregate(expression.operand) || containsAggregate(expression.pattern) + is CaseExpression -> + (expression.operand?.let { containsAggregate(it) } ?: false) || + expression.branches.any { containsAggregate(it.condition) || containsAggregate(it.result) } || + (expression.elseExpression?.let { containsAggregate(it) } ?: false) + is CastExpression -> containsAggregate(expression.operand) + is CollateExpression -> containsAggregate(expression.operand) + // A window function never triggers GROUP BY collapsing on its own - it is deliberately not + // recursed into here, so an aggregate nested inside its OVER clause (invalid SQL) is instead + // caught uniformly at evaluation time by the same guard that rejects any bare aggregate + // outside GROUP BY (see Evaluator.evaluate's FunctionExpression branch). + is WindowExpression -> false else -> false } + /** Window functions are only legal in the SELECT list and ORDER BY - never WHERE/GROUP BY/HAVING/ON. */ + private fun containsWindowFunction(expression: Expression): Boolean = when (expression) { + is WindowExpression -> true + is BinaryExpression -> containsWindowFunction(expression.left) || containsWindowFunction(expression.right) + is UnaryExpression -> containsWindowFunction(expression.operand) + is CallExpression -> expression.arguments.any { containsWindowFunction(it) } + is IsNullExpression -> containsWindowFunction(expression.operand) + is InExpression -> containsWindowFunction(expression.operand) || expression.values.any { containsWindowFunction(it) } + is BetweenExpression -> + containsWindowFunction(expression.operand) || containsWindowFunction(expression.low) || containsWindowFunction(expression.high) + is LikeExpression -> containsWindowFunction(expression.operand) || containsWindowFunction(expression.pattern) + is CaseExpression -> + (expression.operand?.let { containsWindowFunction(it) } ?: false) || + expression.branches.any { containsWindowFunction(it.condition) || containsWindowFunction(it.result) } || + (expression.elseExpression?.let { containsWindowFunction(it) } ?: false) + is CastExpression -> containsWindowFunction(expression.operand) + is CollateExpression -> containsWindowFunction(expression.operand) + else -> false + } + + /** Every distinct window-function call reachable from [expression] (nested inside arithmetic, CASE, ...). */ + private fun collectWindowExpressions(expression: Expression): List = when (expression) { + is WindowExpression -> listOf(expression) + is BinaryExpression -> collectWindowExpressions(expression.left) + collectWindowExpressions(expression.right) + is UnaryExpression -> collectWindowExpressions(expression.operand) + is CallExpression -> expression.arguments.flatMap { collectWindowExpressions(it) } + is IsNullExpression -> collectWindowExpressions(expression.operand) + is InExpression -> collectWindowExpressions(expression.operand) + expression.values.flatMap { collectWindowExpressions(it) } + is BetweenExpression -> + collectWindowExpressions(expression.operand) + collectWindowExpressions(expression.low) + collectWindowExpressions(expression.high) + is LikeExpression -> collectWindowExpressions(expression.operand) + collectWindowExpressions(expression.pattern) + is CaseExpression -> + (expression.operand?.let { collectWindowExpressions(it) } ?: emptyList()) + + expression.branches.flatMap { collectWindowExpressions(it.condition) + collectWindowExpressions(it.result) } + + (expression.elseExpression?.let { collectWindowExpressions(it) } ?: emptyList()) + is CastExpression -> collectWindowExpressions(expression.operand) + is CollateExpression -> collectWindowExpressions(expression.operand) + else -> emptyList() + } + private fun isTrue(value: Value): Boolean = value is Value.BooleanValue && value.value private fun describe(expression: ColumnExpression): String = @@ -336,16 +1597,70 @@ class SelectRunner(private val catalog: Catalog) { private fun describeExpression(expression: Expression): String = when (expression) { is ColumnExpression -> describe(expression) is LiteralExpression -> literalText(expression.value) - is FunctionExpression -> "${expression.name}(${if (expression.star) "*" else expression.argument?.let { describeExpression(it) } ?: ""})" + is FunctionExpression -> "${expression.name}(${describeFunctionArguments(expression)})${describeFilterClause(expression)}" is BinaryExpression -> "${describeExpression(expression.left)} ${expression.operator.symbol} ${describeExpression(expression.right)}" is UnaryExpression -> "${expression.operator.symbol} ${describeExpression(expression.operand)}" + is CallExpression -> "${expression.name}(${expression.arguments.joinToString(", ") { describeExpression(it) }})" + is IsNullExpression -> "${describeExpression(expression.operand)} IS ${if (expression.negated) "NOT " else ""}NULL" + is InExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (${expression.values.joinToString(", ") { describeExpression(it) }})" + is BetweenExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}BETWEEN ${describeExpression(expression.low)} AND ${describeExpression(expression.high)}" + is LikeExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}LIKE ${describeExpression(expression.pattern)}" + is CaseExpression -> buildString { + append("CASE ") + expression.operand?.let { append(describeExpression(it)); append(' ') } + expression.branches.forEach { append("WHEN ${describeExpression(it.condition)} THEN ${describeExpression(it.result)} ") } + expression.elseExpression?.let { append("ELSE ${describeExpression(it)} ") } + append("END") + } + is ScalarSubqueryExpression -> "(subquery)" + is ExistsExpression -> "EXISTS (subquery)" + is InSubqueryExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (subquery)" + is CastExpression -> + "CAST(${describeExpression(expression.operand)} AS ${expression.targetType.displayWith(expression.precision, expression.scale)})" + is CollateExpression -> "${describeExpression(expression.operand)} COLLATE ${expression.collation.sqlName}" + is WindowExpression -> + "${expression.name}(${if (expression.star) "*" else expression.argument?.let { describeExpression(it) } ?: ""}) OVER (...)" + is NextValueExpression -> "NEXTVAL('${expression.sequenceName}')" + is CurrentValueExpression -> "CURRVAL('${expression.sequenceName}')" + is MergeActionExpression -> "MERGE_ACTION()" + is ParameterExpression -> "\$${expression.index}" } + private fun describeFunctionArguments(expression: FunctionExpression): String { + if (expression.star) return "*" + val distinctPrefix = if (expression.distinct) "DISTINCT " else "" + val value = expression.argument?.let { describeExpression(it) } ?: "" + val separator = expression.separator ?: return "$distinctPrefix$value" + val orderBy = if (expression.orderBy.isNotEmpty()) { + " ORDER BY " + expression.orderBy.joinToString(", ") { key -> describeExpression(key.expression) + orderKeySuffix(key) } + } else "" + return "$distinctPrefix$value, ${describeExpression(separator)}$orderBy" + } + + private fun orderKeySuffix(key: OrderKey): String { + val direction = if (key.descending) " DESC" else "" + val nulls = when (key.nullsFirst) { + true -> " NULLS FIRST" + false -> " NULLS LAST" + null -> "" + } + return direction + nulls + } + + private fun describeFilterClause(expression: FunctionExpression): String = + expression.filter?.let { " FILTER (WHERE ${describeExpression(it)})" } ?: "" + private fun literalText(value: Value): String = when (value) { is Value.IntegerValue -> value.value.toString() is Value.RealValue -> value.value.toString() is Value.TextValue -> "'${value.value}'" is Value.BooleanValue -> value.value.toString() + is Value.DecimalValue -> value.value.toPlainString() + is Value.DateValue -> value.value.toString() is Value.NullValue -> "NULL" } @@ -357,5 +1672,12 @@ class SelectRunner(private val catalog: Catalog) { val offset: Int ) - private class HashKeys(val buildExpression: Expression, val probeExpression: Expression) + private class HashKeys(val buildExpression: Expression, val probeExpression: Expression, val collation: Collation) + + private companion object { + const val MAX_RECURSION_ITERATIONS = 10_000 + + /** Kinds whose result actually depends on which rows fall inside the frame. */ + val FRAME_CAPABLE_KINDS = setOf(WindowFunctionKind.Aggregate, WindowFunctionKind.FirstValue, WindowFunctionKind.LastValue) + } } diff --git a/src/main/kotlin/kestrel/engine/StatementResult.kt b/src/main/kotlin/kestrel/engine/StatementResult.kt index ede1cd9..af09e8c 100644 --- a/src/main/kotlin/kestrel/engine/StatementResult.kt +++ b/src/main/kotlin/kestrel/engine/StatementResult.kt @@ -6,20 +6,46 @@ sealed interface StatementResult data class SelectResult(val columns: List, val rows: List>) : StatementResult -data class InsertResult(val count: Int) : StatementResult +data class InsertResult(val count: Int, val returning: SelectResult? = null) : StatementResult -data class UpdateResult(val count: Int) : StatementResult +data class UpdateResult(val count: Int, val returning: SelectResult? = null) : StatementResult -data class DeleteResult(val count: Int) : StatementResult +data class DeleteResult(val count: Int, val returning: SelectResult? = null) : StatementResult data class CreateTableResult(val tableName: String) : StatementResult data class DropTableResult(val tableName: String) : StatementResult +/** [tableNames] is every table actually truncated - the statement's own list plus, under CASCADE, every transitive FOREIGN KEY referencer added to it. */ +data class TruncateTableResult(val tableNames: List) : StatementResult + +data class MergeResult( + val insertedCount: Int, + val updatedCount: Int, + val deletedCount: Int, + val returning: SelectResult? = null +) : StatementResult + +data class AlterTableResult(val tableName: String) : StatementResult + data class CreateIndexResult(val indexName: String) : StatementResult data class DropIndexResult(val indexName: String) : StatementResult +data class CreateViewResult(val viewName: String) : StatementResult + +data class DropViewResult(val viewName: String) : StatementResult + +data class CreateTriggerResult(val triggerName: String) : StatementResult + +data class DropTriggerResult(val triggerName: String) : StatementResult + +data class CreateSequenceResult(val sequenceName: String) : StatementResult + +data class DropSequenceResult(val sequenceName: String) : StatementResult + +data class AlterSequenceResult(val sequenceName: String) : StatementResult + data class ExplainResult(val plan: List) : StatementResult data object BeginResult : StatementResult @@ -27,3 +53,14 @@ data object BeginResult : StatementResult data object CommitResult : StatementResult data object RollbackResult : StatementResult + +data class SavepointResult(val name: String) : StatementResult + +data class ReleaseSavepointResult(val name: String) : StatementResult + +data class RollbackToSavepointResult(val name: String) : StatementResult + +data class PrepareResult(val name: String) : StatementResult + +/** [name] is `null` exactly when [all] is `true` (`DEALLOCATE ALL`), mirroring [kestrel.sql.ast.DeallocateStatement]. */ +data class DeallocateResult(val name: String?, val all: Boolean) : StatementResult diff --git a/src/main/kotlin/kestrel/engine/TriggerRowContext.kt b/src/main/kotlin/kestrel/engine/TriggerRowContext.kt new file mode 100644 index 0000000..20282e1 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/TriggerRowContext.kt @@ -0,0 +1,39 @@ +package kestrel.engine + +import kestrel.KestrelException +import kestrel.catalog.TableSchema +import kestrel.sql.ast.ColumnExpression +import kestrel.storage.Collation +import kestrel.storage.Row +import kestrel.storage.Value + +/** + * Binds `NEW`/`OLD` for the one row a firing trigger is currently processing, against [schema] - + * always the trigger's OWN table (the one named in its `ON
`), never whatever table a + * nested body statement happens to target. [newRow] is null for a DELETE trigger, [oldRow] is null + * for an INSERT trigger; both are non-null only inside an UPDATE trigger. + */ +class TriggerRowContext(private val schema: TableSchema, private val newRow: Row?, private val oldRow: Row?) { + fun matches(expression: ColumnExpression): Boolean { + val qualifier = expression.qualifier ?: return false + return (qualifier.equals("new", ignoreCase = true) && newRow != null) || + (qualifier.equals("old", ignoreCase = true) && oldRow != null) + } + + fun resolve(expression: ColumnExpression): Value { + val row = if (expression.qualifier!!.equals("new", ignoreCase = true)) newRow!! else oldRow!! + val index = schema.columnIndex(expression.name) + ?: throw KestrelException("no such column: ${expression.name}") + return row.values[index] + } + + /** Column-existence check with no row required, for validating a reference before it is ever fired. */ + fun checkColumnExists(expression: ColumnExpression) { + if (schema.columnIndex(expression.name) == null) { + throw KestrelException("no such column: ${expression.name}") + } + } + + /** `NEW.col`/`OLD.col` reads the same table's schema either way, so its declared COLLATE never depends on which side [expression] names. */ + fun declaredCollation(expression: ColumnExpression): Collation? = schema.column(expression.name)?.collation +} diff --git a/src/main/kotlin/kestrel/engine/function/ScalarFunctions.kt b/src/main/kotlin/kestrel/engine/function/ScalarFunctions.kt new file mode 100644 index 0000000..5fc0a83 --- /dev/null +++ b/src/main/kotlin/kestrel/engine/function/ScalarFunctions.kt @@ -0,0 +1,251 @@ +package kestrel.engine.function + +import java.math.RoundingMode +import java.time.LocalDate +import kestrel.KestrelException +import kestrel.storage.Value +import kestrel.storage.ValueOrdering +import kestrel.storage.typeName +import kotlin.math.abs +import kotlin.math.ceil +import kotlin.math.floor +import kotlin.math.pow +import kotlin.math.roundToLong + +class ScalarFunction( + val name: String, + private val minArgs: Int, + private val maxArgs: Int, + private val nullPropagating: Boolean, + private val body: (List) -> Value +) { + fun apply(arguments: List): Value { + if (arguments.size < minArgs || arguments.size > maxArgs) { + val expected = if (minArgs == maxArgs) "$minArgs" else "$minArgs to $maxArgs" + throw KestrelException("function $name expects $expected argument(s) but got ${arguments.size}") + } + if (nullPropagating && arguments.any { it.isNull }) return Value.NullValue + return body(arguments) + } +} + +object ScalarFunctions { + private val registry: Map = buildRegistry() + + fun lookup(name: String): ScalarFunction? = registry[name.uppercase()] + + private fun buildRegistry(): Map { + val functions = mutableListOf() + fun register( + name: String, + minArgs: Int, + maxArgs: Int = minArgs, + nullPropagating: Boolean = true, + body: (List) -> Value + ) { + functions.add(ScalarFunction(name, minArgs, maxArgs, nullPropagating, body)) + } + + register("UPPER", 1) { args -> Value.TextValue(text(args[0], "UPPER").uppercase()) } + register("LOWER", 1) { args -> Value.TextValue(text(args[0], "LOWER").lowercase()) } + register("LENGTH", 1) { args -> Value.IntegerValue(text(args[0], "LENGTH").length.toLong()) } + register("TRIM", 1) { args -> Value.TextValue(text(args[0], "TRIM").trim()) } + register("LTRIM", 1) { args -> Value.TextValue(text(args[0], "LTRIM").trimStart()) } + register("RTRIM", 1) { args -> Value.TextValue(text(args[0], "RTRIM").trimEnd()) } + register("SUBSTR", 2, 3) { args -> substr(args) } + register("SUBSTRING", 2, 3) { args -> substr(args) } + register("REPLACE", 3) { args -> + Value.TextValue(text(args[0], "REPLACE").replace(text(args[1], "REPLACE"), text(args[2], "REPLACE"))) + } + register("CONCAT", 2, Int.MAX_VALUE) { args -> Value.TextValue(args.joinToString("") { text(it, "CONCAT") }) } + register("REVERSE", 1) { args -> Value.TextValue(text(args[0], "REVERSE").reversed()) } + register("LEFT", 2) { args -> Value.TextValue(edge(args, "LEFT", fromStart = true)) } + register("RIGHT", 2) { args -> Value.TextValue(edge(args, "RIGHT", fromStart = false)) } + register("ABS", 1) { args -> absoluteValue(args[0]) } + register("ROUND", 1, 2) { args -> round(args) } + register("CEIL", 1) { args -> Value.IntegerValue(ceilValue(args[0])) } + register("CEILING", 1) { args -> Value.IntegerValue(ceilValue(args[0])) } + register("FLOOR", 1) { args -> Value.IntegerValue(floorValue(args[0])) } + register("COALESCE", 1, Int.MAX_VALUE, nullPropagating = false) { args -> + args.firstOrNull { !it.isNull } ?: Value.NullValue + } + register("IFNULL", 2, nullPropagating = false) { args -> if (!args[0].isNull) args[0] else args[1] } + register("NULLIF", 2, nullPropagating = false) { args -> + if (!args[0].isNull && !args[1].isNull && ValueOrdering.compare(args[0], args[1]) == 0) { + Value.NullValue + } else { + args[0] + } + } + register("TYPEOF", 1, nullPropagating = false) { args -> Value.TextValue(args[0].typeName) } + register("CURRENT_DATE", 0, nullPropagating = false) { Value.DateValue(LocalDate.now()) } + register("YEAR", 1) { args -> Value.IntegerValue(date(args[0], "YEAR").year.toLong()) } + register("MONTH", 1) { args -> Value.IntegerValue(date(args[0], "MONTH").monthValue.toLong()) } + register("DAY", 1) { args -> Value.IntegerValue(date(args[0], "DAY").dayOfMonth.toLong()) } + register("DAYOFWEEK", 1) { args -> Value.IntegerValue(date(args[0], "DAYOFWEEK").dayOfWeek.value.toLong()) } + register("DATE_ADD", 2) { args -> Value.DateValue(date(args[0], "DATE_ADD").plusDays(integer(args[1], "DATE_ADD"))) } + register("DATE_SUB", 2) { args -> Value.DateValue(date(args[0], "DATE_SUB").minusDays(integer(args[1], "DATE_SUB"))) } + register("DATEDIFF", 2) { args -> + Value.IntegerValue(date(args[0], "DATEDIFF").toEpochDay() - date(args[1], "DATEDIFF").toEpochDay()) + } + register("REPEAT", 2) { args -> + val count = integer(args[1], "REPEAT") + Value.TextValue(if (count <= 0) "" else text(args[0], "REPEAT").repeat(count.toInt())) + } + register("LPAD", 2, 3) { args -> pad(args, "LPAD", left = true) } + register("RPAD", 2, 3) { args -> pad(args, "RPAD", left = false) } + register("INSTR", 2) { args -> Value.IntegerValue((text(args[0], "INSTR").indexOf(text(args[1], "INSTR")) + 1).toLong()) } + register("POWER", 2) { args -> power(args[0], args[1]) } + register("MOD", 2) { args -> mod(args[0], args[1]) } + register("SIGN", 1) { args -> Value.IntegerValue(sign(args[0]).toLong()) } + register("TRUNC", 1, 2) { args -> trunc(args) } + // Any NULL argument already short-circuited to NULL above (nullPropagating, the default), + // so by the time the body runs every argument is non-NULL and args is non-empty (min 2) - + // maxWith/minWith can never return null here despite the nullable Comparator.compare contract. + register("GREATEST", 2, Int.MAX_VALUE) { args -> args.maxWith(ValueOrdering) } + register("LEAST", 2, Int.MAX_VALUE) { args -> args.minWith(ValueOrdering) } + + return functions.associateBy { it.name } + } + + private fun text(value: Value, function: String): String = when (value) { + is Value.TextValue -> value.value + else -> throw KestrelException("function $function requires a TEXT argument but got ${value.typeName}") + } + + private fun date(value: Value, function: String): LocalDate = when (value) { + is Value.DateValue -> value.value + else -> throw KestrelException("function $function requires a DATE argument but got ${value.typeName}") + } + + private fun integer(value: Value, function: String): Long = when (value) { + is Value.IntegerValue -> value.value + else -> throw KestrelException("function $function requires an INTEGER argument but got ${value.typeName}") + } + + private fun number(value: Value, function: String): Double = when (value) { + is Value.IntegerValue -> value.value.toDouble() + is Value.RealValue -> value.value + else -> throw KestrelException("function $function requires a numeric argument but got ${value.typeName}") + } + + private fun absoluteValue(value: Value): Value = when (value) { + is Value.IntegerValue -> Value.IntegerValue(abs(value.value)) + is Value.RealValue -> Value.RealValue(abs(value.value)) + is Value.DecimalValue -> Value.DecimalValue(value.value.abs()) + else -> throw KestrelException("function ABS requires a numeric argument but got ${value.typeName}") + } + + private fun ceilValue(value: Value): Long = when (value) { + is Value.DecimalValue -> value.value.setScale(0, RoundingMode.CEILING).toLong() + else -> ceil(number(value, "CEIL")).toLong() + } + + private fun floorValue(value: Value): Long = when (value) { + is Value.DecimalValue -> value.value.setScale(0, RoundingMode.FLOOR).toLong() + else -> floor(number(value, "FLOOR")).toLong() + } + + private fun substr(args: List): Value { + val source = text(args[0], "SUBSTR") + val requestedStart = (args[1] as? Value.IntegerValue)?.value + ?: throw KestrelException("function SUBSTR requires an INTEGER start position") + val start = (if (requestedStart > 0) requestedStart - 1 else 0).coerceAtMost(source.length.toLong()).toInt() + val length = if (args.size == 3) { + (args[2] as? Value.IntegerValue)?.value + ?: throw KestrelException("function SUBSTR requires an INTEGER length") + } else { + null + } + val end = if (length != null) (start + length.coerceAtLeast(0)).coerceAtMost(source.length.toLong()).toInt() else source.length + return Value.TextValue(if (start >= end) "" else source.substring(start, end)) + } + + /** `LEFT(str, n)`/`RIGHT(str, n)`: the first/last `n` characters, clamped to `[0, str.length]` (a non-positive `n` is empty, an `n` past the string's length is the whole string). */ + private fun edge(args: List, function: String, fromStart: Boolean): String { + val source = text(args[0], function) + val count = integer(args[1], function).coerceIn(0, source.length.toLong()).toInt() + return if (fromStart) source.substring(0, count) else source.substring(source.length - count) + } + + /** + * MySQL-compatible LPAD/RPAD: the result is always exactly [target length][Value.IntegerValue] + * characters (never longer) - a source already at or past that length is truncated to it (from + * the start, regardless of which side would have been padded), and a source shorter than it is + * padded by cycling [padString] (default a single space) until the target length is reached. + */ + private fun pad(args: List, function: String, left: Boolean): Value { + val source = text(args[0], function) + val targetLength = integer(args[1], function) + if (targetLength <= 0) return Value.TextValue("") + val length = targetLength.toInt() + if (length <= source.length) return Value.TextValue(source.substring(0, length)) + val padString = if (args.size == 3) text(args[2], function) else " " + if (padString.isEmpty()) throw KestrelException("function $function requires a non-empty pad string") + val needed = length - source.length + val padding = buildString { + while (this.length < needed) append(padString) + }.substring(0, needed) + return Value.TextValue(if (left) padding + source else source + padding) + } + + /** Two INTEGER operands with a non-negative exponent stay exact (repeated Long multiplication); anything else - a negative exponent or a REAL operand on either side - falls back to `Math.pow`. */ + private fun power(base: Value, exponent: Value): Value { + if (base is Value.IntegerValue && exponent is Value.IntegerValue && exponent.value >= 0) { + var result = 1L + repeat(exponent.value.toInt()) { result *= base.value } + return Value.IntegerValue(result) + } + return Value.RealValue(number(base, "POWER").pow(number(exponent, "POWER"))) + } + + /** Two INTEGER operands use exact `Long` remainder; a REAL operand on either side uses floating-point remainder instead. */ + private fun mod(dividend: Value, divisor: Value): Value { + if (dividend is Value.IntegerValue && divisor is Value.IntegerValue) { + if (divisor.value == 0L) throw KestrelException("division by zero") + return Value.IntegerValue(dividend.value % divisor.value) + } + val divisorNumber = number(divisor, "MOD") + if (divisorNumber == 0.0) throw KestrelException("division by zero") + return Value.RealValue(number(dividend, "MOD") % divisorNumber) + } + + private fun round(args: List): Value { + val digits = if (args.size == 2) { + (args[1] as? Value.IntegerValue)?.value?.toInt() + ?: throw KestrelException("function ROUND requires an INTEGER digit count") + } else { + 0 + } + if (args[0] is Value.IntegerValue && digits >= 0) return args[0] + val decimal = args[0] as? Value.DecimalValue + if (decimal != null) return Value.DecimalValue(decimal.value.setScale(digits, RoundingMode.HALF_UP)) + val value = number(args[0], "ROUND") + val factor = 10.0.pow(digits) + return Value.RealValue((value * factor).roundToLong() / factor) + } + + private fun sign(value: Value): Int = when (value) { + is Value.IntegerValue -> if (value.value > 0) 1 else if (value.value < 0) -1 else 0 + is Value.RealValue -> if (value.value > 0) 1 else if (value.value < 0) -1 else 0 + is Value.DecimalValue -> value.value.signum() + else -> throw KestrelException("function SIGN requires a numeric argument but got ${value.typeName}") + } + + /** Same digit-count/type handling as [round], but truncates toward zero (`RoundingMode.DOWN`) instead of rounding half-up. */ + private fun trunc(args: List): Value { + val digits = if (args.size == 2) { + (args[1] as? Value.IntegerValue)?.value?.toInt() + ?: throw KestrelException("function TRUNC requires an INTEGER digit count") + } else { + 0 + } + if (args[0] is Value.IntegerValue && digits >= 0) return args[0] + val decimal = args[0] as? Value.DecimalValue + if (decimal != null) return Value.DecimalValue(decimal.value.setScale(digits, RoundingMode.DOWN)) + val value = number(args[0], "TRUNC") + val factor = 10.0.pow(digits) + val scaled = value * factor + return Value.RealValue((if (scaled >= 0) floor(scaled) else ceil(scaled)) / factor) + } +} diff --git a/src/main/kotlin/kestrel/persistence/SqlWriter.kt b/src/main/kotlin/kestrel/persistence/SqlWriter.kt index c0ef7d7..3b1d682 100644 --- a/src/main/kotlin/kestrel/persistence/SqlWriter.kt +++ b/src/main/kotlin/kestrel/persistence/SqlWriter.kt @@ -2,61 +2,309 @@ package kestrel.persistence import java.math.BigDecimal import kestrel.catalog.Catalog +import kestrel.catalog.CheckConstraint import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.Sequence +import kestrel.catalog.SequenceDefinition +import kestrel.catalog.UniqueConstraint +import kestrel.catalog.TriggerDefinition +import kestrel.catalog.sqlKeyword +import kestrel.sql.ast.AddColumn +import kestrel.sql.ast.AddConstraint +import kestrel.sql.ast.AddedCheck +import kestrel.sql.ast.AddedConstraint +import kestrel.sql.ast.AddedForeignKey +import kestrel.sql.ast.AddedUnique +import kestrel.sql.ast.AlterTableStatement import kestrel.sql.ast.Assignment +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.ChangeColumnType +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.ConflictDoNothing +import kestrel.sql.ast.ConflictDoUpdate +import kestrel.sql.ast.AlterSequenceStatement import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateSequenceStatement import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.CurrentValueExpression import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropColumn +import kestrel.sql.ast.DropConstraint import kestrel.sql.ast.DropIndexStatement +import kestrel.sql.ast.DropMode +import kestrel.sql.ast.DropSequenceStatement import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.DropTriggerStatement +import kestrel.sql.ast.DropViewStatement +import kestrel.sql.ast.ExistsExpression import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FrameBoundType +import kestrel.sql.ast.FrameExclude +import kestrel.sql.ast.FrameUnit +import kestrel.sql.ast.FromItem import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.GroupingSpec +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.JoinType +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.MergeWhenClause +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.OnConflictClause +import kestrel.sql.ast.OrderBy +import kestrel.sql.ast.OrderKey +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.RenameColumn +import kestrel.sql.ast.RenameTable +import kestrel.sql.ast.RollupSpec +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.SetOperator +import kestrel.sql.ast.StarProjection import kestrel.sql.ast.Statement +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.sql.ast.TruncateTableStatement import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WithQuery +import kestrel.storage.Collation import kestrel.storage.Value +import kestrel.storage.displayWith object SqlWriter { + fun expressionText(expression: Expression): String = expression(expression) + fun canPersist(statement: Statement): Boolean = statement is CreateTableStatement || statement is DropTableStatement || + statement is TruncateTableStatement || + statement is AlterTableStatement || statement is CreateIndexStatement || statement is DropIndexStatement || + statement is CreateViewStatement || + statement is DropViewStatement || + statement is CreateTriggerStatement || + statement is DropTriggerStatement || + statement is CreateSequenceStatement || + statement is DropSequenceStatement || + statement is AlterSequenceStatement || statement is InsertStatement || statement is UpdateStatement || - statement is DeleteStatement + statement is DeleteStatement || + statement is MergeStatement fun write(statement: Statement): String = when (statement) { - is CreateTableStatement -> "CREATE TABLE ${statement.tableName} (" + - statement.columns.joinToString(", ") { columnDefinition(it) } + ");" - is DropTableStatement -> "DROP TABLE ${statement.tableName};" + is CreateTableStatement -> "CREATE ${if (statement.temporary) "TEMPORARY " else ""}TABLE ${statement.tableName} (" + + createTableBodyText( + statement.columns, + statement.foreignKeys, + statement.checks, + statement.uniqueConstraints, + statement.compositePrimaryKey + ) + ");" + is DropTableStatement -> "DROP TABLE ${statement.tableName}${dropModeText(statement.mode)};" + is TruncateTableStatement -> writeTruncate(statement) + is AlterTableStatement -> alterTableText(statement) is CreateIndexStatement -> - "CREATE INDEX ${statement.indexName} ON ${statement.tableName} (${statement.columnName});" + "CREATE ${if (statement.unique) "UNIQUE " else ""}INDEX ${statement.indexName} ON " + + "${statement.tableName} (${statement.columnNames.joinToString(", ")});" is DropIndexStatement -> "DROP INDEX ${statement.indexName};" + is CreateViewStatement -> + "CREATE ${if (statement.orReplace) "OR REPLACE " else ""}VIEW ${statement.viewName} AS " + + "${queryText(statement.query)};" + is DropViewStatement -> "DROP VIEW ${statement.viewName}${dropModeText(statement.mode)};" + is CreateTriggerStatement -> triggerText( + statement.triggerName, statement.timing, statement.event, statement.tableName, statement.whenClause, statement.body + ) + is DropTriggerStatement -> "DROP TRIGGER ${statement.triggerName};" + is CreateSequenceStatement -> createSequenceText(statement) + is DropSequenceStatement -> "DROP SEQUENCE ${statement.sequenceName};" + is AlterSequenceStatement -> alterSequenceText(statement) is InsertStatement -> writeInsert(statement) is UpdateStatement -> writeUpdate(statement) is DeleteStatement -> writeDelete(statement) + is MergeStatement -> writeMerge(statement) else -> throw IllegalArgumentException("statement is not persistable") } + fun triggerText(definition: TriggerDefinition): String = + triggerText(definition.name, definition.timing, definition.event, definition.tableName, definition.whenClause, definition.body) + + private fun triggerText( + name: String, + timing: TriggerTiming, + event: TriggerEvent, + tableName: String, + whenClause: Expression?, + body: List + ): String = buildString { + append("CREATE TRIGGER $name ") + append(if (timing == TriggerTiming.Before) "BEFORE" else "AFTER") + append(" ") + append( + when (event) { + TriggerEvent.Insert -> "INSERT" + TriggerEvent.Update -> "UPDATE" + TriggerEvent.Delete -> "DELETE" + } + ) + append(" ON $tableName FOR EACH ROW") + whenClause?.let { append(" WHEN ${expression(it)}") } + append(" BEGIN ") + for (statement in body) { + append(write(statement)) + append(" ") + } + append("END;") + } + + private fun createSequenceText(statement: CreateSequenceStatement): String = buildString { + append("CREATE SEQUENCE ${statement.sequenceName}") + statement.incrementBy?.let { append(" INCREMENT BY $it") } + statement.minValue?.let { append(" MINVALUE $it") } + statement.maxValue?.let { append(" MAXVALUE $it") } + statement.startValue?.let { append(" START WITH $it") } + if (statement.cycle) append(" CYCLE") + append(";") + } + + private fun alterSequenceText(statement: AlterSequenceStatement): String = buildString { + append("ALTER SEQUENCE ${statement.sequenceName}") + statement.incrementBy?.let { append(" INCREMENT BY $it") } + when { + statement.clearMinValue -> append(" NO MINVALUE") + statement.minValue != null -> append(" MINVALUE ${statement.minValue}") + } + when { + statement.clearMaxValue -> append(" NO MAXVALUE") + statement.maxValue != null -> append(" MAXVALUE ${statement.maxValue}") + } + if (statement.restart) { + append(" RESTART") + statement.restartWith?.let { append(" WITH $it") } + } + when (statement.cycle) { + true -> append(" CYCLE") + false -> append(" NO CYCLE") + null -> Unit + } + append(";") + } + + /** + * Bakes the sequence's CURRENT position into `START WITH`, rather than dumping + * [SequenceDefinition.startValue] as written at CREATE time, so a reload resumes NEXTVAL + * exactly where this session left off instead of reissuing already-handed-out values - see + * [Sequence]'s own doc on why the counter (unlike the CREATE/DROP/ALTER SEQUENCE structure + * every other part of [dump] is free to just replay literally) needs this special treatment. + */ + private fun sequenceDumpText(sequence: Sequence): String { + val definition = sequence.definition + val cycle = if (definition.cycle) " CYCLE" else "" + return "CREATE SEQUENCE ${definition.name} INCREMENT BY ${definition.incrementBy} " + + "MINVALUE ${definition.minValue} MAXVALUE ${definition.maxValue} START WITH ${sequence.peekNext()}$cycle;" + } + fun dump(catalog: Catalog): String { val builder = StringBuilder() + // Sequences are dumped FIRST, strictly before every table - a table's own column DEFAULT + // can read `NEXTVAL('seq')` (see kestrel.engine.Executor.executeCreateTable's eager + // `catalog.sequence(...)` existence check), so replaying that CREATE TABLE on reload would + // fail with "no such sequence" if the sequence it names did not already exist yet. + for (definition in catalog.sequenceDefinitions()) { + builder.append("${sequenceDumpText(catalog.sequence(definition.name))}\n") + } for (tableName in catalog.tableNames) { val table = catalog.table(tableName) + // A TEMP/TEMPORARY table (see kestrel.catalog.TableSchema.temporary) is session-scoped by + // definition - dumping it (or its rows below) into a checkpoint would make it survive a + // reload, which is exactly what "temporary" promises it never does. + if (table.schema.temporary) continue builder.append("CREATE TABLE ${table.schema.name} (") - builder.append(table.schema.columns.joinToString(", ") { columnDefinition(it) }) + builder.append( + createTableBodyText( + table.schema.columns, + table.schema.foreignKeys, + table.schema.checks, + table.schema.uniqueConstraints, + table.schema.compositePrimaryKey + ) + ) builder.append(");\n") for (row in table.rows) { builder.append("INSERT INTO ${table.schema.name} VALUES (") - builder.append(row.values.joinToString(", ") { literal(it) }) + // A GENERATED column's position is dumped as the literal `DEFAULT` keyword rather + // than its already-computed value - the executor rejects an explicit value for a + // GENERATED column outright (see Executor.rejectGeneratedAssignment), and `DEFAULT` + // here triggers the exact same fresh-recompute path an omitted column would, which + // reproduces the same value on replay since the expression is a pure function of + // this row's other (literal, already-dumped) columns. + builder.append( + row.values.mapIndexed { index, value -> + if (table.schema.columns[index].generated != null) "DEFAULT" else literal(value) + }.joinToString(", ") + ) builder.append(");\n") } } for (definition in catalog.indexDefinitions()) { - builder.append("CREATE INDEX ${definition.name} ON ${definition.tableName} (${definition.columnName});\n") + // An index defined on a TEMP table never gets a CREATE TABLE to attach to above, so + // dumping it would replay as "no such table" - skip it right along with its table. + if (catalog.tableOrNull(definition.tableName)?.schema?.temporary == true) continue + builder.append( + "CREATE ${if (definition.unique) "UNIQUE " else ""}INDEX ${definition.name} ON " + + "${definition.tableName} (${definition.columnDisplayName});\n" + ) + } + // A VIEW whose query reads a TEMP table is NOT specially excluded here (unlike the index/ + // trigger cases above, its dependency can be buried arbitrarily deep in a join/subquery, not + // one flat tableName field) - dumping it and replaying it against a reload that never + // recreated that TEMP table fails loudly with "no such table", the same honest failure mode + // this engine already accepts for e.g. a prepared statement that does not survive reopen, + // rather than silently materializing an empty or wrong view. + for (viewName in catalog.viewNames) { + val view = catalog.viewOrNull(viewName)!! + builder.append("CREATE VIEW ${view.name} AS ${queryText(view.query)};\n") + } + // Triggers are dumped LAST, strictly after every row above has already been inserted as + // plain data - a trigger created before its table's INSERTs would re-fire (duplicating an + // AFTER trigger's side effects, or re-running a BEFORE trigger's validation) the moment + // this very dump is replayed on reload, since replay re-executes each of these statements + // for real. Creating the trigger only once the table is already fully populated means + // those historical rows never pass through it, while every INSERT/UPDATE/DELETE a user + // issues after reload still does. + for (definition in catalog.triggerDefinitions()) { + // Same reasoning as the index skip above: a trigger ON a TEMP table has no table left to + // attach to after reload. + if (catalog.tableOrNull(definition.tableName)?.schema?.temporary == true) continue + builder.append("${triggerText(definition)}\n") } return builder.toString() } @@ -64,9 +312,23 @@ object SqlWriter { private fun writeInsert(statement: InsertStatement): String { val columns = statement.columns?.let { " (${it.joinToString(", ")})" } ?: "" val rows = statement.rows.joinToString(", ") { row -> - "(" + row.joinToString(", ") { expression(it) } + ")" + "(" + row.joinToString(", ") { it?.let { expression(it) } ?: "DEFAULT" } + ")" } - return "INSERT INTO ${statement.tableName}$columns VALUES $rows;" + val onConflict = statement.onConflict?.let { " ${onConflictText(it)}" } ?: "" + return "INSERT INTO ${statement.tableName}$columns VALUES $rows$onConflict;" + } + + private fun onConflictText(clause: OnConflictClause): String { + val target = if (clause.targetColumns.isEmpty()) "" else " (${clause.targetColumns.joinToString(", ")})" + val action = when (val action = clause.action) { + ConflictDoNothing -> "DO NOTHING" + is ConflictDoUpdate -> { + val assignments = action.assignments.joinToString(", ") { assignment(it) } + val where = action.where?.let { " WHERE ${expression(it)}" } ?: "" + "DO UPDATE SET $assignments$where" + } + } + return "ON CONFLICT$target $action" } private fun writeUpdate(statement: UpdateStatement): String { @@ -75,38 +337,338 @@ object SqlWriter { return "UPDATE ${statement.tableName} SET $assignments$where;" } + private fun writeTruncate(statement: TruncateTableStatement): String { + val identity = if (statement.restartIdentity) " RESTART IDENTITY" else "" + return "TRUNCATE TABLE ${statement.tableNames.joinToString(", ")}$identity${dropModeText(statement.mode)};" + } + private fun writeDelete(statement: DeleteStatement): String { val where = statement.where?.let { " WHERE ${expression(it)}" } ?: "" return "DELETE FROM ${statement.tableName}$where;" } - private fun assignment(assignment: Assignment): String = - "${assignment.columnName} = ${expression(assignment.value)}" - - private fun columnDefinition(column: Column): String { - val constraint = when { - column.primaryKey -> " PRIMARY KEY" - column.notNull -> " NOT NULL" - else -> "" - } - return "${column.name} ${column.type.displayName}$constraint" + /** + * Persisted literally like every other DML statement (see [writeInsert]/[writeUpdate]): by the + * time this redo-log entry replays, every earlier statement - including whatever populated a + * table-shaped `USING` source - has already replayed too, so re-running the exact same MERGE + * text against the freshly rebuilt catalog reproduces the same result deterministically. + */ + private fun writeMerge(statement: MergeStatement): String { + val targetAlias = statement.targetAlias?.let { " AS $it" } ?: "" + val whenClauses = statement.whenClauses.joinToString(" ") { mergeWhenClauseText(it) } + return "MERGE INTO ${statement.targetTable}$targetAlias USING ${fromItemText(statement.source)} " + + "ON ${expression(statement.onCondition)} $whenClauses;" } + private fun mergeWhenClauseText(clause: MergeWhenClause): String { + val condition = clause.condition?.let { " AND ${expression(it)}" } ?: "" + return when (clause) { + is MergeMatchedUpdate -> + "WHEN MATCHED$condition THEN UPDATE SET ${clause.assignments.joinToString(", ") { assignment(it) }}" + is MergeMatchedDelete -> "WHEN MATCHED$condition THEN DELETE" + is MergeNotMatchedInsert -> { + val columns = clause.columns?.let { " (${it.joinToString(", ")})" } ?: "" + val values = clause.values.joinToString(", ") { it?.let { value -> expression(value) } ?: "DEFAULT" } + "WHEN NOT MATCHED$condition THEN INSERT$columns VALUES ($values)" + } + } + } + + private fun assignment(assignment: Assignment): String = + "${assignment.columnName} = ${assignment.value?.let { expression(it) } ?: "DEFAULT"}" + + private fun columnDefinition(column: Column): String = buildString { + append(column.name) + append(" ") + append(column.type.displayWith(column.precision, column.scale)) + when { + column.primaryKey -> append(" PRIMARY KEY") + column.notNull -> append(" NOT NULL") + } + if (column.collation != Collation.Binary) append(" COLLATE ${column.collation.sqlName}") + column.default?.let { append(" DEFAULT ${expression(it)}") } + column.generated?.let { append(" GENERATED ALWAYS AS (${expression(it)}) STORED") } + } + + private fun alterTableText(statement: AlterTableStatement): String { + val action = when (val action = statement.action) { + is AddColumn -> "ADD COLUMN ${columnDefinition(action.column)}" + is DropColumn -> "DROP COLUMN ${action.columnName}${dropModeText(action.mode)}" + is RenameColumn -> "RENAME COLUMN ${action.oldName} TO ${action.newName}" + is RenameTable -> "RENAME TO ${action.newName}" + is AddConstraint -> "ADD ${addedConstraintText(action.constraint)}" + is DropConstraint -> "DROP CONSTRAINT ${action.name}${dropModeText(action.mode)}" + is ChangeColumnType -> { + val using = action.using?.let { " USING ${expression(it)}" } ?: "" + "ALTER COLUMN ${action.columnName} TYPE ${action.newType.displayWith(action.precision, action.scale)}$using" + } + } + return "ALTER TABLE ${statement.tableName} $action;" + } + + private fun addedConstraintText(constraint: AddedConstraint): String = when (constraint) { + is AddedCheck -> constraintText(constraint.check.name, "CHECK (${expression(constraint.check.expression)})") + is AddedUnique -> constraintText(constraint.unique.name, "UNIQUE (${constraint.unique.columns.joinToString(", ")})") + is AddedForeignKey -> constraintText(constraint.foreignKey.name, foreignKeyText(constraint.foreignKey)) + } + + private fun constraintText(name: String?, body: String): String = + if (name != null) "CONSTRAINT $name $body" else body + + private fun createTableBodyText( + columns: List, + foreignKeys: List, + checks: List, + uniqueConstraints: List, + compositePrimaryKey: List = emptyList() + ): String { + val parts = columns.map { columnDefinition(it) } + + (if (compositePrimaryKey.isNotEmpty()) listOf("PRIMARY KEY (${compositePrimaryKey.joinToString(", ")})") else emptyList()) + + foreignKeys.map { constraintText(it.name, foreignKeyText(it)) } + + checks.map { constraintText(it.name, "CHECK (${expression(it.expression)})") } + + uniqueConstraints.map { constraintText(it.name, "UNIQUE (${it.columns.joinToString(", ")})") } + return parts.joinToString(", ") + } + + private fun foreignKeyText(fk: ForeignKeyConstraint): String = buildString { + append("FOREIGN KEY (${fk.columns.joinToString(", ")}) REFERENCES ${fk.referencedTable} (${fk.referencedColumns.joinToString(", ")})") + if (fk.onDelete != ForeignKeyAction.NoAction) append(" ON DELETE ${fk.onDelete.sqlKeyword}") + if (fk.onUpdate != ForeignKeyAction.NoAction) append(" ON UPDATE ${fk.onUpdate.sqlKeyword}") + } + + private fun dropModeText(mode: DropMode): String = if (mode == DropMode.Cascade) " CASCADE" else "" + private fun expression(expression: Expression): String = when (expression) { is LiteralExpression -> literal(expression.value) is ColumnExpression -> expression.qualifier?.let { "$it.${expression.name}" } ?: expression.name - is FunctionExpression -> - "${expression.name}(${if (expression.star) "*" else expression.argument?.let { expression(it) } ?: ""})" + is FunctionExpression -> "${expression.name}(${functionArgumentsText(expression)})${filterClauseText(expression)}" is BinaryExpression -> "(${expression(expression.left)} ${expression.operator.symbol} ${expression(expression.right)})" is UnaryExpression -> "(${expression.operator.symbol} ${expression(expression.operand)})" + is CallExpression -> "${expression.name}(${expression.arguments.joinToString(", ") { expression(it) }})" + is IsNullExpression -> + "(${expression(expression.operand)} IS ${if (expression.negated) "NOT " else ""}NULL)" + is InExpression -> + "(${expression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (${expression.values.joinToString(", ") { expression(it) }}))" + is BetweenExpression -> + "(${expression(expression.operand)} ${if (expression.negated) "NOT " else ""}BETWEEN ${expression(expression.low)} AND ${expression(expression.high)})" + is LikeExpression -> + "(${expression(expression.operand)} ${if (expression.negated) "NOT " else ""}LIKE ${expression(expression.pattern)})" + is CaseExpression -> buildString { + append("CASE ") + expression.operand?.let { append(expression(it)); append(' ') } + expression.branches.forEach { append("WHEN ${expression(it.condition)} THEN ${expression(it.result)} ") } + expression.elseExpression?.let { append("ELSE ${expression(it)} ") } + append("END") + } + is ScalarSubqueryExpression -> "(${selectText(expression.select)})" + is ExistsExpression -> "EXISTS (${selectText(expression.select)})" + is InSubqueryExpression -> + "(${expression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (${selectText(expression.select)}))" + is CastExpression -> + "CAST(${expression(expression.operand)} AS ${expression.targetType.displayWith(expression.precision, expression.scale)})" + is CollateExpression -> "(${expression(expression.operand)} COLLATE ${expression.collation.sqlName})" + is WindowExpression -> "${expression.name}(${windowArgumentsText(expression)}) OVER (${overClauseText(expression)})" + is NextValueExpression -> "NEXTVAL(${literal(Value.TextValue(expression.sequenceName))})" + is CurrentValueExpression -> "CURRVAL(${literal(Value.TextValue(expression.sequenceName))})" + // Unreachable in practice: writeMerge never emits statement.returning at all (RETURNING is + // dropped from every persisted DML statement - see writeInsert/writeUpdate/writeDelete's own + // callers), so MERGE_ACTION() - only ever legal inside a MERGE's RETURNING - never reaches + // this writer. Kept here only to keep `when` exhaustive. + is MergeActionExpression -> "MERGE_ACTION()" + // Never actually reaches here in practice - canPersist excludes PREPARE/EXECUTE/DEALLOCATE + // outright (see its own doc), and resolveExecute always substitutes every ParameterExpression + // for a literal before the concrete INSERT/UPDATE/DELETE/SELECT it resolved to is ever handed + // to this writer. Kept only so `when` stays exhaustive, and so a `$1` typed directly into a + // persistable statement outside PREPARE (structurally legal, just meaningless - Evaluator + // throws on it) still round-trips its own text faithfully rather than being silently mangled. + is ParameterExpression -> "\$${expression.index}" } + /** STRING_AGG's `separator`/`orderBy` and DISTINCT are the only [FunctionExpression] fields beyond `argument`/`star` - every other aggregate falls through the `?: ""` exactly as before. */ + private fun functionArgumentsText(function: FunctionExpression): String { + if (function.star) return "*" + val distinctPrefix = if (function.distinct) "DISTINCT " else "" + val value = function.argument?.let { expression(it) } ?: "" + val separator = function.separator ?: return "$distinctPrefix$value" + val orderBy = if (function.orderBy.isNotEmpty()) { + " ORDER BY " + function.orderBy.joinToString(", ") { key -> expression(key.expression) + orderKeySuffix(key) } + } else "" + return "$distinctPrefix$value, ${expression(separator)}$orderBy" + } + + /** `DESC`/`NULLS FIRST`/`NULLS LAST` are omitted when unspecified, so a plain ascending key with the engine's own default null ordering dumps exactly as written, matching every other optional-clause writer in this file. */ + private fun orderKeySuffix(key: OrderKey): String { + val direction = if (key.descending) " DESC" else "" + val nulls = when (key.nullsFirst) { + true -> " NULLS FIRST" + false -> " NULLS LAST" + null -> "" + } + return direction + nulls + } + + /** Without this, a view with `COUNT(x) FILTER (WHERE ...)` would dump as plain `COUNT(x)` and silently drop the predicate on reopen. */ + private fun filterClauseText(function: FunctionExpression): String = + function.filter?.let { " FILTER (WHERE ${expression(it)})" } ?: "" + + private fun windowArgumentsText(window: WindowExpression): String { + if (window.star) return "*" + return listOfNotNull(window.argument, window.offset, window.defaultValue).joinToString(", ") { expression(it) } + } + + private fun overClauseText(window: WindowExpression): String { + val parts = mutableListOf() + if (window.partitionBy.isNotEmpty()) { + parts.add("PARTITION BY " + window.partitionBy.joinToString(", ") { expression(it) }) + } + if (window.orderBy.isNotEmpty()) { + parts.add( + "ORDER BY " + window.orderBy.joinToString(", ") { key -> expression(key.expression) + orderKeySuffix(key) } + ) + } + window.frame?.let { frame -> + val unit = when (frame.unit) { + FrameUnit.Rows -> "ROWS" + FrameUnit.Range -> "RANGE" + FrameUnit.Groups -> "GROUPS" + } + val exclude = when (frame.exclude) { + FrameExclude.NoOthers -> "" + FrameExclude.CurrentRow -> " EXCLUDE CURRENT ROW" + FrameExclude.Group -> " EXCLUDE GROUP" + FrameExclude.Ties -> " EXCLUDE TIES" + } + parts.add("$unit BETWEEN ${frameBoundText(frame.start)} AND ${frameBoundText(frame.end)}$exclude") + } + return parts.joinToString(" ") + } + + private fun frameBoundText(bound: FrameBound): String = when (bound.type) { + FrameBoundType.UnboundedPreceding -> "UNBOUNDED PRECEDING" + FrameBoundType.UnboundedFollowing -> "UNBOUNDED FOLLOWING" + FrameBoundType.CurrentRow -> "CURRENT ROW" + FrameBoundType.Preceding -> "${expression(bound.offset!!)} PRECEDING" + FrameBoundType.Following -> "${expression(bound.offset!!)} FOLLOWING" + } + + private fun queryText(query: QueryStatement): String = when (query) { + is SelectStatement -> selectText(query) + is SetOperationStatement -> setOperationText(query) + is WithQuery -> withQueryText(query) + } + + private fun setOperationText(statement: SetOperationStatement): String = buildString { + append(queryText(statement.left)) + append(" ") + append(setOperatorKeyword(statement.operator)) + append(" ") + append(queryText(statement.right)) + statement.orderBy?.let { append(orderByText(it)) } + statement.limit?.let { append(" LIMIT $it") } + statement.offset?.let { append(" OFFSET $it") } + } + + private fun setOperatorKeyword(operator: SetOperator): String = when (operator) { + SetOperator.Union -> "UNION" + SetOperator.UnionAll -> "UNION ALL" + SetOperator.Intersect -> "INTERSECT" + SetOperator.IntersectAll -> "INTERSECT ALL" + SetOperator.Except -> "EXCEPT" + SetOperator.ExceptAll -> "EXCEPT ALL" + } + + private fun withQueryText(query: WithQuery): String = buildString { + append("WITH ") + if (query.recursive) append("RECURSIVE ") + append(query.ctes.joinToString(", ") { cte -> + val columns = cte.columns?.let { "(${it.joinToString(", ")})" } ?: "" + "${cte.name}$columns AS (${queryText(cte.query)})" + }) + append(" ") + append(queryText(query.body)) + } + + private fun selectText(select: SelectStatement): String = buildString { + append("SELECT ") + if (select.distinct) { + append("DISTINCT ") + if (select.distinctOn.isNotEmpty()) { + append("ON (") + append(select.distinctOn.joinToString(", ") { expression(it) }) + append(") ") + } + } + append(projectionText(select.projection)) + append(" FROM ") + append(fromItemText(select.from)) + for (join in select.joins) { + append( + when (join.type) { + JoinType.Left -> " LEFT JOIN " + JoinType.Right -> " RIGHT JOIN " + JoinType.Full -> " FULL JOIN " + JoinType.Inner -> " INNER JOIN " + JoinType.Cross -> " CROSS JOIN " + } + ) + append(fromItemText(join.from)) + join.on?.let { append(" ON "); append(expression(it)) } + } + select.where?.let { append(" WHERE "); append(expression(it)) } + if (select.groupingSpec != null || select.groupBy.isNotEmpty()) { + append(" GROUP BY ") + append(groupByText(select.groupingSpec, select.groupBy)) + } + select.having?.let { append(" HAVING "); append(expression(it)) } + select.orderBy?.let { append(orderByText(it)) } + select.limit?.let { append(" LIMIT $it") } + select.offset?.let { append(" OFFSET $it") } + } + + private fun groupByText(spec: GroupingSpec?, groupBy: List): String = when (spec) { + null -> groupBy.joinToString(", ") { expression(it) } + is RollupSpec -> "ROLLUP (" + spec.expressions.joinToString(", ") { expression(it) } + ")" + is CubeSpec -> "CUBE (" + spec.expressions.joinToString(", ") { expression(it) } + ")" + is GroupingSetsSpec -> "GROUPING SETS (" + spec.sets.joinToString(", ") { groupingSetElementText(it) } + ")" + } + + private fun groupingSetElementText(set: List): String = + if (set.isEmpty()) "()" else "(" + set.joinToString(", ") { expression(it) } + ")" + + private fun orderByText(orderBy: OrderBy): String = + " ORDER BY " + orderBy.keys.joinToString(", ") { key -> expression(key.expression) + orderKeySuffix(key) } + + private fun fromItemText(item: FromItem): String = when (item) { + is TableSource -> item.name + (item.alias?.let { " AS $it" } ?: "") + is SubquerySource -> (if (item.lateral) "LATERAL " else "") + "(${queryText(item.select)}) AS ${item.alias}" + } + + private fun projectionText(projection: Projection): String = when (projection) { + is StarProjection -> "*" + is ExpressionProjection -> projection.items.joinToString(", ") { item -> + val base = expression(item.expression) + item.alias?.let { "$base AS $it" } ?: base + } + } + + /** + * DECIMAL/DATE values render as `CAST('...' AS type)` (not a bare quoted string) + * because column coercion is strict everywhere else in this engine (no implicit + * TEXT-to-DECIMAL/DATE on INSERT/UPDATE) - the CAST is what actually converts the + * text back on replay. The re-cast uses the value's own scale (already the column's + * declared scale, since it can only have gotten here via coercion or CAST) with a + * generous precision, so it reproduces the exact BigDecimal rather than rounding + * through a Double the way a bare numeric literal would. + */ private fun literal(value: Value): String = when (value) { is Value.IntegerValue -> value.value.toString() is Value.RealValue -> realText(value.value) is Value.TextValue -> "'" + value.value.replace("'", "''") + "'" is Value.BooleanValue -> value.value.toString() + is Value.DecimalValue -> + "CAST('${value.value.toPlainString()}' AS DECIMAL($LITERAL_DECIMAL_PRECISION, ${value.value.scale()}))" + is Value.DateValue -> "CAST('${value.value}' AS DATE)" is Value.NullValue -> "NULL" } @@ -114,4 +676,6 @@ object SqlWriter { val plain = value.toString() return if (plain.contains('E') || plain.contains('e')) BigDecimal(value).toPlainString() else plain } + + private const val LITERAL_DECIMAL_PRECISION = 38 } diff --git a/src/main/kotlin/kestrel/sql/ast/Ast.kt b/src/main/kotlin/kestrel/sql/ast/Ast.kt index 3c9ec59..2490118 100644 --- a/src/main/kotlin/kestrel/sql/ast/Ast.kt +++ b/src/main/kotlin/kestrel/sql/ast/Ast.kt @@ -1,49 +1,359 @@ package kestrel.sql.ast +import kestrel.catalog.CheckConstraint import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.UniqueConstraint +import kestrel.storage.Collation +import kestrel.storage.ColumnType import kestrel.storage.Value sealed interface Statement -data class CreateTableStatement(val tableName: String, val columns: List) : Statement +data class CreateTableStatement( + val tableName: String, + val columns: List, + val foreignKeys: List = emptyList(), + val checks: List = emptyList(), + val uniqueConstraints: List = emptyList(), + val compositePrimaryKey: List = emptyList(), + /** `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` - see [kestrel.catalog.TableSchema.temporary]. */ + val temporary: Boolean = false +) : Statement -data class DropTableStatement(val tableName: String) : Statement +enum class DropMode { Restrict, Cascade } + +data class DropTableStatement(val tableName: String, val mode: DropMode = DropMode.Restrict) : Statement + +/** + * `TRUNCATE TABLE t1 [, t2, ...] [RESTART IDENTITY | CONTINUE IDENTITY] [CASCADE | RESTRICT]`. + * Unlike a WHERE-less DELETE, TRUNCATE never fires row-level triggers and never runs a FOREIGN KEY's + * own ON DELETE action row by row - see [kestrel.catalog.Catalog.truncateTables]/ + * [kestrel.store.sql.DiskCatalog.truncateTables] for why [mode] means something structurally + * different here than it does for [DropTableStatement]/[DropViewStatement]. + */ +data class TruncateTableStatement( + val tableNames: List, + val restartIdentity: Boolean = false, + val mode: DropMode = DropMode.Restrict +) : Statement data class CreateIndexStatement( val indexName: String, val tableName: String, - val columnName: String + val columnNames: List, + val unique: Boolean = false ) : Statement data class DropIndexStatement(val indexName: String) : Statement +sealed interface AlterTableAction + +data class AddColumn(val column: Column) : AlterTableAction + +data class DropColumn(val columnName: String, val mode: DropMode = DropMode.Restrict) : AlterTableAction + +data class RenameColumn(val oldName: String, val newName: String) : AlterTableAction + +data class RenameTable(val newName: String) : AlterTableAction + +/** + * `ALTER TABLE t ALTER [COLUMN] col TYPE newtype [USING expression]` (Postgres syntax). [using], + * when given, is evaluated per existing row against that row's OLD values (exactly like an UPDATE + * SET assignment) and the result coerced into [newType]/[precision]/[scale]; when omitted it + * defaults to a bare reference to [columnName] itself - "recompute this column as its own old + * value, coerced to the new type" - so a widening change (INTEGER -> DECIMAL, say) needs no + * explicit `USING` at all, matching Postgres' own default behavior. + */ +data class ChangeColumnType( + val columnName: String, + val newType: ColumnType, + val precision: Int? = null, + val scale: Int? = null, + val using: Expression? = null +) : AlterTableAction + +/** + * `ALTER TABLE t ADD CONSTRAINT name CHECK (...) | UNIQUE (...) | FOREIGN KEY (...) REFERENCES ...` - + * the constraint's own name lives inside [AddedCheck.check]/[AddedUnique.unique]/[AddedForeignKey.foreignKey] + * (each already carries a `name: String?`, shared with the identical table-level `CONSTRAINT name` clause + * a `CREATE TABLE` can declare inline), not as a separate field here. + */ +sealed interface AddedConstraint +data class AddedCheck(val check: CheckConstraint) : AddedConstraint +data class AddedUnique(val unique: UniqueConstraint) : AddedConstraint +data class AddedForeignKey(val foreignKey: ForeignKeyConstraint) : AddedConstraint + +data class AddConstraint(val constraint: AddedConstraint) : AlterTableAction + +/** `ALTER TABLE t DROP CONSTRAINT name [RESTRICT | CASCADE]` - see [kestrel.catalog.Catalog.dropConstraint]. */ +data class DropConstraint(val name: String, val mode: DropMode = DropMode.Restrict) : AlterTableAction + +data class AlterTableStatement(val tableName: String, val action: AlterTableAction) : Statement + +data class CreateViewStatement( + val viewName: String, + val query: QueryStatement, + val orReplace: Boolean +) : Statement + +data class DropViewStatement(val viewName: String, val mode: DropMode = DropMode.Restrict) : Statement + data class InsertStatement( val tableName: String, val columns: List?, - val rows: List> -) : Statement + val rows: List>, + val onConflict: OnConflictClause? = null, + val returning: Projection? = null +) : PreparableStatement + +/** Empty [targetColumns] means "any unique or primary key violation" (bare `ON CONFLICT DO ...`), matching Postgres. */ +data class OnConflictClause(val targetColumns: List, val action: ConflictAction) + +sealed interface ConflictAction + +data object ConflictDoNothing : ConflictAction + +data class ConflictDoUpdate(val assignments: List, val where: Expression?) : ConflictAction + +/** + * The statement kinds [PrepareStatement] can wrap: a SELECT/set-operation/CTE ([QueryStatement]) + * or a data-mutating [InsertStatement]/[UpdateStatement]/[DeleteStatement]/[MergeStatement] - the + * same "any DML or query, never DDL/transaction control" boundary Postgres' own PREPARE draws. + * Declared as its own sealed interface (rather than checking `is QueryStatement || is InsertStatement + * || ...` by hand at every call site) so the compiler enforces [ParameterSubstitution]'s `when` stays + * exhaustive as new statement kinds are added. + */ +sealed interface PreparableStatement : Statement + +sealed interface QueryStatement : Statement, PreparableStatement data class SelectStatement( val projection: Projection, - val tableName: String, - val tableAlias: String? = null, + val from: FromItem, val joins: List = emptyList(), val where: Expression? = null, + /** For [RollupSpec]/[CubeSpec]/[GroupingSetsSpec], this is the deduplicated union of every expression across every one of [groupingSpec]'s grouping sets (order = first appearance) - NOT the flat comma list SQL text would suggest. Every existing consumer that only needs "which expressions are legal grouping columns" (validation, dependency analysis, rename propagation, the plain-`GROUP BY a, b` write-back path) reads this field and needs no awareness of [groupingSpec] at all; only [kestrel.engine.GroupAggregation], which needs the actual per-set breakdown, reads [groupingSpec]. */ val groupBy: List = emptyList(), + val groupingSpec: GroupingSpec? = null, val having: Expression? = null, val orderBy: OrderBy? = null, - val limit: Int? = null -) : Statement + val limit: Int? = null, + /** `LIMIT`/`OFFSET` may be written in either order (matching Postgres) - applied together, after ORDER BY/DISTINCT dedup and before projection, skipping this many rows before [limit] (if any) caps what remains. */ + val offset: Int? = null, + /** + * `true` for both `SELECT DISTINCT ...` and `SELECT DISTINCT ON (...) ...` - `false` (the + * default, also written explicitly as `SELECT ALL ...`) means every source row that survives + * WHERE/GROUP BY/HAVING produces its own output row. See [distinctOn] for what distinguishes + * the two `true` forms from each other. + */ + val distinct: Boolean = false, + /** + * Empty for plain `SELECT DISTINCT` (dedup by the *entire* output row, evaluated after + * projection - see [kestrel.engine.SelectRunner]'s `Plan.projectRows`). Non-empty for + * `SELECT DISTINCT ON (expr, ...) ...`: keeps only the first source row per distinct + * combination of these expressions, evaluated the same way `GROUP BY` expressions are + * (pre-projection, against the joined/filtered row) - never resolved against a `SELECT`-list + * alias, matching Postgres' own grammar where `DISTINCT ON (...)` is parsed before the select + * list even exists. "First" means first in `ORDER BY` order when one is given (which is why the + * parser requires `ORDER BY`, if present at all, to start with exactly these expressions - the + * whole point of `DISTINCT ON` is picking a deterministic row per group), otherwise first in + * scan/join order. Deliberately rejected outright when combined with `GROUP BY` or aggregates - + * unlike plain [distinct], there is no unsurprising way to define "first row of a group" once + * the underlying rows have already been collapsed into aggregate groups. + */ + val distinctOn: List = emptyList() +) : QueryStatement -data class DeleteStatement(val tableName: String, val where: Expression?) : Statement +/** + * Distinguishes `GROUP BY ROLLUP (...)` / `CUBE (...)` / `GROUPING SETS (...)` from an ordinary + * `GROUP BY a, b, ...` (represented by `groupingSpec == null`, the only case before this existed). + * [kestrel.engine.GroupingSetExpansion] turns any of these into the concrete list of grouping sets + * [kestrel.engine.GroupAggregation] actually groups by - one full pass over the source rows per set, + * unioned together, so a single query reports subtotals (and, for [Rollup]/[Cube]/an explicit `()` + * element, a grand total) alongside whatever detail rows survive. Mixing plain columns with + * ROLLUP/CUBE in the same `GROUP BY` list (SQL allows this, producing a cross product of sets) is a + * scope cut - the parser only accepts one of these forms standing alone as the entire clause. + */ +sealed interface GroupingSpec + +data class RollupSpec(val expressions: List) : GroupingSpec + +data class CubeSpec(val expressions: List) : GroupingSpec + +/** Each element of [sets] is one grouping set as written; an empty inner list is the `()` grand-total set. */ +data class GroupingSetsSpec(val sets: List>) : GroupingSpec + +enum class SetOperator { Union, UnionAll, Intersect, IntersectAll, Except, ExceptAll } + +data class SetOperationStatement( + val operator: SetOperator, + val left: QueryStatement, + val right: QueryStatement, + val orderBy: OrderBy? = null, + val limit: Int? = null, + val offset: Int? = null +) : QueryStatement + +data class CommonTableExpression( + val name: String, + val columns: List?, + val query: QueryStatement +) + +data class WithQuery( + val recursive: Boolean, + val ctes: List, + val body: QueryStatement +) : QueryStatement + +sealed interface FromItem + +data class TableSource(val name: String, val alias: String? = null) : FromItem + +data class SubquerySource(val select: QueryStatement, val alias: String, val lateral: Boolean = false) : FromItem + +data class DeleteStatement( + val tableName: String, + val where: Expression?, + val returning: Projection? = null +) : PreparableStatement data class UpdateStatement( val tableName: String, val assignments: List, - val where: Expression? + val where: Expression?, + val returning: Projection? = null +) : PreparableStatement + +/** + * `MERGE INTO target [AS targetAlias] USING source ON onCondition whenClauses...`. [source] is the + * same [FromItem] a `FROM`/`USING`-shaped clause anywhere else in this grammar produces (a bare + * table name or a derived-table subquery, carrying its own alias), reused rather than inventing a + * MERGE-specific source type. [whenClauses] preserves the exact order they were written in - see + * [kestrel.engine.Executor]'s executeMerge for why that order (not a MATCHED-then-NOT-MATCHED split) + * is what standard MERGE semantics actually require: for a given source row, at most one category + * (MATCHED or NOT MATCHED) can ever apply, so evaluating the whole list in written order and + * skipping clauses of the other category naturally reproduces "first matching WHEN clause wins". + * Use from inside a trigger body is deliberately unsupported (the parser's trigger-body grammar only + * ever accepts INSERT/UPDATE/DELETE, a structural scope cut, not a runtime check). [returning], when + * present, projects one result row per action MERGE actually took (insert/update/delete), in the + * order those actions ran - see [kestrel.engine.Executor.evaluateMergeReturning]. + */ +data class MergeStatement( + val targetTable: String, + val targetAlias: String?, + val source: FromItem, + val onCondition: Expression, + val whenClauses: List, + val returning: Projection? = null +) : PreparableStatement + +sealed interface MergeWhenClause { + val condition: Expression? +} + +/** WHEN MATCHED [AND condition] THEN UPDATE SET assignments... - `assignments` targets are always unqualified target columns, matching plain UPDATE. */ +data class MergeMatchedUpdate(override val condition: Expression?, val assignments: List) : MergeWhenClause + +/** WHEN MATCHED [AND condition] THEN DELETE */ +data class MergeMatchedDelete(override val condition: Expression?) : MergeWhenClause + +/** + * WHEN NOT MATCHED [AND condition] THEN INSERT [(columns)] VALUES (values). `columns == null` means + * every target column in schema order, exactly like a column-list-less plain INSERT. Unlike an + * ordinary INSERT, [values] is always exactly one row (MERGE inserts at most one row per + * unmatched source row) and can only reference the source side - there is no target row yet. + */ +data class MergeNotMatchedInsert(override val condition: Expression?, val columns: List?, val values: List) : MergeWhenClause + +data class ExplainStatement(val select: QueryStatement) : Statement + +enum class TriggerTiming { Before, After } + +enum class TriggerEvent { Insert, Update, Delete } + +/** + * `body` is restricted by the parser to INSERT/UPDATE/DELETE statements (matching SQLite's own + * trigger-body grammar) - those are the only statement kinds with a side effect worth firing, and + * the only ones [kestrel.engine.Executor] knows how to run with a [kestrel.engine.TriggerRowContext] + * bound. `whenClause`, when present, is evaluated once per candidate row before [body] runs at all; + * a row for which it is false or NULL never fires this trigger. + */ +data class CreateTriggerStatement( + val triggerName: String, + val timing: TriggerTiming, + val event: TriggerEvent, + val tableName: String, + val whenClause: Expression?, + val body: List ) : Statement -data class ExplainStatement(val select: SelectStatement) : Statement +data class DropTriggerStatement(val triggerName: String) : Statement + +/** + * `incrementBy`/`minValue`/`maxValue`/`startValue` are already fully resolved by the parser's + * defaulting rules (see `Parser.parseSequenceOptions`) - `null` only ever means "not written by the + * user", never "no bound"; [kestrel.engine.Executor.executeCreateSequence] is what turns a `null` + * into Postgres' own ascending/descending default (MINVALUE 1 / MAXVALUE Long.MAX_VALUE and the + * reverse for a negative [incrementBy]) and validates the combination. + */ +data class CreateSequenceStatement( + val sequenceName: String, + val incrementBy: Long? = null, + val minValue: Long? = null, + val maxValue: Long? = null, + val startValue: Long? = null, + val cycle: Boolean = false +) : Statement + +data class DropSequenceStatement(val sequenceName: String) : Statement + +/** + * Every field besides [sequenceName] is "not mentioned in this ALTER" when `null`/`false` - + * [clearMinValue]/[clearMaxValue] (Postgres' `NO MINVALUE`/`NO MAXVALUE`) are tracked separately + * from [minValue]/[maxValue] because, unlike CREATE SEQUENCE, "not mentioned" and "explicitly + * cleared back to the type default" are different instructions on an ALTER: the former leaves + * today's bound untouched, the latter recomputes it from [incrementBy]'s sign exactly like a fresh + * CREATE SEQUENCE with no bound written at all would. [restart] with a `null` [restartWith] is a + * bare `RESTART` (reset to the sequence's original `START WITH`, unaffected by any earlier ALTER); + * [restart] `false` means no RESTART clause at all, leaving the counter wherever NEXTVAL last left it. + */ +data class AlterSequenceStatement( + val sequenceName: String, + val incrementBy: Long? = null, + val minValue: Long? = null, + val clearMinValue: Boolean = false, + val maxValue: Long? = null, + val clearMaxValue: Boolean = false, + val restart: Boolean = false, + val restartWith: Long? = null, + val cycle: Boolean? = null +) : Statement + +/** + * `PREPARE name AS ` - parses and stores [target] under [name] without running it. + * [kestrel.engine.Executor] keeps prepared statements in a session-scoped registry OUTSIDE + * [kestrel.catalog.Catalog] (unlike CREATE VIEW/TRIGGER/SEQUENCE) - a deliberate choice, not an + * oversight: they carry no data of their own (only an AST plus, at EXECUTE time, caller-supplied + * argument values), so BEGIN/ROLLBACK's catalog snapshot/restore has nothing meaningful to undo for + * them, and a name PREPARE-d before a transaction started should not vanish just because that + * transaction later rolled back. See [kestrel.engine.ParameterSubstitution] for how `$1`, `$2`, ... + * inside [target] are resolved once an EXECUTE actually supplies values. + */ +data class PrepareStatement(val name: String, val target: PreparableStatement) : Statement + +/** + * `EXECUTE name(arg, arg, ...)` (or `EXECUTE name` for a statement with no parameters) - + * [kestrel.engine.Executor.resolveExecute] looks [name] up, evaluates each [arguments] expression + * as a constant (no column/row context exists to resolve against), and substitutes the results for + * every [ParameterExpression] in the prepared statement before it actually runs - so whatever this + * resolves to is always one of [PreparableStatement]'s own concrete kinds, never `EXECUTE` itself. + */ +data class ExecuteStatement(val name: String, val arguments: List = emptyList()) : Statement + +/** `DEALLOCATE name` removes one prepared statement; `DEALLOCATE ALL` ([all] `true`, [name] `null`) removes every one. */ +data class DeallocateStatement(val name: String?, val all: Boolean = false) : Statement data object BeginStatement : Statement @@ -51,15 +361,22 @@ data object CommitStatement : Statement data object RollbackStatement : Statement -data class Assignment(val columnName: String, val value: Expression) +data class SavepointStatement(val name: String) : Statement -enum class JoinType { Inner, Left } +data class ReleaseSavepointStatement(val name: String) : Statement +data class RollbackToSavepointStatement(val name: String) : Statement + +/** `value == null` means the assignment was written as `col = DEFAULT` - the column's declared DEFAULT (or NULL without one), evaluated fresh per row rather than carrying a literal at parse time. */ +data class Assignment(val columnName: String, val value: Expression?) + +enum class JoinType { Inner, Left, Right, Full, Cross } + +/** [on] is null exactly for [JoinType.Cross] - a CROSS JOIN has no condition, matching every pairing. */ data class JoinClause( val type: JoinType, - val tableName: String, - val alias: String?, - val on: Expression + val from: FromItem, + val on: Expression? ) sealed interface Projection @@ -70,7 +387,16 @@ data class ExpressionProjection(val items: List) : Projection data class SelectItem(val expression: Expression, val alias: String?) -data class OrderBy(val expression: Expression, val descending: Boolean) +/** + * [nullsFirst] is `null` when the SQL text had no explicit `NULLS FIRST`/`NULLS LAST` - every + * comparator consuming this (see [kestrel.engine.compareOrderKeyValues]) then falls back to + * `!descending`, preserving this engine's original convention (NULL as the smallest value + * regardless of direction) from before this syntax existed. `true`/`false` here always means an + * explicit clause was present, independent of [descending]. + */ +data class OrderKey(val expression: Expression, val descending: Boolean, val nullsFirst: Boolean? = null) + +data class OrderBy(val keys: List) sealed interface Expression @@ -78,10 +404,175 @@ data class LiteralExpression(val value: Value) : Expression data class ColumnExpression(val name: String, val qualifier: String? = null) : Expression +/** + * [separator] and [orderBy] exist only for `STRING_AGG(value, separator [ORDER BY ...])` - every + * other aggregate (COUNT/SUM/AVG/MIN/MAX) leaves both null/empty. Kept on the shared node rather than + * a dedicated `StringAggExpression` variant so STRING_AGG rides every existing [FunctionExpression] + * validator (GROUP BY membership, CHECK/DEFAULT aggregate bans, subquery bans, ...) for free instead + * of needing a twelfth near-parallel `when` branch across the codebase. + * + * [distinct] is `COUNT(DISTINCT x)`/`SUM(DISTINCT x)`/`AVG(DISTINCT x)`/`MIN(DISTINCT x)`/ + * `MAX(DISTINCT x)` - never true together with [star] or alongside [separator] (the parser rejects + * `DISTINCT` on STRING_AGG's own call shape and on `*`), and never true on a windowed aggregate (the + * parser rejects `DISTINCT ... OVER (...)` outright, matching Postgres - there is no per-row windowed + * notion of "distinct so far"). + * + * [filter] is `aggregate_name(args) FILTER (WHERE condition)` - a per-row predicate evaluated against + * the same group the aggregate would otherwise see in full, restricting which rows reach the aggregate + * without needing a separate `CASE WHEN ... END` inside every argument. Never set on STRING_AGG (its + * own call shape returns before this field would ever be populated) or on a windowed aggregate (the + * parser rejects `FILTER` together with `OVER`, matching how [distinct] is rejected there too - there + * is no per-row windowed notion of a group to filter). + */ data class FunctionExpression( val name: String, val argument: Expression?, - val star: Boolean + val star: Boolean, + val separator: Expression? = null, + val orderBy: List = emptyList(), + val distinct: Boolean = false, + val filter: Expression? = null +) : Expression + +data class CallExpression(val name: String, val arguments: List) : Expression + +/** + * `NEXTVAL('seq')`/`CURRVAL('seq')`. Deliberately their own [Expression] variants rather than + * folding into [CallExpression] like every other function call - [sequenceName] is a name resolved + * once at parse time (the parser requires the argument to literally be a string literal, not an + * arbitrary expression), never a per-row value, which keeps every existing [CallExpression]-shaped + * validator (`Executor.validateConstantDefault`, `CheckExpressionColumns`, `GroupAggregation`, ...) + * untouched by NEXTVAL's side effect instead of needing a special case bolted onto each one. See + * [kestrel.engine.Executor.executeInsertInner]/`executeUpdateInner` for where these two are actually + * allowed to evaluate - CHECK constraints, trigger `WHEN` clauses, and SELECT/WHERE all reject them. + */ +data class NextValueExpression(val sequenceName: String) : Expression + +data class CurrentValueExpression(val sequenceName: String) : Expression + +/** + * `MERGE_ACTION()` - takes no arguments (the parser rejects any), so unlike [CallExpression] it + * carries no per-call state and is a `data object` rather than a `data class`. Only meaningful inside + * a [MergeStatement]'s own [MergeStatement.returning] projection, where it evaluates to `'INSERT'`, + * `'UPDATE'`, or `'DELETE'` depending on which action a given RETURNING row came from - see + * [kestrel.engine.Executor.evaluateMergeReturning] for where that per-row action is actually bound. + * Every other [Expression] consumer (CHECK constraints, WHERE, trigger bodies, plain INSERT/UPDATE/ + * DELETE RETURNING, ...) treats it as structurally harmless - no column reference to check, nothing + * to rename, nothing to depend on - exactly like [NextValueExpression]/[CurrentValueExpression] do, + * and instead rejects it the same way they reject NEXTVAL/CURRVAL: by never handing the + * [kestrel.engine.Evaluator] that runs it the per-row action string it needs, so evaluation itself + * throws. + */ +data object MergeActionExpression : Expression + +/** + * `$1`, `$2`, ... - a positional parameter placeholder, 1-based to match the SQL text. Only ever + * produced by the parser inside a [PrepareStatement]'s [PrepareStatement.target] (or, harmlessly, + * anywhere else an [Expression] is grammatically legal, since the parser has no context at parse + * time to reject it elsewhere) - [kestrel.engine.ParameterSubstitution] replaces every occurrence + * with a [LiteralExpression] before the surrounding statement actually runs, so by evaluation time + * ([kestrel.engine.Evaluator]) one reaching [kestrel.engine.Evaluator.evaluate] unsubstituted is + * always a usage error (typed outside of PREPARE/EXECUTE entirely) and throws there, exactly like + * [MergeActionExpression] throws when evaluated outside a MERGE's own RETURNING. + */ +data class ParameterExpression(val index: Int) : Expression + +data class IsNullExpression(val operand: Expression, val negated: Boolean) : Expression + +data class InExpression(val operand: Expression, val values: List, val negated: Boolean) : Expression + +data class BetweenExpression( + val operand: Expression, + val low: Expression, + val high: Expression, + val negated: Boolean +) : Expression + +data class LikeExpression(val operand: Expression, val pattern: Expression, val negated: Boolean) : Expression + +data class CaseExpression( + val operand: Expression?, + val branches: List, + val elseExpression: Expression? +) : Expression + +data class CaseBranch(val condition: Expression, val result: Expression) + +data class ScalarSubqueryExpression(val select: SelectStatement) : Expression + +data class ExistsExpression(val select: SelectStatement) : Expression + +data class InSubqueryExpression( + val operand: Expression, + val select: SelectStatement, + val negated: Boolean +) : Expression + +data class CastExpression( + val operand: Expression, + val targetType: ColumnType, + val precision: Int? = null, + val scale: Int? = null +) : Expression + +enum class WindowFunctionKind { RowNumber, Rank, DenseRank, Aggregate, Lag, Lead, FirstValue, LastValue, Ntile } + +/** Where a [FrameBound] sits relative to the current row; ordered exactly as SQL requires a frame's start to precede its end. */ +enum class FrameBoundType { UnboundedPreceding, Preceding, CurrentRow, Following, UnboundedFollowing } + +/** `UNBOUNDED PRECEDING`/`CURRENT ROW`/`UNBOUNDED FOLLOWING` carry no [offset]; `N PRECEDING`/`N FOLLOWING` evaluate it per row. */ +data class FrameBound(val type: FrameBoundType, val offset: Expression? = null) + +/** + * [Rows] counts physical row positions (`ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING` = the row before + * and after, regardless of ties). [Range] measures distance in the value of a single ORDER BY column + * (numeric or DATE) - `RANGE BETWEEN 5 PRECEDING AND 5 FOLLOWING` on `ORDER BY amount` includes every + * row whose `amount` is within 5 of the current row's, ties included, however many physical rows that + * is. [Groups] counts whole peer groups (ties on the OVER's ORDER BY) instead of rows or values - + * `GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING` includes every row in the peer group before the + * current one, the current peer group, and the one after, however many rows each group has. Unlike + * [Range] it works with any ORDER BY (zero, one, or many columns, any type), not just a single + * numeric/DATE column. + */ +enum class FrameUnit { Rows, Range, Groups } + +/** + * `EXCLUDE` narrows a frame after its [FrameBound]s are resolved, by removing rows tied to the + * current row on the OVER's ORDER BY. [NoOthers] (the default - no `EXCLUDE` clause at all) removes + * nothing. [CurrentRow] removes only the current row itself. [Group] removes the current row's + * entire peer group, including the current row. [Ties] removes every *other* row in the peer group + * but keeps the current row - the one exclusion mode where the current row's own value always still + * counts. With no ORDER BY the whole partition is one peer group, matching the same convention + * already used for the frame's own `CURRENT ROW` bound and for the implicit default frame. + */ +enum class FrameExclude { NoOthers, CurrentRow, Group, Ties } + +/** `ROWS|RANGE BETWEEN AND [EXCLUDE ...]`. */ +data class WindowFrame(val unit: FrameUnit, val start: FrameBound, val end: FrameBound, val exclude: FrameExclude = FrameExclude.NoOthers) + +/** + * `name(...) OVER (PARTITION BY ... ORDER BY ... [ROWS BETWEEN ... AND ...])`. [RowNumber]/[Rank]/ + * [DenseRank] never take an argument (always `argument == null && star == false`); [Aggregate] + * reuses the same COUNT/SUM/AVG/MIN/MAX vocabulary as [FunctionExpression] (including COUNT(*) via + * [star]) but computes per row over a partition/frame instead of collapsing the whole group into + * one row. [Lag]/[Lead] read [argument] from a row [offset] rows before/after the current one + * (default 1), falling back to [defaultValue] (or NULL) past the partition's edge - both are `null` + * for every other kind. [FirstValue]/[LastValue] read [argument] from the first/last row of the + * current frame; [Ntile] treats [argument] as the bucket count and ignores [offset]/[defaultValue]. + * [frame] is only meaningful for [Aggregate]/[FirstValue]/[LastValue] - `null` means the default SQL + * frame (`RANGE UNBOUNDED PRECEDING AND CURRENT ROW` with an ORDER BY, the whole partition without + * one); every other kind rejects an explicit [frame] outright since it would not change their result. + */ +data class WindowExpression( + val name: String, + val kind: WindowFunctionKind, + val argument: Expression?, + val star: Boolean, + val partitionBy: List, + val orderBy: List, + val offset: Expression? = null, + val defaultValue: Expression? = null, + val frame: WindowFrame? = null ) : Expression data class BinaryExpression( @@ -92,6 +583,25 @@ data class BinaryExpression( data class UnaryExpression(val operator: UnaryOperator, val operand: Expression) : Expression +/** + * `expr COLLATE name` (SQL:1999) - overrides the text-comparison sequence used for [operand] + * wherever it participates in a comparison (`=`/`<>`/`<`/`<=`/`>`/`>=`, `IN`, `BETWEEN`, a + * searched-`CASE`'s operand match) - see [kestrel.engine.Evaluator]'s own comparison helpers, the + * only place a [Collation] other than [Collation.Binary] changes behavior this way. Evaluating + * [operand] itself is unaffected ([kestrel.engine.Evaluator.evaluate] just recurses into it) - + * COLLATE only ever changes how two already-evaluated values are ordered against each other, never + * what value an expression itself produces. Always wins over a column's own declared COLLATE (a + * `col TYPE COLLATE name` on its [kestrel.catalog.Column], resolved automatically for a bare + * [ColumnExpression] with no [CollateExpression] of its own via + * [kestrel.engine.ColumnResolver.declaredCollation]) - explicit beats declared, exactly like + * SQLite. `ORDER BY`/window `OVER (... ORDER BY ...)` (see [kestrel.engine.compareOrderKeyValues]) + * only ever honor an explicit [CollateExpression] on the sort key, never a bare column's declared + * COLLATE - a documented, narrower scope cut than before: sorting a `COLLATE NOCASE` column still + * needs `ORDER BY col COLLATE NOCASE` spelled out, only comparisons pick the declared one up for + * free. + */ +data class CollateExpression(val operand: Expression, val collation: Collation) : Expression + enum class BinaryOperator(val symbol: String) { Add("+"), Subtract("-"), Multiply("*"), Divide("/"), Equal("="), NotEqual("<>"), Less("<"), LessEqual("<="), diff --git a/src/main/kotlin/kestrel/sql/lexer/Lexer.kt b/src/main/kotlin/kestrel/sql/lexer/Lexer.kt index 3ff5380..b1094aa 100644 Binary files a/src/main/kotlin/kestrel/sql/lexer/Lexer.kt and b/src/main/kotlin/kestrel/sql/lexer/Lexer.kt differ diff --git a/src/main/kotlin/kestrel/sql/lexer/Token.kt b/src/main/kotlin/kestrel/sql/lexer/Token.kt index 105ad56..da85448 100644 --- a/src/main/kotlin/kestrel/sql/lexer/Token.kt +++ b/src/main/kotlin/kestrel/sql/lexer/Token.kt @@ -3,14 +3,28 @@ package kestrel.sql.lexer enum class TokenType { Identifier, IntLiteral, RealLiteral, TextLiteral, Create, Table, Insert, Into, Values, Select, From, Where, - Order, By, Asc, Desc, Limit, Delete, Update, Set, Drop, + Order, By, Asc, Desc, Limit, Offset, Delete, Update, Set, Drop, Truncate, Identity, Continue, + Alter, Add, Column, Rename, To, Default, And, Or, Not, Null, True, False, Primary, Key, As, - Index, On, Inner, Left, Outer, Join, Group, Having, - Begin, Commit, Rollback, Transaction, Explain, - Integer, Text, Boolean, Real, + Index, On, Inner, Left, Right, Full, Cross, Outer, Join, Lateral, Group, Having, Over, Partition, + Rows, Row, Range, Groups, Preceding, Following, Unbounded, Current, Exclude, Ties, Others, + Begin, Commit, Rollback, Transaction, Explain, Savepoint, Release, + Conflict, Nothing, Do, Returning, + Trigger, Before, After, For, Each, + Is, In, Between, Like, Case, When, Then, Else, End, Exists, View, Replace, + Union, All, Intersect, Except, With, Recursive, Distinct, + Cast, Cascade, Restrict, References, Foreign, Check, No, Action, Unique, Constraint, + Sequence, Increment, Minvalue, Maxvalue, Start, Cycle, Restart, + Merge, Using, Matched, + Prepare, Execute, Deallocate, + Temp, Temporary, + Collate, + Integer, Text, Boolean, Real, Decimal, Date, Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual, Plus, Minus, Star, Slash, Dot, LeftParen, RightParen, Comma, Semicolon, + /** `$1`, `$2`, ... - see [kestrel.sql.ast.ParameterExpression]. [Token.intValue] carries the 1-based index. */ + Parameter, EndOfInput } @@ -38,10 +52,20 @@ internal val keywords: Map = mapOf( "ASC" to TokenType.Asc, "DESC" to TokenType.Desc, "LIMIT" to TokenType.Limit, + "OFFSET" to TokenType.Offset, "DELETE" to TokenType.Delete, "UPDATE" to TokenType.Update, "SET" to TokenType.Set, "DROP" to TokenType.Drop, + "TRUNCATE" to TokenType.Truncate, + "IDENTITY" to TokenType.Identity, + "CONTINUE" to TokenType.Continue, + "ALTER" to TokenType.Alter, + "ADD" to TokenType.Add, + "COLUMN" to TokenType.Column, + "RENAME" to TokenType.Rename, + "TO" to TokenType.To, + "DEFAULT" to TokenType.Default, "AND" to TokenType.And, "OR" to TokenType.Or, "NOT" to TokenType.Not, @@ -55,17 +79,92 @@ internal val keywords: Map = mapOf( "ON" to TokenType.On, "INNER" to TokenType.Inner, "LEFT" to TokenType.Left, + "RIGHT" to TokenType.Right, + "FULL" to TokenType.Full, + "CROSS" to TokenType.Cross, "OUTER" to TokenType.Outer, "JOIN" to TokenType.Join, + "LATERAL" to TokenType.Lateral, "GROUP" to TokenType.Group, "HAVING" to TokenType.Having, + "OVER" to TokenType.Over, + "PARTITION" to TokenType.Partition, + "ROWS" to TokenType.Rows, + "ROW" to TokenType.Row, + "RANGE" to TokenType.Range, + "GROUPS" to TokenType.Groups, + "PRECEDING" to TokenType.Preceding, + "FOLLOWING" to TokenType.Following, + "UNBOUNDED" to TokenType.Unbounded, + "CURRENT" to TokenType.Current, + "EXCLUDE" to TokenType.Exclude, + "TIES" to TokenType.Ties, + "OTHERS" to TokenType.Others, "BEGIN" to TokenType.Begin, "COMMIT" to TokenType.Commit, "ROLLBACK" to TokenType.Rollback, "TRANSACTION" to TokenType.Transaction, "EXPLAIN" to TokenType.Explain, + "SAVEPOINT" to TokenType.Savepoint, + "RELEASE" to TokenType.Release, + "CONFLICT" to TokenType.Conflict, + "NOTHING" to TokenType.Nothing, + "DO" to TokenType.Do, + "RETURNING" to TokenType.Returning, + "TRIGGER" to TokenType.Trigger, + "BEFORE" to TokenType.Before, + "AFTER" to TokenType.After, + "FOR" to TokenType.For, + "EACH" to TokenType.Each, "INTEGER" to TokenType.Integer, "TEXT" to TokenType.Text, "BOOLEAN" to TokenType.Boolean, - "REAL" to TokenType.Real + "REAL" to TokenType.Real, + "IS" to TokenType.Is, + "IN" to TokenType.In, + "BETWEEN" to TokenType.Between, + "LIKE" to TokenType.Like, + "CASE" to TokenType.Case, + "WHEN" to TokenType.When, + "THEN" to TokenType.Then, + "ELSE" to TokenType.Else, + "END" to TokenType.End, + "EXISTS" to TokenType.Exists, + "VIEW" to TokenType.View, + "REPLACE" to TokenType.Replace, + "UNION" to TokenType.Union, + "ALL" to TokenType.All, + "INTERSECT" to TokenType.Intersect, + "EXCEPT" to TokenType.Except, + "WITH" to TokenType.With, + "RECURSIVE" to TokenType.Recursive, + "DISTINCT" to TokenType.Distinct, + "CAST" to TokenType.Cast, + "CASCADE" to TokenType.Cascade, + "RESTRICT" to TokenType.Restrict, + "DECIMAL" to TokenType.Decimal, + "DATE" to TokenType.Date, + "REFERENCES" to TokenType.References, + "FOREIGN" to TokenType.Foreign, + "CHECK" to TokenType.Check, + "NO" to TokenType.No, + "ACTION" to TokenType.Action, + "UNIQUE" to TokenType.Unique, + "CONSTRAINT" to TokenType.Constraint, + "SEQUENCE" to TokenType.Sequence, + "INCREMENT" to TokenType.Increment, + "MINVALUE" to TokenType.Minvalue, + "MAXVALUE" to TokenType.Maxvalue, + "START" to TokenType.Start, + "CYCLE" to TokenType.Cycle, + "RESTART" to TokenType.Restart, + "MERGE" to TokenType.Merge, + "USING" to TokenType.Using, + "MATCHED" to TokenType.Matched, + "PREPARE" to TokenType.Prepare, + "EXECUTE" to TokenType.Execute, + "DEALLOCATE" to TokenType.Deallocate, + "TEMP" to TokenType.Temp, + "TEMPORARY" to TokenType.Temporary, + "COLLATE" to TokenType.Collate ) diff --git a/src/main/kotlin/kestrel/sql/parser/Parser.kt b/src/main/kotlin/kestrel/sql/parser/Parser.kt index aa132a3..f20a2cf 100644 --- a/src/main/kotlin/kestrel/sql/parser/Parser.kt +++ b/src/main/kotlin/kestrel/sql/parser/Parser.kt @@ -1,38 +1,119 @@ package kestrel.sql.parser import kestrel.ParseException +import kestrel.catalog.CheckConstraint import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.UniqueConstraint +import kestrel.sql.ast.AddColumn +import kestrel.sql.ast.AddConstraint +import kestrel.sql.ast.AddedCheck +import kestrel.sql.ast.AddedConstraint +import kestrel.sql.ast.AddedForeignKey +import kestrel.sql.ast.AddedUnique +import kestrel.sql.ast.AlterSequenceStatement +import kestrel.sql.ast.AlterTableAction +import kestrel.sql.ast.AlterTableStatement import kestrel.sql.ast.Assignment import kestrel.sql.ast.BeginStatement +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseBranch +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.ChangeColumnType +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression import kestrel.sql.ast.CommitStatement +import kestrel.sql.ast.CommonTableExpression +import kestrel.sql.ast.ConflictAction +import kestrel.sql.ast.ConflictDoNothing +import kestrel.sql.ast.ConflictDoUpdate import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateSequenceStatement import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.DeallocateStatement import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropColumn +import kestrel.sql.ast.DropConstraint import kestrel.sql.ast.DropIndexStatement +import kestrel.sql.ast.DropMode +import kestrel.sql.ast.DropSequenceStatement import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.DropTriggerStatement +import kestrel.sql.ast.DropViewStatement +import kestrel.sql.ast.ExecuteStatement import kestrel.sql.ast.Expression import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.ExistsExpression import kestrel.sql.ast.ExplainStatement +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FrameBoundType +import kestrel.sql.ast.FrameExclude +import kestrel.sql.ast.FrameUnit +import kestrel.sql.ast.FromItem import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.GroupingSpec +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression import kestrel.sql.ast.JoinClause import kestrel.sql.ast.JoinType +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.MergeWhenClause +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.OnConflictClause import kestrel.sql.ast.OrderBy +import kestrel.sql.ast.OrderKey +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.PrepareStatement +import kestrel.sql.ast.PreparableStatement import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.ReleaseSavepointStatement +import kestrel.sql.ast.RenameColumn +import kestrel.sql.ast.RenameTable import kestrel.sql.ast.RollbackStatement +import kestrel.sql.ast.RollbackToSavepointStatement +import kestrel.sql.ast.RollupSpec +import kestrel.sql.ast.SavepointStatement +import kestrel.sql.ast.ScalarSubqueryExpression import kestrel.sql.ast.SelectItem import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.SetOperator import kestrel.sql.ast.StarProjection import kestrel.sql.ast.Statement +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.TruncateTableStatement +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UnaryOperator import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WindowFrame +import kestrel.sql.ast.WindowFunctionKind +import kestrel.sql.ast.WithQuery import kestrel.sql.lexer.Token import kestrel.sql.lexer.TokenType +import kestrel.storage.Collation import kestrel.storage.ColumnType import kestrel.storage.Value @@ -56,41 +137,182 @@ class Parser(private val tokens: List) { return when (token.type) { TokenType.Create -> parseCreate() TokenType.Insert -> parseInsert() - TokenType.Select -> parseSelect() + TokenType.Select -> parseQueryStatement() + TokenType.With -> parseQueryStatement() TokenType.Delete -> parseDelete() TokenType.Update -> parseUpdate() TokenType.Drop -> parseDrop() + TokenType.Truncate -> parseTruncate() + TokenType.Merge -> parseMerge() + TokenType.Alter -> parseAlter() TokenType.Explain -> parseExplain() TokenType.Begin -> parseBegin() TokenType.Commit -> parseCommit() TokenType.Rollback -> parseRollback() + TokenType.Savepoint -> parseSavepoint() + TokenType.Release -> parseReleaseSavepoint() + TokenType.Prepare -> parsePrepare() + TokenType.Execute -> parseExecute() + TokenType.Deallocate -> parseDeallocate() else -> throw error("expected a statement but found '${describe(token)}'", token) } } private fun parseCreate(): Statement { expect(TokenType.Create, "expected CREATE") + var orReplace = false + if (match(TokenType.Or)) { + expect(TokenType.Replace, "expected REPLACE after OR") + orReplace = true + } + if (orReplace && !check(TokenType.View)) { + throw error("OR REPLACE is only supported for VIEW but found '${describe(peek())}'", peek()) + } + val temporary = match(TokenType.Temp) || match(TokenType.Temporary) + if (temporary && !check(TokenType.Table)) { + throw error("TEMP/TEMPORARY is only supported for TABLE but found '${describe(peek())}'", peek()) + } return when { - check(TokenType.Table) -> parseCreateTable() - check(TokenType.Index) -> parseCreateIndex() - else -> throw error("expected TABLE or INDEX after CREATE but found '${describe(peek())}'", peek()) + check(TokenType.Table) -> parseCreateTable(temporary) + check(TokenType.Index) -> parseCreateIndex(unique = false) + check(TokenType.Unique) -> parseCreateUniqueIndex() + check(TokenType.View) -> parseCreateView(orReplace) + check(TokenType.Trigger) -> parseCreateTrigger() + check(TokenType.Sequence) -> parseCreateSequence() + else -> throw error( + "expected TABLE, INDEX, UNIQUE INDEX, VIEW, TRIGGER, or SEQUENCE after CREATE but found '${describe(peek())}'", + peek() + ) } } - private fun parseCreateIndex(): CreateIndexStatement { + /** + * Every numeric option (`INCREMENT [BY]`/`MINVALUE`/`MAXVALUE`/`START [WITH]`) may appear at + * most once and in any order, matching Postgres; a repeated option silently keeps the last one + * written rather than erroring, the same "last one wins" leniency [parseAlterTable] already + * gives `ADD`/`DROP`/`RENAME` if written more than once in spirit (though that one can only ever + * carry a single action). `NO MINVALUE`/`NO MAXVALUE` here are never distinguished from "not + * written at all" - see [CreateSequenceStatement]'s own doc for why that's fine for CREATE + * specifically (unlike [AlterSequenceStatement], there is no earlier bound to preserve). + */ + private fun parseCreateSequence(): CreateSequenceStatement { + expect(TokenType.Sequence, "expected SEQUENCE") + val name = expectIdentifier("expected sequence name") + var incrementBy: Long? = null + var minValue: Long? = null + var maxValue: Long? = null + var startValue: Long? = null + var cycle = false + while (true) { + when { + match(TokenType.Increment) -> { + match(TokenType.By) + incrementBy = expectLongLiteral("expected a value after INCREMENT BY") + } + match(TokenType.Minvalue) -> minValue = expectLongLiteral("expected a value after MINVALUE") + match(TokenType.Maxvalue) -> maxValue = expectLongLiteral("expected a value after MAXVALUE") + match(TokenType.Start) -> { + match(TokenType.With) + startValue = expectLongLiteral("expected a value after START WITH") + } + match(TokenType.Cycle) -> cycle = true + check(TokenType.No) && peekIsSequenceNoClause() -> { + advance() + when { + match(TokenType.Minvalue) -> minValue = null + match(TokenType.Maxvalue) -> maxValue = null + match(TokenType.Cycle) -> cycle = false + else -> error("unreachable: guarded by peekIsSequenceNoClause") + } + } + else -> break + } + } + return CreateSequenceStatement(name, incrementBy, minValue, maxValue, startValue, cycle) + } + + /** Distinguishes a sequence option's `NO MINVALUE|MAXVALUE|CYCLE` from `ON DELETE NO ACTION`'s unrelated `NO` - both start with the same token. */ + private fun peekIsSequenceNoClause(): Boolean = + peekAt(1)?.type in setOf(TokenType.Minvalue, TokenType.Maxvalue, TokenType.Cycle) + + private fun peekAt(offset: Int): Token? = tokens.getOrNull(position + offset) + + /** + * `body` is deliberately restricted to INSERT/UPDATE/DELETE ([parseTriggerBodyStatement]) - + * the only statement kinds with a side effect worth firing, matching SQLite's own trigger-body + * grammar. The closing `END` here is the trigger body's own terminator, distinct from the `;` + * `parseProgram` still expects after the whole CREATE TRIGGER statement. + */ + private fun parseCreateTrigger(): CreateTriggerStatement { + expect(TokenType.Trigger, "expected TRIGGER") + val triggerName = expectIdentifier("expected trigger name") + val timing = when { + match(TokenType.Before) -> TriggerTiming.Before + match(TokenType.After) -> TriggerTiming.After + else -> throw error("expected BEFORE or AFTER after trigger name but found '${describe(peek())}'", peek()) + } + val event = when { + match(TokenType.Insert) -> TriggerEvent.Insert + match(TokenType.Update) -> TriggerEvent.Update + match(TokenType.Delete) -> TriggerEvent.Delete + else -> throw error("expected INSERT, UPDATE, or DELETE after ${timing.name.uppercase()} but found '${describe(peek())}'", peek()) + } + expect(TokenType.On, "expected ON after trigger event") + val tableName = expectIdentifier("expected table name") + expect(TokenType.For, "expected FOR after table name") + expect(TokenType.Each, "expected EACH after FOR") + expect(TokenType.Row, "expected ROW after EACH") + val whenClause = if (match(TokenType.When)) parseExpression() else null + expect(TokenType.Begin, "expected BEGIN to start trigger body") + val body = mutableListOf() + while (!check(TokenType.End)) { + body.add(parseTriggerBodyStatement()) + expect(TokenType.Semicolon, "expected ';' after trigger body statement") + } + expect(TokenType.End, "expected END to close trigger body") + return CreateTriggerStatement(triggerName, timing, event, tableName, whenClause, body) + } + + private fun parseTriggerBodyStatement(): Statement = when (peek().type) { + TokenType.Insert -> parseInsert() + TokenType.Update -> parseUpdate() + TokenType.Delete -> parseDelete() + else -> throw error( + "trigger body only supports INSERT, UPDATE, or DELETE statements but found '${describe(peek())}'", + peek() + ) + } + + private fun parseCreateView(orReplace: Boolean): CreateViewStatement { + expect(TokenType.View, "expected VIEW") + val viewName = expectIdentifier("expected view name") + expect(TokenType.As, "expected AS after view name") + val query = parseQueryStatement() + return CreateViewStatement(viewName, query, orReplace) + } + + private fun parseCreateUniqueIndex(): CreateIndexStatement { + expect(TokenType.Unique, "expected UNIQUE") + return parseCreateIndex(unique = true) + } + + private fun parseCreateIndex(unique: Boolean): CreateIndexStatement { expect(TokenType.Index, "expected INDEX") val indexName = expectIdentifier("expected index name") expect(TokenType.On, "expected ON after index name") val tableName = expectIdentifier("expected table name") expect(TokenType.LeftParen, "expected '(' before indexed column") - val columnName = expectIdentifier("expected column name") + val columnNames = mutableListOf(expectIdentifier("expected column name")) + while (match(TokenType.Comma)) { + columnNames.add(expectIdentifier("expected column name")) + } expect(TokenType.RightParen, "expected ')' after indexed column") - return CreateIndexStatement(indexName, tableName, columnName) + return CreateIndexStatement(indexName, tableName, columnNames, unique) } private fun parseExplain(): ExplainStatement { expect(TokenType.Explain, "expected EXPLAIN") - return ExplainStatement(parseSelect()) + return ExplainStatement(parseQueryStatement()) } private fun parseBegin(): Statement { @@ -107,34 +329,149 @@ class Parser(private val tokens: List) { private fun parseRollback(): Statement { expect(TokenType.Rollback, "expected ROLLBACK") + if (match(TokenType.To)) { + match(TokenType.Savepoint) + val name = expectIdentifier("expected savepoint name after ROLLBACK TO") + return RollbackToSavepointStatement(name) + } match(TokenType.Transaction) return RollbackStatement } - private fun parseCreateTable(): CreateTableStatement { + private fun parseSavepoint(): Statement { + expect(TokenType.Savepoint, "expected SAVEPOINT") + val name = expectIdentifier("expected savepoint name after SAVEPOINT") + return SavepointStatement(name) + } + + private fun parseReleaseSavepoint(): Statement { + expect(TokenType.Release, "expected RELEASE") + match(TokenType.Savepoint) + val name = expectIdentifier("expected savepoint name after RELEASE") + return ReleaseSavepointStatement(name) + } + + private fun parsePrepare(): Statement { + expect(TokenType.Prepare, "expected PREPARE") + val name = expectIdentifier("expected prepared statement name") + expect(TokenType.As, "expected AS after prepared statement name") + return PrepareStatement(name, parsePreparableStatement()) + } + + private fun parsePreparableStatement(): PreparableStatement = when (peek().type) { + TokenType.Select, TokenType.With -> parseQueryStatement() + TokenType.Insert -> parseInsert() + TokenType.Update -> parseUpdate() + TokenType.Delete -> parseDelete() + TokenType.Merge -> parseMerge() + else -> throw error( + "PREPARE only supports SELECT, INSERT, UPDATE, DELETE, or MERGE but found '${describe(peek())}'", + peek() + ) + } + + private fun parseExecute(): Statement { + expect(TokenType.Execute, "expected EXECUTE") + val name = expectIdentifier("expected prepared statement name") + val arguments = mutableListOf() + if (match(TokenType.LeftParen)) { + if (!check(TokenType.RightParen)) { + arguments.add(parseExpression()) + while (match(TokenType.Comma)) arguments.add(parseExpression()) + } + expect(TokenType.RightParen, "expected ')' after EXECUTE arguments") + } + return ExecuteStatement(name, arguments) + } + + private fun parseDeallocate(): Statement { + expect(TokenType.Deallocate, "expected DEALLOCATE") + match(TokenType.Prepare) + if (match(TokenType.All)) return DeallocateStatement(null, all = true) + val name = expectIdentifier("expected prepared statement name or ALL after DEALLOCATE") + return DeallocateStatement(name) + } + + private fun parseCreateTable(temporary: Boolean = false): CreateTableStatement { expect(TokenType.Table, "expected TABLE after CREATE") val tableName = expectIdentifier("expected table name") expect(TokenType.LeftParen, "expected '(' after table name") val columns = mutableListOf() + val foreignKeys = mutableListOf() + val checks = mutableListOf() + val uniqueConstraints = mutableListOf() + var compositePrimaryKey: List = emptyList() do { - columns.add(parseColumnDefinition()) + when { + check(TokenType.Constraint) -> parseNamedTableConstraint(foreignKeys, checks, uniqueConstraints) + check(TokenType.Foreign) -> foreignKeys.add(parseTableForeignKey()) + check(TokenType.Check) -> checks.add(parseTableCheck()) + check(TokenType.Unique) -> uniqueConstraints.add(parseTableUnique()) + check(TokenType.Primary) -> { + if (compositePrimaryKey.isNotEmpty()) { + throw error("table declares more than one PRIMARY KEY clause", peek()) + } + compositePrimaryKey = parseTablePrimaryKey() + } + else -> columns.add(parseColumnDefinition(foreignKeys, checks, uniqueConstraints)) + } } while (match(TokenType.Comma)) expect(TokenType.RightParen, "expected ')' after column definitions") - return CreateTableStatement(tableName, columns) + return CreateTableStatement(tableName, columns, foreignKeys, checks, uniqueConstraints, compositePrimaryKey, temporary) } - private fun parseColumnDefinition(): Column { - val columnName = expectIdentifier("expected column name") - val typeToken = advance() - val type = when (typeToken.type) { - TokenType.Integer -> ColumnType.Integer - TokenType.Text -> ColumnType.Text - TokenType.Boolean -> ColumnType.Boolean - TokenType.Real -> ColumnType.Real - else -> throw error("expected a column type but found '${describe(typeToken)}'", typeToken) + private fun parseTablePrimaryKey(): List { + expect(TokenType.Primary, "expected PRIMARY") + expect(TokenType.Key, "expected KEY after PRIMARY") + expect(TokenType.LeftParen, "expected '(' after PRIMARY KEY") + val columns = mutableListOf() + do { + columns.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after PRIMARY KEY column list") + return columns + } + + /** + * `CONSTRAINT name FOREIGN KEY (...) | CHECK (...) | UNIQUE (...)` - a named table-level + * constraint, standard SQL syntax this engine accepts both inline in `CREATE TABLE` (here) and + * via `ALTER TABLE ... ADD` (see [parseAlterTable]'s identical dispatch). `PRIMARY KEY` is + * deliberately not accepted after `CONSTRAINT name`: [kestrel.catalog.TableSchema] has no name + * field for a table's PRIMARY KEY at all, so there would be nowhere to put it, and + * `ALTER TABLE ... DROP CONSTRAINT` never needs to target a PRIMARY KEY (see + * [kestrel.catalog.Catalog.dropColumn]'s own doc on why dropping a PRIMARY KEY is out of scope + * entirely - redesign it or drop the table instead). + */ + private fun parseNamedTableConstraint( + foreignKeys: MutableList, + checks: MutableList, + uniqueConstraints: MutableList + ) { + advance() + val name = expectIdentifier("expected constraint name after CONSTRAINT") + when { + check(TokenType.Foreign) -> foreignKeys.add(parseTableForeignKey().copy(name = name)) + check(TokenType.Check) -> checks.add(parseTableCheck().copy(name = name)) + check(TokenType.Unique) -> uniqueConstraints.add(parseTableUnique().copy(name = name)) + else -> throw error( + "expected FOREIGN KEY, CHECK, or UNIQUE after CONSTRAINT $name but found '${describe(peek())}'", + peek() + ) } + } + + private fun parseColumnDefinition( + foreignKeys: MutableList, + checks: MutableList, + uniqueConstraints: MutableList + ): Column { + val columnName = expectIdentifier("expected column name") + val spec = parseTypeSpec() var notNull = false var primaryKey = false + var default: Expression? = null + var generated: Expression? = null + var collation = Collation.Binary while (true) { when { check(TokenType.Not) -> { @@ -148,10 +485,168 @@ class Parser(private val tokens: List) { primaryKey = true notNull = true } + check(TokenType.Unique) -> { + advance() + uniqueConstraints.add(UniqueConstraint(null, listOf(columnName))) + } + check(TokenType.References) -> foreignKeys.add(parseInlineForeignKey(columnName)) + check(TokenType.Check) -> checks.add(parseColumnCheck()) + check(TokenType.Default) -> { + advance() + default = parseExpression() + } + isContextualKeyword("GENERATED") -> generated = parseGeneratedClause() + check(TokenType.Collate) -> collation = parseColumnCollation() else -> break } } - return Column(columnName, type, notNull, primaryKey) + if (generated != null && default != null) { + throw error("a column cannot declare both DEFAULT and GENERATED ALWAYS AS", peek()) + } + if (generated != null && primaryKey) { + throw error("a GENERATED column cannot be a PRIMARY KEY", peek()) + } + return Column(columnName, spec.type, notNull, primaryKey, spec.precision, spec.scale, default, generated, collation) + } + + /** `COLLATE name` on a column definition (`CREATE TABLE`/`ALTER TABLE ADD COLUMN`) - reuses the same name validation as expression-level `expr COLLATE name` (see [parseCollate]). */ + private fun parseColumnCollation(): Collation { + advance() + val nameToken = expect(TokenType.Identifier, "expected a collation name after COLLATE") + return Collation.fromName(nameToken.lexeme) ?: throw error("no such collating sequence: ${nameToken.lexeme}", nameToken) + } + + /** + * `GENERATED ALWAYS AS ( expression ) STORED` - Postgres only ever supports `STORED` (unlike + * MySQL/SQLite, which also allow `VIRTUAL`), so this engine matches Postgres exactly rather than + * accepting a keyword it would then have to silently reinterpret. `GENERATED`/`ALWAYS`/`STORED` + * are contextual (same mechanism as `ROLLUP`/`CUBE`/`FILTER`), not reserved keywords, so a table + * can still name a column `generated` without quoting. + */ + private fun parseGeneratedClause(): Expression { + advance() + if (!isContextualKeyword("ALWAYS")) throw error("expected ALWAYS after GENERATED", peek()) + advance() + expect(TokenType.As, "expected AS after GENERATED ALWAYS") + expect(TokenType.LeftParen, "expected '(' after GENERATED ALWAYS AS") + val expression = parseExpression() + expect(TokenType.RightParen, "expected ')' to close GENERATED ALWAYS AS expression") + if (!isContextualKeyword("STORED")) throw error("expected STORED after GENERATED ALWAYS AS (...)", peek()) + advance() + return expression + } + + private fun parseTableUnique(): UniqueConstraint { + expect(TokenType.Unique, "expected UNIQUE") + expect(TokenType.LeftParen, "expected '(' after UNIQUE") + val columns = mutableListOf() + do { + columns.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after UNIQUE column list") + return UniqueConstraint(null, columns) + } + + private fun parseTableForeignKey(): ForeignKeyConstraint { + expect(TokenType.Foreign, "expected FOREIGN") + expect(TokenType.Key, "expected KEY after FOREIGN") + expect(TokenType.LeftParen, "expected '(' after FOREIGN KEY") + val columnNames = mutableListOf() + do { + columnNames.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after foreign key column list") + return parseInlineForeignKey(columnNames) + } + + private fun parseInlineForeignKey(columnName: String): ForeignKeyConstraint = parseInlineForeignKey(listOf(columnName)) + + private fun parseInlineForeignKey(columnNames: List): ForeignKeyConstraint { + expect(TokenType.References, "expected REFERENCES") + val refTable = expectIdentifier("expected referenced table name") + expect(TokenType.LeftParen, "expected '(' before referenced column list") + val refColumns = mutableListOf() + do { + refColumns.add(expectIdentifier("expected referenced column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after referenced column list") + if (refColumns.size != columnNames.size) { + throw error( + "FOREIGN KEY (${columnNames.joinToString(", ")}) has ${columnNames.size} column(s) but " + + "REFERENCES $refTable(${refColumns.joinToString(", ")}) has ${refColumns.size}", + peek() + ) + } + var onDelete = ForeignKeyAction.NoAction + var onUpdate = ForeignKeyAction.NoAction + while (check(TokenType.On)) { + advance() + when { + match(TokenType.Delete) -> onDelete = parseForeignKeyAction() + match(TokenType.Update) -> onUpdate = parseForeignKeyAction() + else -> throw error("expected DELETE or UPDATE after ON", peek()) + } + } + return ForeignKeyConstraint(columnNames, refTable, refColumns, onDelete, onUpdate) + } + + private fun parseForeignKeyAction(): ForeignKeyAction = when { + match(TokenType.Cascade) -> ForeignKeyAction.Cascade + match(TokenType.Restrict) -> ForeignKeyAction.Restrict + match(TokenType.Set) -> { + expect(TokenType.Null, "expected NULL after SET") + ForeignKeyAction.SetNull + } + match(TokenType.No) -> { + expect(TokenType.Action, "expected ACTION after NO") + ForeignKeyAction.NoAction + } + else -> throw error("expected CASCADE, RESTRICT, SET NULL, or NO ACTION but found '${describe(peek())}'", peek()) + } + + private fun parseTableCheck(): CheckConstraint = parseCheckConstraint() + + private fun parseColumnCheck(): CheckConstraint = parseCheckConstraint() + + private fun parseCheckConstraint(): CheckConstraint { + expect(TokenType.Check, "expected CHECK") + expect(TokenType.LeftParen, "expected '(' after CHECK") + val expression = parseExpression() + expect(TokenType.RightParen, "expected ')' after CHECK expression") + return CheckConstraint(null, expression) + } + + private data class TypeSpec(val type: ColumnType, val precision: Int?, val scale: Int?) + + /** Shared by column definitions (`CREATE TABLE`) and `CAST(... AS type)`. */ + private fun parseTypeSpec(): TypeSpec { + val typeToken = advance() + val type = when (typeToken.type) { + TokenType.Integer -> ColumnType.Integer + TokenType.Text -> ColumnType.Text + TokenType.Boolean -> ColumnType.Boolean + TokenType.Real -> ColumnType.Real + TokenType.Decimal -> ColumnType.Decimal + TokenType.Date -> ColumnType.Date + else -> throw error("expected a type but found '${describe(typeToken)}'", typeToken) + } + var precision: Int? = null + var scale: Int? = null + if (type == ColumnType.Decimal && match(TokenType.LeftParen)) { + precision = expectIntLiteral("expected precision in DECIMAL(...)") + scale = if (match(TokenType.Comma)) expectIntLiteral("expected scale in DECIMAL(precision, scale)") else 0 + expect(TokenType.RightParen, "expected ')' after DECIMAL precision/scale") + } + return TypeSpec(type, precision, scale) + } + + private fun expectIntLiteral(message: String): Int = expect(TokenType.IntLiteral, message).intValue!!.toInt() + + /** A sequence option's bound (MINVALUE/MAXVALUE/START WITH/RESTART WITH) needs the full 64-bit range, unlike [expectIntLiteral]'s callers - and, unlike those, may be negative for a descending sequence. */ + private fun expectLongLiteral(message: String): Long { + val negative = match(TokenType.Minus) + val value = expect(TokenType.IntLiteral, message).intValue!! + return if (negative) -value else value } private fun parseInsert(): InsertStatement { @@ -168,31 +663,136 @@ class Parser(private val tokens: List) { columns = names } expect(TokenType.Values, "expected VALUES") - val rows = mutableListOf>() + val rows = mutableListOf>() do { expect(TokenType.LeftParen, "expected '(' before value list") - val values = mutableListOf() + val values = mutableListOf() do { - values.add(parseExpression()) + values.add(if (match(TokenType.Default)) null else parseExpression()) } while (match(TokenType.Comma)) expect(TokenType.RightParen, "expected ')' after value list") rows.add(values) } while (match(TokenType.Comma)) - return InsertStatement(tableName, columns, rows) + val onConflict = if (check(TokenType.On)) parseOnConflict() else null + val returning = parseReturning() + return InsertStatement(tableName, columns, rows, onConflict, returning) + } + + private fun parseReturning(): Projection? = if (match(TokenType.Returning)) parseProjection() else null + + private fun parseOnConflict(): OnConflictClause { + expect(TokenType.On, "expected ON") + expect(TokenType.Conflict, "expected CONFLICT after ON") + var targetColumns: List = emptyList() + if (match(TokenType.LeftParen)) { + val names = mutableListOf() + do { + names.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after ON CONFLICT column list") + targetColumns = names + } + expect(TokenType.Do, "expected DO after ON CONFLICT") + val action: ConflictAction = when { + match(TokenType.Nothing) -> ConflictDoNothing + match(TokenType.Update) -> { + expect(TokenType.Set, "expected SET after DO UPDATE") + val assignments = parseAssignmentList() + val where = if (match(TokenType.Where)) parseExpression() else null + ConflictDoUpdate(assignments, where) + } + else -> throw error("expected NOTHING or UPDATE after DO but found '${describe(peek())}'", peek()) + } + return OnConflictClause(targetColumns, action) } private fun parseSelect(): SelectStatement { + val core = parseSelectCore() + val orderBy = parseOrderBy() + val (limit, offset) = parseLimitOffset() + return core.copy(orderBy = orderBy, limit = limit, offset = offset) + } + + private fun parseSelectCore(): SelectStatement { expect(TokenType.Select, "expected SELECT") + val (distinct, distinctOn) = parseDistinctClause() val projection = parseProjection() expect(TokenType.From, "expected FROM") - val (tableName, tableAlias) = parseTableRef() + val from = parseFromItem() val joins = parseJoins() val where = if (match(TokenType.Where)) parseExpression() else null val groupBy = parseGroupBy() val having = if (match(TokenType.Having)) parseExpression() else null + return SelectStatement( + projection, from, joins, where, groupBy.groupBy, groupBy.spec, having, + distinct = distinct, distinctOn = distinctOn + ) + } + + /** `SELECT [ALL | DISTINCT | DISTINCT ON (expr, ...)] ...` - `ALL` (the default) is accepted and discarded, matching Postgres. */ + private fun parseDistinctClause(): Pair> { + if (match(TokenType.All)) return false to emptyList() + if (!match(TokenType.Distinct)) return false to emptyList() + if (!match(TokenType.On)) return true to emptyList() + expect(TokenType.LeftParen, "expected '(' after DISTINCT ON") + val expressions = mutableListOf() + do { + expressions.add(parseExpression()) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after DISTINCT ON expressions") + return true to expressions + } + + private fun parseQueryStatement(): QueryStatement { + if (check(TokenType.With)) return parseWithQuery() + return parseCompoundQuery() + } + + private fun parseWithQuery(): WithQuery { + expect(TokenType.With, "expected WITH") + val recursive = match(TokenType.Recursive) + val ctes = mutableListOf() + do { + val name = expectIdentifier("expected common table expression name") + var columns: List? = null + if (match(TokenType.LeftParen)) { + val names = mutableListOf() + do { + names.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after column list") + columns = names + } + expect(TokenType.As, "expected AS after common table expression name") + expect(TokenType.LeftParen, "expected '(' after AS") + val query = parseCompoundQuery() + expect(TokenType.RightParen, "expected ')' after common table expression") + ctes.add(CommonTableExpression(name, columns, query)) + } while (match(TokenType.Comma)) + val body = parseCompoundQuery() + return WithQuery(recursive, ctes, body) + } + + private fun parseCompoundQuery(): QueryStatement { + var left: QueryStatement = parseSelectCore() + while (true) { + val operator = when { + match(TokenType.Union) -> if (match(TokenType.All)) SetOperator.UnionAll else SetOperator.Union + match(TokenType.Intersect) -> if (match(TokenType.All)) SetOperator.IntersectAll else SetOperator.Intersect + match(TokenType.Except) -> if (match(TokenType.All)) SetOperator.ExceptAll else SetOperator.Except + else -> break + } + val right = parseSelectCore() + left = SetOperationStatement(operator, left, right) + } val orderBy = parseOrderBy() - val limit = parseLimit() - return SelectStatement(projection, tableName, tableAlias, joins, where, groupBy, having, orderBy, limit) + val (limit, offset) = parseLimitOffset() + if (orderBy == null && limit == null && offset == null) return left + return when (left) { + is SelectStatement -> left.copy(orderBy = orderBy, limit = limit, offset = offset) + is SetOperationStatement -> left.copy(orderBy = orderBy, limit = limit, offset = offset) + is WithQuery -> error("unreachable: parseSelectCore never produces a WITH query") + } } private fun parseProjection(): Projection { @@ -206,14 +806,27 @@ class Parser(private val tokens: List) { return ExpressionProjection(items) } - private fun parseTableRef(): Pair { + private fun parseFromItem(): FromItem { + if (match(TokenType.LeftParen)) { + if (!check(TokenType.Select) && !check(TokenType.With)) { + throw error("expected SELECT or WITH after '(' in FROM but found '${describe(peek())}'", peek()) + } + val select = parseQueryStatement() + expect(TokenType.RightParen, "expected ')' after derived table subquery") + val alias = when { + match(TokenType.As) -> expectIdentifier("expected alias after AS") + check(TokenType.Identifier) -> advance().lexeme + else -> throw error("a derived table in FROM requires an alias", peek()) + } + return SubquerySource(select, alias) + } val tableName = expectIdentifier("expected table name") val alias = when { match(TokenType.As) -> expectIdentifier("expected alias after AS") check(TokenType.Identifier) -> advance().lexeme else -> null } - return tableName to alias + return TableSource(tableName, alias) } private fun parseJoins(): List { @@ -226,41 +839,173 @@ class Parser(private val tokens: List) { match(TokenType.Left) -> { match(TokenType.Outer); expect(TokenType.Join, "expected JOIN after LEFT"); JoinType.Left } + match(TokenType.Right) -> { + match(TokenType.Outer); expect(TokenType.Join, "expected JOIN after RIGHT"); JoinType.Right + } + match(TokenType.Full) -> { + match(TokenType.Outer); expect(TokenType.Join, "expected JOIN after FULL"); JoinType.Full + } + match(TokenType.Cross) -> { + expect(TokenType.Join, "expected JOIN after CROSS"); JoinType.Cross + } match(TokenType.Join) -> JoinType.Inner else -> break } - val (tableName, alias) = parseTableRef() - expect(TokenType.On, "expected ON after joined table") - joins.add(JoinClause(type, tableName, alias, parseExpression())) + val lateral = match(TokenType.Lateral) + // A LATERAL right-hand side has no independent row set (it is re-evaluated per left + // row), so there is nothing to pad with NULLs for an unmatched *right* row - RIGHT/FULL + // JOIN LATERAL is therefore meaningless and rejected here rather than at execution time. + if (lateral && (type == JoinType.Right || type == JoinType.Full)) { + throw error("RIGHT/FULL JOIN LATERAL is not supported: LATERAL has no independent right-hand row set", peek()) + } + val from = parseFromItem() + if (lateral && from !is SubquerySource) { + throw error("LATERAL is only allowed before a derived table subquery", peek()) + } + val taggedFrom = if (lateral) (from as SubquerySource).copy(lateral = true) else from + // CROSS JOIN has no ON clause at all - matches every left/right pairing unconditionally. + val on = if (type == JoinType.Cross) { + if (check(TokenType.On)) throw error("CROSS JOIN does not take an ON condition", peek()) + null + } else { + expect(TokenType.On, "expected ON after joined table") + parseExpression() + } + joins.add(JoinClause(type, taggedFrom, on)) } return joins } - private fun parseGroupBy(): List { - if (!match(TokenType.Group)) return emptyList() + private data class GroupByClause(val groupBy: List, val spec: GroupingSpec?) + + /** + * `ROLLUP`/`CUBE`/`GROUPING SETS` are not reserved keywords (matching how most SQL engines treat + * them - only meaningful right after `GROUP BY`), so they are recognized here by peeking at an + * [TokenType.Identifier]'s text rather than via a dedicated token type; a bare column actually + * named e.g. `rollup` is unaffected since the lookahead also requires the following token to open + * the parenthesized argument list. Mixing plain columns with one of these in the same clause (SQL + * allows it, producing a cross product of sets) is a scope cut - exactly one form fills the whole + * clause. + */ + private fun parseGroupBy(): GroupByClause { + if (!match(TokenType.Group)) return GroupByClause(emptyList(), null) expect(TokenType.By, "expected BY after GROUP") + if (isContextualKeyword("ROLLUP") && peekNext().type == TokenType.LeftParen) { + advance() + val expressions = parseParenthesizedExpressionList("ROLLUP") + return GroupByClause(expressions, RollupSpec(expressions)) + } + if (isContextualKeyword("CUBE") && peekNext().type == TokenType.LeftParen) { + advance() + val expressions = parseParenthesizedExpressionList("CUBE") + return GroupByClause(expressions, CubeSpec(expressions)) + } + if (isContextualKeyword("GROUPING") && peekNext().let { it.type == TokenType.Identifier && it.lexeme.equals("SETS", ignoreCase = true) }) { + advance() + advance() + expect(TokenType.LeftParen, "expected '(' after GROUPING SETS") + val sets = mutableListOf>() + do { + sets.add(parseGroupingSetElement()) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' to close GROUPING SETS") + val union = sets.flatten().distinct() + return GroupByClause(union, GroupingSetsSpec(sets)) + } val expressions = mutableListOf() do { expressions.add(parseExpression()) } while (match(TokenType.Comma)) + return GroupByClause(expressions, null) + } + + private fun isContextualKeyword(name: String): Boolean = + check(TokenType.Identifier) && peek().lexeme.equals(name, ignoreCase = true) + + private fun peekNext(): Token = tokens.getOrElse(position + 1) { tokens.last() } + + private fun parseParenthesizedExpressionList(clauseName: String): List { + expect(TokenType.LeftParen, "expected '(' after $clauseName") + val expressions = mutableListOf() + if (!check(TokenType.RightParen)) { + do { + expressions.add(parseExpression()) + } while (match(TokenType.Comma)) + } + expect(TokenType.RightParen, "expected ')' to close $clauseName argument list") + return expressions + } + + /** One element of `GROUPING SETS (...)`: `()` (the grand-total set) or a parenthesized expression list. */ + private fun parseGroupingSetElement(): List { + expect(TokenType.LeftParen, "expected '(' to open a GROUPING SETS element") + if (match(TokenType.RightParen)) return emptyList() + val expressions = mutableListOf() + do { + expressions.add(parseExpression()) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' to close a GROUPING SETS element") return expressions } private fun parseOrderBy(): OrderBy? { - if (!match(TokenType.Order)) return null - expect(TokenType.By, "expected BY after ORDER") - val expression = parseExpression() - val descending = when { - match(TokenType.Desc) -> true - match(TokenType.Asc) -> false - else -> false - } - return OrderBy(expression, descending) + val keys = parseOrderKeys() + return if (keys.isEmpty()) null else OrderBy(keys) } - private fun parseLimit(): Int? { - if (!match(TokenType.Limit)) return null - return expect(TokenType.IntLiteral, "expected an integer after LIMIT").intValue!!.toInt() + /** Shared by top-level ORDER BY and a window function's `OVER (... ORDER BY ...)` clause. */ + private fun parseOrderKeys(): List { + if (!match(TokenType.Order)) return emptyList() + expect(TokenType.By, "expected BY after ORDER") + val keys = mutableListOf() + do { + val expression = parseExpression() + val descending = when { + match(TokenType.Desc) -> true + match(TokenType.Asc) -> false + else -> false + } + val nullsFirst = parseOptionalNullsClause() + keys.add(OrderKey(expression, descending, nullsFirst)) + } while (match(TokenType.Comma)) + return keys + } + + /** + * `NULLS FIRST` / `NULLS LAST`, optional after each ORDER BY key's own `ASC`/`DESC`. Like + * `ROLLUP`/`CUBE`/`FILTER`/`GENERATED` elsewhere in this parser, `NULLS`/`FIRST`/`LAST` are not + * reserved keywords - a column legitimately named any of the three is unaffected outside this + * exact position, since [isContextualKeyword] only fires here, right after an order key. + */ + private fun parseOptionalNullsClause(): Boolean? { + if (!isContextualKeyword("NULLS")) return null + advance() + return when { + isContextualKeyword("FIRST") -> { advance(); true } + isContextualKeyword("LAST") -> { advance(); false } + else -> throw error("expected FIRST or LAST after NULLS but found '${describe(peek())}'", peek()) + } + } + + /** + * `LIMIT`/`OFFSET` may appear in either order (matching Postgres, unlike the standard's + * `OFFSET ... FETCH ...`), but not repeated - a second `LIMIT` or `OFFSET` stops the loop and is + * left for whatever follows (typically a parse error from the caller, e.g. a stray token before + * `;`) rather than silently overwriting the first. + */ + private fun parseLimitOffset(): Pair { + var limit: Int? = null + var offset: Int? = null + while (true) { + if (limit == null && match(TokenType.Limit)) { + limit = expect(TokenType.IntLiteral, "expected an integer after LIMIT").intValue!!.toInt() + } else if (offset == null && match(TokenType.Offset)) { + offset = expect(TokenType.IntLiteral, "expected an integer after OFFSET").intValue!!.toInt() + } else { + break + } + } + return limit to offset } private fun parseDelete(): DeleteStatement { @@ -268,32 +1013,313 @@ class Parser(private val tokens: List) { expect(TokenType.From, "expected FROM after DELETE") val tableName = expectIdentifier("expected table name") val where = if (match(TokenType.Where)) parseExpression() else null - return DeleteStatement(tableName, where) + val returning = parseReturning() + return DeleteStatement(tableName, where, returning) } private fun parseUpdate(): UpdateStatement { expect(TokenType.Update, "expected UPDATE") val tableName = expectIdentifier("expected table name") expect(TokenType.Set, "expected SET after table name") + val assignments = parseAssignmentList() + val where = if (match(TokenType.Where)) parseExpression() else null + val returning = parseReturning() + return UpdateStatement(tableName, assignments, where, returning) + } + + private fun parseAssignmentList(): List { val assignments = mutableListOf() do { val columnName = expectIdentifier("expected column name") expect(TokenType.Equal, "expected '=' in assignment") - assignments.add(Assignment(columnName, parseExpression())) + val value = if (match(TokenType.Default)) null else parseExpression() + assignments.add(Assignment(columnName, value)) } while (match(TokenType.Comma)) - val where = if (match(TokenType.Where)) parseExpression() else null - return UpdateStatement(tableName, assignments, where) + return assignments + } + + /** + * `MERGE INTO target [[AS] alias] USING source ON condition (WHEN ...)+`. [parseFromItem] is + * reused verbatim for the `USING` source: it is exactly a `FROM`-shaped item (bare table name + * or a parenthesized, aliased derived-table subquery), not a MERGE-specific grammar production. + */ + private fun parseMerge(): MergeStatement { + expect(TokenType.Merge, "expected MERGE") + expect(TokenType.Into, "expected INTO after MERGE") + val targetTable = expectIdentifier("expected target table name") + val targetAlias = parseOptionalAlias() + expect(TokenType.Using, "expected USING after MERGE target") + val source = parseFromItem() + expect(TokenType.On, "expected ON after MERGE USING source") + val onCondition = parseExpression() + val whenClauses = mutableListOf() + while (match(TokenType.When)) { + whenClauses.add(parseMergeWhenClause()) + } + if (whenClauses.isEmpty()) { + throw error("MERGE requires at least one WHEN clause", peek()) + } + val returning = parseReturning() + return MergeStatement(targetTable, targetAlias, source, onCondition, whenClauses, returning) + } + + private fun parseOptionalAlias(): String? = when { + match(TokenType.As) -> expectIdentifier("expected alias after AS") + check(TokenType.Identifier) -> advance().lexeme + else -> null + } + + /** + * `WHEN [NOT] MATCHED [AND condition] THEN action` - the `AND condition` sits between the + * MATCHED/NOT MATCHED keyword and THEN in both categories, matching Postgres. A NOT MATCHED + * action is always INSERT (there is no target row for NOT MATCHED to UPDATE or DELETE); + * [MergeNotMatchedInsert.values] uses the same "bare DEFAULT keyword or expression" per-slot + * grammar as plain INSERT's own VALUES list ([parseInsert]). + */ + private fun parseMergeWhenClause(): MergeWhenClause { + val matched = when { + match(TokenType.Matched) -> true + match(TokenType.Not) -> { + expect(TokenType.Matched, "expected MATCHED after NOT") + false + } + else -> throw error("expected MATCHED or NOT MATCHED after WHEN but found '${describe(peek())}'", peek()) + } + val condition = if (match(TokenType.And)) parseExpression() else null + expect(TokenType.Then, "expected THEN after WHEN ${if (matched) "MATCHED" else "NOT MATCHED"}") + if (matched) { + return when { + match(TokenType.Update) -> { + expect(TokenType.Set, "expected SET after WHEN MATCHED THEN UPDATE") + MergeMatchedUpdate(condition, parseAssignmentList()) + } + match(TokenType.Delete) -> MergeMatchedDelete(condition) + else -> throw error("expected UPDATE or DELETE after WHEN MATCHED THEN but found '${describe(peek())}'", peek()) + } + } + expect(TokenType.Insert, "expected INSERT after WHEN NOT MATCHED THEN") + var columns: List? = null + if (match(TokenType.LeftParen)) { + val names = mutableListOf() + do { + names.add(expectIdentifier("expected column name")) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after column list") + columns = names + } + expect(TokenType.Values, "expected VALUES after WHEN NOT MATCHED THEN INSERT") + expect(TokenType.LeftParen, "expected '(' before value list") + val values = mutableListOf() + do { + values.add(if (match(TokenType.Default)) null else parseExpression()) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after value list") + return MergeNotMatchedInsert(condition, columns, values) } private fun parseDrop(): Statement { expect(TokenType.Drop, "expected DROP") return when { - match(TokenType.Table) -> DropTableStatement(expectIdentifier("expected table name")) + match(TokenType.Table) -> DropTableStatement(expectIdentifier("expected table name"), parseDropMode()) match(TokenType.Index) -> DropIndexStatement(expectIdentifier("expected index name")) - else -> throw error("expected TABLE or INDEX after DROP but found '${describe(peek())}'", peek()) + match(TokenType.View) -> DropViewStatement(expectIdentifier("expected view name"), parseDropMode()) + match(TokenType.Trigger) -> DropTriggerStatement(expectIdentifier("expected trigger name")) + match(TokenType.Sequence) -> DropSequenceStatement(expectIdentifier("expected sequence name")) + else -> throw error("expected TABLE, INDEX, VIEW, TRIGGER, or SEQUENCE after DROP but found '${describe(peek())}'", peek()) } } + /** + * `TRUNCATE [TABLE] t1 [, t2, ...] [RESTART IDENTITY | CONTINUE IDENTITY] [CASCADE | RESTRICT]`. + * `TABLE` and the identity clause are both optional, matching Postgres (which also defaults to + * CONTINUE IDENTITY when neither RESTART nor CONTINUE is written). + */ + private fun parseTruncate(): TruncateTableStatement { + expect(TokenType.Truncate, "expected TRUNCATE") + match(TokenType.Table) + val tableNames = mutableListOf(expectIdentifier("expected table name")) + while (match(TokenType.Comma)) { + tableNames.add(expectIdentifier("expected table name")) + } + val restartIdentity = when { + match(TokenType.Restart) -> { + expect(TokenType.Identity, "expected IDENTITY after RESTART") + true + } + match(TokenType.Continue) -> { + expect(TokenType.Identity, "expected IDENTITY after CONTINUE") + false + } + else -> false + } + return TruncateTableStatement(tableNames, restartIdentity, parseDropMode()) + } + + private fun parseDropMode(): DropMode = when { + match(TokenType.Cascade) -> DropMode.Cascade + match(TokenType.Restrict) -> DropMode.Restrict + else -> DropMode.Restrict + } + + private fun parseAlter(): Statement { + expect(TokenType.Alter, "expected ALTER") + return when { + check(TokenType.Table) -> parseAlterTable() + check(TokenType.Sequence) -> parseAlterSequence() + else -> throw error("expected TABLE or SEQUENCE after ALTER but found '${describe(peek())}'", peek()) + } + } + + /** + * Every option may appear at most once and in any order, same leniency as + * [parseCreateSequence]. `NO MINVALUE`/`NO MAXVALUE` here are NOT equivalent to leaving the + * option out entirely - see [AlterSequenceStatement]'s own doc. + */ + private fun parseAlterSequence(): AlterSequenceStatement { + expect(TokenType.Sequence, "expected SEQUENCE") + val name = expectIdentifier("expected sequence name") + var incrementBy: Long? = null + var minValue: Long? = null + var clearMinValue = false + var maxValue: Long? = null + var clearMaxValue = false + var restart = false + var restartWith: Long? = null + var cycle: Boolean? = null + while (true) { + when { + match(TokenType.Increment) -> { + match(TokenType.By) + incrementBy = expectLongLiteral("expected a value after INCREMENT BY") + } + match(TokenType.Minvalue) -> minValue = expectLongLiteral("expected a value after MINVALUE") + match(TokenType.Maxvalue) -> maxValue = expectLongLiteral("expected a value after MAXVALUE") + match(TokenType.Cycle) -> cycle = true + match(TokenType.Restart) -> { + restart = true + if (match(TokenType.With)) restartWith = expectLongLiteral("expected a value after RESTART WITH") + } + check(TokenType.No) && peekIsSequenceNoClause() -> { + advance() + when { + match(TokenType.Minvalue) -> clearMinValue = true + match(TokenType.Maxvalue) -> clearMaxValue = true + match(TokenType.Cycle) -> cycle = false + else -> error("unreachable: guarded by peekIsSequenceNoClause") + } + } + else -> break + } + } + return AlterSequenceStatement(name, incrementBy, minValue, clearMinValue, maxValue, clearMaxValue, restart, restartWith, cycle) + } + + private fun parseAlterTable(): AlterTableStatement { + expect(TokenType.Table, "expected TABLE after ALTER") + val tableName = expectIdentifier("expected table name") + val action: AlterTableAction = when { + match(TokenType.Add) -> { + if (match(TokenType.Column)) { + AddColumn(parseAlterAddColumnDefinition()) + } else { + expect(TokenType.Constraint, "expected COLUMN or CONSTRAINT after ADD") + val name = expectIdentifier("expected constraint name after CONSTRAINT") + val constraint: AddedConstraint = when { + check(TokenType.Foreign) -> AddedForeignKey(parseTableForeignKey().copy(name = name)) + check(TokenType.Check) -> AddedCheck(parseTableCheck().copy(name = name)) + check(TokenType.Unique) -> AddedUnique(parseTableUnique().copy(name = name)) + else -> throw error( + "expected FOREIGN KEY, CHECK, or UNIQUE after CONSTRAINT $name but found '${describe(peek())}'", + peek() + ) + } + AddConstraint(constraint) + } + } + match(TokenType.Drop) -> { + if (match(TokenType.Column)) { + val columnName = expectIdentifier("expected column name") + DropColumn(columnName, parseDropMode()) + } else { + expect(TokenType.Constraint, "expected COLUMN or CONSTRAINT after DROP") + val name = expectIdentifier("expected constraint name") + DropConstraint(name, parseDropMode()) + } + } + match(TokenType.Rename) -> { + if (match(TokenType.To)) { + val newName = expectIdentifier("expected new table name") + RenameTable(newName) + } else { + expect(TokenType.Column, "expected COLUMN or TO after RENAME") + val oldName = expectIdentifier("expected column name") + expect(TokenType.To, "expected TO after column name") + val newName = expectIdentifier("expected new column name") + RenameColumn(oldName, newName) + } + } + match(TokenType.Alter) -> { + match(TokenType.Column) + val columnName = expectIdentifier("expected column name") + if (!isContextualKeyword("TYPE")) { + throw error("expected TYPE after column name in ALTER TABLE ... ALTER COLUMN", peek()) + } + advance() + val spec = parseTypeSpec() + val using = if (match(TokenType.Using)) parseExpression() else null + ChangeColumnType(columnName, spec.type, spec.precision, spec.scale, using) + } + else -> throw error( + "expected ADD, DROP, RENAME, or ALTER after table name in ALTER TABLE but found '${describe(peek())}'", + peek() + ) + } + return AlterTableStatement(tableName, action) + } + + /** + * A deliberately restricted column definition: no PRIMARY KEY/REFERENCES/UNIQUE/CHECK, since + * enforcing those against a table that may already hold rows (composite-key interactions, + * FK target validation, CASCADE bookkeeping) is out of scope for a single ADD COLUMN - CREATE + * UNIQUE INDEX already covers UNIQUE on an existing table, and CREATE TABLE covers the rest. + * GENERATED ALWAYS AS is allowed, unlike those - see `Executor.executeAddColumn`/ + * `DiskExecutor.executeAddColumn` for why it does not share that "out of scope" reasoning + * (it backfills by evaluating the expression per existing row, not by writing one constant). + */ + private fun parseAlterAddColumnDefinition(): Column { + val columnName = expectIdentifier("expected column name") + val spec = parseTypeSpec() + var notNull = false + var default: Expression? = null + var generated: Expression? = null + var collation = Collation.Binary + while (true) { + when { + check(TokenType.Not) -> { + advance() + expect(TokenType.Null, "expected NULL after NOT") + notNull = true + } + check(TokenType.Default) -> { + advance() + default = parseExpression() + } + isContextualKeyword("GENERATED") -> generated = parseGeneratedClause() + check(TokenType.Collate) -> collation = parseColumnCollation() + check(TokenType.Primary) -> throw error("ALTER TABLE ADD COLUMN cannot declare PRIMARY KEY", peek()) + check(TokenType.References) -> throw error("ALTER TABLE ADD COLUMN cannot declare REFERENCES", peek()) + check(TokenType.Unique) -> + throw error("ALTER TABLE ADD COLUMN cannot declare UNIQUE; use CREATE UNIQUE INDEX instead", peek()) + check(TokenType.Check) -> throw error("ALTER TABLE ADD COLUMN cannot declare CHECK", peek()) + else -> break + } + } + if (generated != null && default != null) { + throw error("a column cannot declare both DEFAULT and GENERATED ALWAYS AS", peek()) + } + return Column(columnName, spec.type, notNull, primaryKey = false, spec.precision, spec.scale, default, generated, collation) + } + private fun parseExpression(): Expression = parseOr() private fun parseOr(): Expression { @@ -322,19 +1348,61 @@ class Parser(private val tokens: List) { private fun parseComparison(): Expression { var left = parseAdditive() while (true) { - val operator = when { - check(TokenType.Equal) -> BinaryOperator.Equal - check(TokenType.NotEqual) -> BinaryOperator.NotEqual - check(TokenType.Less) -> BinaryOperator.Less - check(TokenType.LessEqual) -> BinaryOperator.LessEqual - check(TokenType.Greater) -> BinaryOperator.Greater - check(TokenType.GreaterEqual) -> BinaryOperator.GreaterEqual - else -> break + val negated = match(TokenType.Not) + left = when { + check(TokenType.In) -> { advance(); parseInSuffix(left, negated) } + check(TokenType.Between) -> { advance(); parseBetweenSuffix(left, negated) } + check(TokenType.Like) -> { advance(); parseLikeSuffix(left, negated) } + negated -> throw error("expected IN, BETWEEN, or LIKE after NOT", peek()) + check(TokenType.Is) -> { advance(); parseIsSuffix(left) } + else -> { + val operator = when { + check(TokenType.Equal) -> BinaryOperator.Equal + check(TokenType.NotEqual) -> BinaryOperator.NotEqual + check(TokenType.Less) -> BinaryOperator.Less + check(TokenType.LessEqual) -> BinaryOperator.LessEqual + check(TokenType.Greater) -> BinaryOperator.Greater + check(TokenType.GreaterEqual) -> BinaryOperator.GreaterEqual + else -> return left + } + advance() + BinaryExpression(operator, left, parseAdditive()) + } } - advance() - left = BinaryExpression(operator, left, parseAdditive()) } - return left + } + + private fun parseIsSuffix(left: Expression): Expression { + val negated = match(TokenType.Not) + expect(TokenType.Null, "expected NULL after IS" + if (negated) " NOT" else "") + return IsNullExpression(left, negated) + } + + private fun parseInSuffix(left: Expression, negated: Boolean): Expression { + expect(TokenType.LeftParen, "expected '(' after IN") + if (check(TokenType.Select)) { + val select = parseSelect() + expect(TokenType.RightParen, "expected ')' after IN subquery") + return InSubqueryExpression(left, select, negated) + } + val values = mutableListOf() + do { + values.add(parseExpression()) + } while (match(TokenType.Comma)) + expect(TokenType.RightParen, "expected ')' after IN list") + return InExpression(left, values, negated) + } + + private fun parseBetweenSuffix(left: Expression, negated: Boolean): Expression { + val low = parseAdditive() + expect(TokenType.And, "expected AND in BETWEEN") + val high = parseAdditive() + return BetweenExpression(left, low, high, negated) + } + + private fun parseLikeSuffix(left: Expression, negated: Boolean): Expression { + val pattern = parseAdditive() + return LikeExpression(left, pattern, negated) } private fun parseAdditive(): Expression { @@ -352,7 +1420,7 @@ class Parser(private val tokens: List) { } private fun parseMultiplicative(): Expression { - var left = parseUnary() + var left = parseCollate() while (true) { val operator = when { check(TokenType.Star) -> BinaryOperator.Multiply @@ -360,11 +1428,30 @@ class Parser(private val tokens: List) { else -> break } advance() - left = BinaryExpression(operator, left, parseUnary()) + left = BinaryExpression(operator, left, parseCollate()) } return left } + /** + * `expr COLLATE name` - sits between unary and multiplicative in precedence, matching + * SQLite's own grammar (binds tighter than every arithmetic/comparison operator, but looser + * than unary `-`/`NOT`, so `-x COLLATE NOCASE` is `(-x) COLLATE NOCASE` and + * `a || b COLLATE NOCASE` would apply to `b` alone). Left-associative chaining + * (`x COLLATE A COLLATE B`) is accepted like SQLite accepts it - only the last one written + * has any effect, since each wraps the previous [CollateExpression] as its own operand. + */ + private fun parseCollate(): Expression { + var expression = parseUnary() + while (match(TokenType.Collate)) { + val nameToken = expect(TokenType.Identifier, "expected a collation name after COLLATE") + val collation = Collation.fromName(nameToken.lexeme) + ?: throw error("no such collating sequence: ${nameToken.lexeme}", nameToken) + expression = CollateExpression(expression, collation) + } + return expression + } + private fun parseUnary(): Expression { if (match(TokenType.Minus)) { return UnaryExpression(UnaryOperator.Negate, parseUnary()) @@ -393,6 +1480,27 @@ class Parser(private val tokens: List) { TokenType.Null -> { advance(); LiteralExpression(Value.NullValue) } + TokenType.Parameter -> { + advance() + val index = token.intValue!! + if (index < 1) throw error("parameter index must be >= 1 but was $index", token) + ParameterExpression(index.toInt()) + } + TokenType.Case -> parseCase() + TokenType.Exists -> parseExists() + TokenType.Cast -> parseCast() + // LEFT/RIGHT are otherwise reserved (LEFT/RIGHT JOIN), but LEFT(str, n)/RIGHT(str, n) are + // real, commonly expected scalar functions - only valid here as a call, never as a bare + // column reference, so a missing '(' is unconditionally an error rather than falling back + // to ColumnExpression like the TokenType.Identifier branch below does. + TokenType.Left -> { + advance() + if (check(TokenType.LeftParen)) parseFunctionCall(token.lexeme) else throw error("expected '(' after LEFT", peek()) + } + TokenType.Right -> { + advance() + if (check(TokenType.LeftParen)) parseFunctionCall(token.lexeme) else throw error("expected '(' after RIGHT", peek()) + } TokenType.Identifier -> { advance() when { @@ -403,23 +1511,266 @@ class Parser(private val tokens: List) { } TokenType.LeftParen -> { advance() - val expression = parseExpression() - expect(TokenType.RightParen, "expected ')' after expression") - expression + if (check(TokenType.Select)) { + val select = parseSelect() + expect(TokenType.RightParen, "expected ')' after subquery") + ScalarSubqueryExpression(select) + } else { + val expression = parseExpression() + expect(TokenType.RightParen, "expected ')' after expression") + expression + } } else -> throw error("expected an expression but found '${describe(token)}'", token) } } + private fun parseCase(): Expression { + expect(TokenType.Case, "expected CASE") + val operand = if (!check(TokenType.When)) parseExpression() else null + val branches = mutableListOf() + while (match(TokenType.When)) { + val condition = parseExpression() + expect(TokenType.Then, "expected THEN after CASE condition") + val result = parseExpression() + branches.add(CaseBranch(condition, result)) + } + if (branches.isEmpty()) throw error("expected at least one WHEN clause in CASE expression", peek()) + val elseExpression = if (match(TokenType.Else)) parseExpression() else null + expect(TokenType.End, "expected END to close CASE expression") + return CaseExpression(operand, branches, elseExpression) + } + + private fun parseCast(): Expression { + expect(TokenType.Cast, "expected CAST") + expect(TokenType.LeftParen, "expected '(' after CAST") + val operand = parseExpression() + expect(TokenType.As, "expected AS in CAST") + val spec = parseTypeSpec() + expect(TokenType.RightParen, "expected ')' after CAST target type") + return CastExpression(operand, spec.type, spec.precision, spec.scale) + } + + private fun parseExists(): Expression { + expect(TokenType.Exists, "expected EXISTS") + expect(TokenType.LeftParen, "expected '(' after EXISTS") + val select = parseSelect() + expect(TokenType.RightParen, "expected ')' after EXISTS subquery") + return ExistsExpression(select) + } + private fun parseFunctionCall(name: String): Expression { expect(TokenType.LeftParen, "expected '(' after function name") - if (match(TokenType.Star)) { - expect(TokenType.RightParen, "expected ')' after '*'") - return FunctionExpression(name, argument = null, star = true) + val upperName = name.uppercase() + if (upperName == "STRING_AGG") return parseStringAggCall(name) + val isAggregate = upperName in AGGREGATE_FUNCTIONS + var star = false + var distinct = false + val arguments = mutableListOf() + if (isAggregate && match(TokenType.Distinct)) { + distinct = true + arguments.add(parseExpression()) + } else if (isAggregate && match(TokenType.Star)) { + star = true + } else if (!check(TokenType.RightParen)) { + do { + arguments.add(parseExpression()) + } while (match(TokenType.Comma)) + } + expect(TokenType.RightParen, "expected ')' after function arguments") + // `FILTER` is not a reserved keyword either (same reasoning as ROLLUP/CUBE/GROUPING above) - + // only meaningful right after an aggregate call's closing paren when immediately followed by + // `(`, so a column or table legitimately named `filter` is unaffected. + var filter: Expression? = null + if (isContextualKeyword("FILTER") && peekNext().type == TokenType.LeftParen) { + if (!isAggregate) throw error("FILTER can only be used with an aggregate function", peek()) + advance() + advance() + expect(TokenType.Where, "expected WHERE after FILTER (") + filter = parseExpression() + expect(TokenType.RightParen, "expected ')' to close FILTER clause") + } + if (check(TokenType.Over)) { + if (distinct) throw error("DISTINCT is not supported in window functions", peek()) + if (filter != null) throw error("FILTER is not supported in window functions", peek()) + return parseWindowExpression(name, upperName, isAggregate, arguments, star) + } + if (isAggregate) { + if (star) return FunctionExpression(name, argument = null, star = true, filter = filter) + if (arguments.size != 1) { + throw error("aggregate function $name requires exactly one argument", peek()) + } + return FunctionExpression(name, arguments.single(), star = false, distinct = distinct, filter = filter) + } + if (upperName == "NEXTVAL" || upperName == "CURRVAL") { + return parseSequenceFunctionCall(name, upperName, arguments) + } + if (upperName == "MERGE_ACTION") { + if (arguments.isNotEmpty()) throw error("MERGE_ACTION takes no arguments", peek()) + return MergeActionExpression + } + return CallExpression(name, arguments) + } + + /** + * `STRING_AGG(value, separator [ORDER BY key [ASC|DESC], ...])` - shaped nothing like the rest of + * [AGGREGATE_FUNCTIONS] (two required arguments, no `*` form, an optional ORDER BY *inside* the + * parens that sorts values within each group independently of the query's own ORDER BY), so it + * gets its own parse path called before `(` is even consumed for the generic aggregate branch. + * The left paren was already consumed by [parseFunctionCall]. Never valid with `OVER` - unlike + * COUNT/SUM/AVG/MIN/MAX, [kestrel.engine.AggregateFunctions] has no per-row windowed form for it + * (concatenation order inside a sliding frame is a separate, unimplemented feature), so `OVER` + * right after the closing `)` is rejected here with a clear message instead of falling through to + * a generic "expected ';'" parse error later. + */ + private fun parseStringAggCall(name: String): Expression { + val value = parseExpression() + expect(TokenType.Comma, "expected ',' after STRING_AGG value argument") + val separator = parseExpression() + val orderBy = parseOrderKeys() + expect(TokenType.RightParen, "expected ')' after STRING_AGG arguments") + if (check(TokenType.Over)) throw error("STRING_AGG cannot be used as a window function", peek()) + return FunctionExpression(name, value, star = false, separator = separator, orderBy = orderBy) + } + + /** + * Unlike every other function call, [arguments] must be exactly one string literal - the + * sequence's name is resolved once here, at parse time, rather than carried as a general + * [Expression] evaluated per row (see [NextValueExpression]'s own doc for why). A bad argument + * (wrong count, wrong shape - a column reference, another call, ...) is a parse-time error here + * rather than something [kestrel.engine.Evaluator] discovers later, matching how this parser + * already treats e.g. a non-identifier index/table name elsewhere. + */ + private fun parseSequenceFunctionCall(name: String, upperName: String, arguments: List): Expression { + val sequenceNameArgument = arguments.singleOrNull() as? LiteralExpression + val sequenceName = (sequenceNameArgument?.value as? Value.TextValue)?.value + ?: throw error("$name requires a single string literal argument naming the sequence", peek()) + return if (upperName == "NEXTVAL") NextValueExpression(sequenceName) else CurrentValueExpression(sequenceName) + } + + /** + * `name(args) OVER (...)`, reached once `name(args)` is already parsed and `OVER` is next. + * Ranking functions (ROW_NUMBER/RANK/DENSE_RANK) take no arguments at all; the aggregate + * vocabulary (COUNT/SUM/AVG/MIN/MAX) is exactly the [AGGREGATE_FUNCTIONS] set, reused here with + * per-row windowed semantics instead of collapsing a group. LAG/LEAD take 1-3 arguments (value, + * offset, default); FIRST_VALUE/LAST_VALUE/NTILE take exactly one. Any other name before OVER is + * a clear error rather than silently falling back to a plain (non-window) call. + */ + private fun parseWindowExpression( + name: String, + upperName: String, + isAggregate: Boolean, + arguments: List, + star: Boolean + ): Expression { + val kind = WINDOW_RANKING_FUNCTIONS[upperName] + ?: WINDOW_OTHER_FUNCTIONS[upperName] + ?: (if (isAggregate) WindowFunctionKind.Aggregate else null) + ?: throw error("function $name cannot be used as a window function", peek()) + when (kind) { + WindowFunctionKind.RowNumber, WindowFunctionKind.Rank, WindowFunctionKind.DenseRank -> + if (star || arguments.isNotEmpty()) throw error("$name does not take arguments", peek()) + WindowFunctionKind.Aggregate -> + if (!star && arguments.size != 1) throw error("window function $name requires exactly one argument", peek()) + WindowFunctionKind.Lag, WindowFunctionKind.Lead -> + if (star || arguments.isEmpty() || arguments.size > 3) { + throw error("$name requires between 1 and 3 arguments", peek()) + } + WindowFunctionKind.FirstValue, WindowFunctionKind.LastValue, WindowFunctionKind.Ntile -> + if (star || arguments.size != 1) throw error("$name requires exactly one argument", peek()) + } + expect(TokenType.Over, "expected OVER") + expect(TokenType.LeftParen, "expected '(' after OVER") + val partitionBy = mutableListOf() + if (match(TokenType.Partition)) { + expect(TokenType.By, "expected BY after PARTITION") + do { + partitionBy.add(parseExpression()) + } while (match(TokenType.Comma)) + } + val orderBy = parseOrderKeys() + val frame = parseWindowFrame() + expect(TokenType.RightParen, "expected ')' to close OVER clause") + val argument = if (kind == WindowFunctionKind.Aggregate) arguments.singleOrNull() else arguments.getOrNull(0) + val offset = if (kind == WindowFunctionKind.Lag || kind == WindowFunctionKind.Lead) arguments.getOrNull(1) else null + val defaultValue = if (kind == WindowFunctionKind.Lag || kind == WindowFunctionKind.Lead) arguments.getOrNull(2) else null + return WindowExpression(name, kind, argument, star, partitionBy, orderBy, offset, defaultValue, frame) + } + + /** + * `ROWS|RANGE|GROUPS BETWEEN AND `, or the shorthand `ROWS|RANGE|GROUPS ` + * (equivalent to `... BETWEEN AND CURRENT ROW`, exactly like every other SQL engine's + * shorthand). + */ + private fun parseWindowFrame(): WindowFrame? { + val unit = when { + match(TokenType.Rows) -> FrameUnit.Rows + match(TokenType.Range) -> FrameUnit.Range + match(TokenType.Groups) -> FrameUnit.Groups + else -> return null + } + val (start, end) = if (match(TokenType.Between)) { + val start = parseFrameBound() + expect(TokenType.And, "expected AND between the two bounds of a frame") + val end = parseFrameBound() + start to end + } else { + parseFrameBound() to FrameBound(FrameBoundType.CurrentRow) + } + if (frameBoundRank(start.type) > frameBoundRank(end.type)) { + throw error("frame start cannot come after frame end", peek()) + } + return WindowFrame(unit, start, end, parseFrameExclude()) + } + + /** + * `EXCLUDE CURRENT ROW | EXCLUDE GROUP | EXCLUDE TIES | EXCLUDE NO OTHERS`, optional and always + * following the frame's bounds. Absent entirely is [FrameExclude.NoOthers], identical to writing + * `EXCLUDE NO OTHERS` out - SQL engines never require it and this parser accepts either. + */ + private fun parseFrameExclude(): FrameExclude { + if (!match(TokenType.Exclude)) return FrameExclude.NoOthers + return when { + match(TokenType.Current) -> { + expect(TokenType.Row, "expected ROW after EXCLUDE CURRENT") + FrameExclude.CurrentRow + } + match(TokenType.Group) -> FrameExclude.Group + match(TokenType.Ties) -> FrameExclude.Ties + match(TokenType.No) -> { + expect(TokenType.Others, "expected OTHERS after EXCLUDE NO") + FrameExclude.NoOthers + } + else -> throw error("expected CURRENT ROW, GROUP, TIES, or NO OTHERS after EXCLUDE", peek()) + } + } + + private fun frameBoundRank(type: FrameBoundType): Int = when (type) { + FrameBoundType.UnboundedPreceding -> 0 + FrameBoundType.Preceding -> 1 + FrameBoundType.CurrentRow -> 2 + FrameBoundType.Following -> 3 + FrameBoundType.UnboundedFollowing -> 4 + } + + private fun parseFrameBound(): FrameBound { + if (match(TokenType.Unbounded)) { + return when { + match(TokenType.Preceding) -> FrameBound(FrameBoundType.UnboundedPreceding) + match(TokenType.Following) -> FrameBound(FrameBoundType.UnboundedFollowing) + else -> throw error("expected PRECEDING or FOLLOWING after UNBOUNDED", peek()) + } + } + if (match(TokenType.Current)) { + expect(TokenType.Row, "expected ROW after CURRENT") + return FrameBound(FrameBoundType.CurrentRow) + } + val offset = parseExpression() + return when { + match(TokenType.Preceding) -> FrameBound(FrameBoundType.Preceding, offset) + match(TokenType.Following) -> FrameBound(FrameBoundType.Following, offset) + else -> throw error("expected PRECEDING or FOLLOWING", peek()) } - val argument = parseExpression() - expect(TokenType.RightParen, "expected ')' after function argument") - return FunctionExpression(name, argument, star = false) } private fun peek(): Token = tokens[position] @@ -454,4 +1805,20 @@ class Parser(private val tokens: List) { private fun describe(token: Token): String = if (token.type == TokenType.EndOfInput) "end of input" else token.lexeme + + private companion object { + val AGGREGATE_FUNCTIONS = setOf("COUNT", "SUM", "AVG", "MIN", "MAX") + val WINDOW_RANKING_FUNCTIONS = mapOf( + "ROW_NUMBER" to WindowFunctionKind.RowNumber, + "RANK" to WindowFunctionKind.Rank, + "DENSE_RANK" to WindowFunctionKind.DenseRank + ) + val WINDOW_OTHER_FUNCTIONS = mapOf( + "LAG" to WindowFunctionKind.Lag, + "LEAD" to WindowFunctionKind.Lead, + "FIRST_VALUE" to WindowFunctionKind.FirstValue, + "LAST_VALUE" to WindowFunctionKind.LastValue, + "NTILE" to WindowFunctionKind.Ntile + ) + } } diff --git a/src/main/kotlin/kestrel/storage/Collation.kt b/src/main/kotlin/kestrel/storage/Collation.kt new file mode 100644 index 0000000..f3f145c --- /dev/null +++ b/src/main/kotlin/kestrel/storage/Collation.kt @@ -0,0 +1,54 @@ +package kestrel.storage + +/** + * A text comparison sequence, selected by the `COLLATE` operator (`expr COLLATE name`) - see + * [kestrel.sql.ast.CollateExpression]. Only the three built-in sequences SQLite ships are + * supported (no `CREATE COLLATION`, no user-defined comparator); [Binary] is what every text + * comparison already used before COLLATE existed, so it stays the default everywhere a caller + * does not thread a collation through at all. + */ +enum class Collation(val sqlName: String) { + /** Plain UTF-16 code-unit order - [String.compareTo] - identical to this engine's pre-COLLATE behavior. */ + Binary("BINARY"), + + /** + * Case-insensitive, but only for ASCII `A-Z`/`a-z` - matching SQLite's own NOCASE exactly + * (not full Unicode case folding, which would depend on locale for a handful of characters + * like Turkish "I"). Folds both operands to uppercase ASCII before comparing. + */ + NoCase("NOCASE"), + + /** + * Ignores trailing ASCII space characters, matching SQLite's built-in RTRIM collation. + * Leading and interior whitespace still participates in the comparison. + */ + Rtrim("RTRIM"); + + fun compare(left: String, right: String): Int = canonicalize(left).compareTo(canonicalize(right)) + + /** + * [value] rewritten so that plain [String.equals]/[String.hashCode]/[Comparable.compareTo] on + * the result matches this collation's own [compare] - shared by [compare] itself and by every + * equality-based dedup (`GROUP BY`, `DISTINCT`, an aggregate's own `DISTINCT`) that needs a + * hashable key rather than a pairwise comparison. [Binary] returns [value] unchanged, so folding + * a value under it and using the result as a key is indistinguishable from the pre-COLLATE + * behavior of comparing/hashing the value directly. + */ + fun canonicalize(value: String): String = when (this) { + Binary -> value + NoCase -> foldAsciiCase(value) + Rtrim -> value.trimEnd(' ') + } + + private fun foldAsciiCase(value: String): String { + val builder = StringBuilder(value.length) + for (character in value) { + builder.append(if (character in 'a'..'z') character.uppercaseChar() else character) + } + return builder.toString() + } + + companion object { + fun fromName(name: String): Collation? = entries.find { it.sqlName.equals(name, ignoreCase = true) } + } +} diff --git a/src/main/kotlin/kestrel/storage/DecimalColumns.kt b/src/main/kotlin/kestrel/storage/DecimalColumns.kt new file mode 100644 index 0000000..7e995ce --- /dev/null +++ b/src/main/kotlin/kestrel/storage/DecimalColumns.kt @@ -0,0 +1,28 @@ +package kestrel.storage + +import java.math.BigDecimal +import java.math.RoundingMode +import kestrel.KestrelException + +/** + * Shared DECIMAL(precision, scale) rules used both when a value is coerced into a + * DECIMAL column (INSERT/UPDATE) and when it is produced by an explicit CAST. Kept as + * a single place so both call sites agree on rounding mode and overflow behavior. + */ +object DecimalColumns { + const val DEFAULT_PRECISION = 18 + const val DEFAULT_SCALE = 2 + + fun normalize(raw: BigDecimal, precision: Int?, scale: Int?, context: String): BigDecimal { + val effectiveScale = scale ?: DEFAULT_SCALE + val effectivePrecision = precision ?: DEFAULT_PRECISION + val rounded = raw.setScale(effectiveScale, RoundingMode.HALF_UP) + val digits = rounded.unscaledValue().abs().toString().length + if (digits > effectivePrecision) { + throw KestrelException( + "$context: $rounded has more than $effectivePrecision digits for DECIMAL($effectivePrecision, $effectiveScale)" + ) + } + return rounded + } +} diff --git a/src/main/kotlin/kestrel/storage/Value.kt b/src/main/kotlin/kestrel/storage/Value.kt index da9aeb2..2799591 100644 --- a/src/main/kotlin/kestrel/storage/Value.kt +++ b/src/main/kotlin/kestrel/storage/Value.kt @@ -1,7 +1,10 @@ package kestrel.storage +import java.math.BigDecimal +import java.time.LocalDate + enum class ColumnType { - Integer, Text, Boolean, Real; + Integer, Text, Boolean, Real, Decimal, Date; val displayName: String get() = when (this) { @@ -9,6 +12,8 @@ enum class ColumnType { Text -> "TEXT" Boolean -> "BOOLEAN" Real -> "REAL" + Decimal -> "DECIMAL" + Date -> "DATE" } companion object { @@ -17,11 +22,20 @@ enum class ColumnType { "TEXT" -> Text "BOOLEAN" -> Boolean "REAL" -> Real + "DECIMAL" -> Decimal + "DATE" -> Date else -> null } } } +/** Renders a type with its DECIMAL(precision[, scale]) parameters, if any were declared. */ +fun ColumnType.displayWith(precision: Int?, scale: Int?): String = when { + this != ColumnType.Decimal || precision == null -> displayName + scale == null -> "$displayName($precision)" + else -> "$displayName($precision, $scale)" +} + sealed class Value { abstract val type: ColumnType? @@ -41,6 +55,30 @@ sealed class Value { override val type get() = ColumnType.Boolean } + /** + * Arbitrary-precision fixed-point number. Equality and hashing compare by numeric + * value (via [BigDecimal.compareTo]) rather than by representation, so `1.5` and + * `1.50` are the same value for GROUP BY/DISTINCT/UNION dedup purposes, matching + * standard SQL DECIMAL semantics instead of Java's stricter [BigDecimal.equals]. + */ + class DecimalValue(val value: BigDecimal) : Value() { + override val type get() = ColumnType.Decimal + + override fun equals(other: Any?): Boolean = + other is DecimalValue && value.compareTo(other.value) == 0 + + override fun hashCode(): Int { + val canonical = if (value.signum() == 0) BigDecimal.ZERO else value.stripTrailingZeros() + return canonical.hashCode() + } + + override fun toString(): String = "DecimalValue(value=$value)" + } + + data class DateValue(val value: LocalDate) : Value() { + override val type get() = ColumnType.Date + } + object NullValue : Value() { override val type: ColumnType? get() = null } diff --git a/src/main/kotlin/kestrel/storage/ValueOrdering.kt b/src/main/kotlin/kestrel/storage/ValueOrdering.kt index f7fb461..5b4bf63 100644 --- a/src/main/kotlin/kestrel/storage/ValueOrdering.kt +++ b/src/main/kotlin/kestrel/storage/ValueOrdering.kt @@ -1,14 +1,26 @@ package kestrel.storage +import java.math.BigDecimal + object ValueOrdering : Comparator { - override fun compare(left: Value, right: Value): Int = when { + override fun compare(left: Value, right: Value): Int = compare(left, right, Collation.Binary) + + /** + * Same ordering as the plain two-argument [compare], except two [Value.TextValue]s are + * compared through [collation] instead of always using [String.compareTo] - see + * [kestrel.engine.compareOrderKeyValues], the only caller that ever passes a non-[Collation.Binary] + * value (an explicit `ORDER BY ... COLLATE name`). + */ + fun compare(left: Value, right: Value, collation: Collation): Int = when { left is Value.NullValue && right is Value.NullValue -> 0 left is Value.NullValue -> -1 right is Value.NullValue -> 1 left is Value.IntegerValue && right is Value.IntegerValue -> left.value.compareTo(right.value) + left is Value.DecimalValue || right is Value.DecimalValue -> compareAsDecimal(left, right) isNumeric(left) && isNumeric(right) -> asDouble(left).compareTo(asDouble(right)) - left is Value.TextValue && right is Value.TextValue -> left.value.compareTo(right.value) + left is Value.TextValue && right is Value.TextValue -> collation.compare(left.value, right.value) left is Value.BooleanValue && right is Value.BooleanValue -> left.value.compareTo(right.value) + left is Value.DateValue && right is Value.DateValue -> left.value.compareTo(right.value) else -> rank(left).compareTo(rank(right)) } @@ -20,11 +32,27 @@ object ValueOrdering : Comparator { else -> 0.0 } + /** Falls back to [rank] ordering (never throws) when one side isn't itself numeric. */ + private fun compareAsDecimal(left: Value, right: Value): Int { + val a = asBigDecimalOrNull(left) ?: return rank(left).compareTo(rank(right)) + val b = asBigDecimalOrNull(right) ?: return rank(left).compareTo(rank(right)) + return a.compareTo(b) + } + + private fun asBigDecimalOrNull(value: Value): BigDecimal? = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> null + } + private fun rank(value: Value): Int = when (value) { is Value.NullValue -> 0 is Value.BooleanValue -> 1 is Value.IntegerValue -> 2 is Value.RealValue -> 2 - is Value.TextValue -> 3 + is Value.DecimalValue -> 2 + is Value.DateValue -> 3 + is Value.TextValue -> 4 } } diff --git a/src/main/kotlin/kestrel/store/btree/BPlusTree.kt b/src/main/kotlin/kestrel/store/btree/BPlusTree.kt new file mode 100644 index 0000000..77b6b6a --- /dev/null +++ b/src/main/kotlin/kestrel/store/btree/BPlusTree.kt @@ -0,0 +1,392 @@ +package kestrel.store.btree + +import kestrel.KestrelException +import kestrel.store.codec.KeyCodec +import kestrel.store.page.Page +import kestrel.store.page.PageStore + +class BTreeCorruptionException(message: String) : KestrelException(message) + +/** + * A disk-backed B+Tree mapping order-preserving byte keys (see [KeyCodec]) to + * row pointers. Nodes live on [Page]s fetched through a [BufferPool]; leaves + * are linked for ordered range scans. Insert splits an overflowing node and + * propagates a promoted separator key upward, growing the root when needed. + * Delete removes the entry and, when a node drops below + * [UNDERFLOW_THRESHOLD], redistributes an entry from a sibling with spare + * capacity or merges with one, shrinking the root when it is left with a + * single child. + */ +class BPlusTree private constructor(private val bufferPool: PageStore, rootPageId: Long) { + companion object { + private const val UNDERFLOW_THRESHOLD = Page.PAGE_SIZE / 4 + + fun create(bufferPool: PageStore): BPlusTree { + val page = bufferPool.newPage() + NodeCodec.writeTo(page, BTreeNode.Leaf(page.id, mutableListOf(), mutableListOf(), NodeCodec.NO_PAGE)) + bufferPool.unpinPage(page.id, true) + return BPlusTree(bufferPool, page.id) + } + + fun open(bufferPool: PageStore, rootPageId: Long): BPlusTree = BPlusTree(bufferPool, rootPageId) + } + + var rootPageId: Long = rootPageId + private set + + fun search(key: ByteArray): Long? { + var pageId = rootPageId + while (true) { + val (_, node) = fetch(pageId) + when (node) { + is BTreeNode.Internal -> { + val nextId = node.children[childIndexFor(node.keys, key)] + bufferPool.unpinPage(pageId, false) + pageId = nextId + } + is BTreeNode.Leaf -> { + val idx = lowerBound(node.keys, key) + val result = if (idx < node.keys.size && KeyCodec.compare(node.keys[idx], key) == 0) { + node.values[idx] + } else { + null + } + bufferPool.unpinPage(pageId, false) + return result + } + } + } + } + + fun insert(key: ByteArray, value: Long) { + val split = insertRecursive(rootPageId, key, value) + if (split != null) { + val newRoot = bufferPool.newPage() + NodeCodec.writeTo( + newRoot, + BTreeNode.Internal(newRoot.id, mutableListOf(split.promotedKey), mutableListOf(rootPageId, split.newPageId)) + ) + bufferPool.unpinPage(newRoot.id, true) + rootPageId = newRoot.id + } + } + + fun delete(key: ByteArray): Boolean { + val found = deleteRecursive(rootPageId, key) + val (_, rootNode) = fetch(rootPageId) + if (rootNode is BTreeNode.Internal && rootNode.keys.isEmpty()) { + val onlyChild = rootNode.children[0] + bufferPool.unpinPage(rootPageId, false) + bufferPool.deletePage(rootPageId) + rootPageId = onlyChild + } else { + bufferPool.unpinPage(rootPageId, false) + } + return found + } + + /** Every page belonging to this tree: every internal node and every leaf, reached by a full traversal from the root. */ + fun allPageIds(): List { + val ids = mutableListOf() + val pending = ArrayDeque() + pending.add(rootPageId) + while (pending.isNotEmpty()) { + val pageId = pending.removeFirst() + ids.add(pageId) + val (_, node) = fetch(pageId) + bufferPool.unpinPage(pageId, false) + if (node is BTreeNode.Internal) pending.addAll(node.children) + } + return ids + } + + /** Ordered scan over `[from, to]` (or `[from, to)` when [toInclusive] is false). Null bounds are open. */ + fun rangeScan(from: ByteArray? = null, to: ByteArray? = null, toInclusive: Boolean = false): Iterator> { + val startLeafId = leftmostLeafFor(from) + return iterator { + var currentLeafId = startLeafId + var firstLeaf = true + outer@ while (currentLeafId != NodeCodec.NO_PAGE) { + val (_, leaf) = fetch(currentLeafId) + leaf as BTreeNode.Leaf + var index = if (firstLeaf && from != null) lowerBound(leaf.keys, from) else 0 + firstLeaf = false + while (index < leaf.keys.size) { + val currentKey = leaf.keys[index] + if (to != null) { + val cmp = KeyCodec.compare(currentKey, to) + if (cmp > 0 || (cmp == 0 && !toInclusive)) { + bufferPool.unpinPage(currentLeafId, false) + break@outer + } + } + yield(currentKey to leaf.values[index]) + index++ + } + val next = leaf.nextLeaf + bufferPool.unpinPage(currentLeafId, false) + currentLeafId = next + } + } + } + + private fun leftmostLeafFor(from: ByteArray?): Long { + var pageId = rootPageId + while (true) { + val (_, node) = fetch(pageId) + when (node) { + is BTreeNode.Internal -> { + val idx = if (from == null) 0 else childIndexFor(node.keys, from) + val nextId = node.children[idx] + bufferPool.unpinPage(pageId, false) + pageId = nextId + } + is BTreeNode.Leaf -> { + bufferPool.unpinPage(pageId, false) + return pageId + } + } + } + } + + private data class SplitResult(val promotedKey: ByteArray, val newPageId: Long) + + private fun insertRecursive(pageId: Long, key: ByteArray, value: Long): SplitResult? { + val (page, node) = fetch(pageId) + var dirty = false + val result: SplitResult? + when (node) { + is BTreeNode.Leaf -> { + val idx = lowerBound(node.keys, key) + if (idx < node.keys.size && KeyCodec.compare(node.keys[idx], key) == 0) { + node.values[idx] = value + } else { + node.keys.add(idx, key) + node.values.add(idx, value) + } + dirty = true + result = if (NodeCodec.fitsInPage(node)) { + NodeCodec.writeTo(page, node) + null + } else { + splitLeaf(page, node) + } + } + is BTreeNode.Internal -> { + val idx = childIndexFor(node.keys, key) + val childSplit = insertRecursive(node.children[idx], key, value) + if (childSplit == null) { + result = null + } else { + node.keys.add(idx, childSplit.promotedKey) + node.children.add(idx + 1, childSplit.newPageId) + dirty = true + result = if (NodeCodec.fitsInPage(node)) { + NodeCodec.writeTo(page, node) + null + } else { + splitInternal(page, node) + } + } + } + } + bufferPool.unpinPage(pageId, dirty) + return result + } + + private fun splitLeaf(page: Page, leaf: BTreeNode.Leaf): SplitResult { + val mid = leaf.keys.size / 2 + val rightKeys = ArrayList(leaf.keys.subList(mid, leaf.keys.size)) + val rightValues = ArrayList(leaf.values.subList(mid, leaf.values.size)) + val leftKeys = ArrayList(leaf.keys.subList(0, mid)) + val leftValues = ArrayList(leaf.values.subList(0, mid)) + val rightPage = bufferPool.newPage() + val rightNode = BTreeNode.Leaf(rightPage.id, rightKeys, rightValues, leaf.nextLeaf) + val leftNode = BTreeNode.Leaf(page.id, leftKeys, leftValues, rightPage.id) + NodeCodec.writeTo(page, leftNode) + NodeCodec.writeTo(rightPage, rightNode) + bufferPool.unpinPage(rightPage.id, true) + return SplitResult(rightKeys[0], rightPage.id) + } + + private fun splitInternal(page: Page, node: BTreeNode.Internal): SplitResult { + val mid = node.keys.size / 2 + val promotedKey = node.keys[mid] + val leftKeys = ArrayList(node.keys.subList(0, mid)) + val leftChildren = ArrayList(node.children.subList(0, mid + 1)) + val rightKeys = ArrayList(node.keys.subList(mid + 1, node.keys.size)) + val rightChildren = ArrayList(node.children.subList(mid + 1, node.children.size)) + val rightPage = bufferPool.newPage() + val rightNode = BTreeNode.Internal(rightPage.id, rightKeys, rightChildren) + val leftNode = BTreeNode.Internal(page.id, leftKeys, leftChildren) + NodeCodec.writeTo(page, leftNode) + NodeCodec.writeTo(rightPage, rightNode) + bufferPool.unpinPage(rightPage.id, true) + return SplitResult(promotedKey, rightPage.id) + } + + private fun deleteRecursive(pageId: Long, key: ByteArray): Boolean { + val (page, node) = fetch(pageId) + val found: Boolean + when (node) { + is BTreeNode.Leaf -> { + val idx = lowerBound(node.keys, key) + found = idx < node.keys.size && KeyCodec.compare(node.keys[idx], key) == 0 + if (found) { + node.keys.removeAt(idx) + node.values.removeAt(idx) + NodeCodec.writeTo(page, node) + } + bufferPool.unpinPage(pageId, found) + } + is BTreeNode.Internal -> { + val idx = childIndexFor(node.keys, key) + val childId = node.children[idx] + found = deleteRecursive(childId, key) + if (found) { + val (_, childNode) = fetch(childId) + val underflow = isUnderflowNode(childNode) + bufferPool.unpinPage(childId, false) + if (underflow) { + rebalanceChild(node, idx) + NodeCodec.writeTo(page, node) + } + } + bufferPool.unpinPage(pageId, found) + } + } + return found + } + + private fun rebalanceChild(parent: BTreeNode.Internal, idx: Int) { + val leftIdx = idx - 1 + val rightIdx = idx + 1 + + if (leftIdx >= 0) { + val (leftPage, leftNode) = fetch(parent.children[leftIdx]) + if (canLend(leftNode)) { + borrowFromLeft(parent, idx, leftPage, leftNode) + bufferPool.unpinPage(leftNode.pageId, true) + return + } + bufferPool.unpinPage(leftNode.pageId, false) + } + if (rightIdx < parent.children.size) { + val (rightPage, rightNode) = fetch(parent.children[rightIdx]) + if (canLend(rightNode)) { + borrowFromRight(parent, idx, rightPage, rightNode) + bufferPool.unpinPage(rightNode.pageId, true) + return + } + bufferPool.unpinPage(rightNode.pageId, false) + } + if (leftIdx >= 0) mergeChildren(parent, leftIdx) else mergeChildren(parent, idx) + } + + private fun canLend(node: BTreeNode): Boolean = when (node) { + is BTreeNode.Leaf -> node.keys.size > 1 + is BTreeNode.Internal -> node.keys.size > 1 + } + + private fun borrowFromLeft(parent: BTreeNode.Internal, idx: Int, leftPage: Page, leftNode: BTreeNode) { + val (childPage, childNode) = fetch(parent.children[idx]) + when { + leftNode is BTreeNode.Leaf && childNode is BTreeNode.Leaf -> { + val lastIdx = leftNode.keys.size - 1 + val borrowedKey = leftNode.keys.removeAt(lastIdx) + val borrowedValue = leftNode.values.removeAt(lastIdx) + childNode.keys.add(0, borrowedKey) + childNode.values.add(0, borrowedValue) + parent.keys[idx - 1] = childNode.keys[0] + } + leftNode is BTreeNode.Internal && childNode is BTreeNode.Internal -> { + val movedChild = leftNode.children.removeAt(leftNode.children.size - 1) + val separatorDown = leftNode.keys.removeAt(leftNode.keys.size - 1) + childNode.children.add(0, movedChild) + childNode.keys.add(0, parent.keys[idx - 1]) + parent.keys[idx - 1] = separatorDown + } + else -> throw BTreeCorruptionException("sibling type mismatch during redistribution") + } + NodeCodec.writeTo(leftPage, leftNode) + NodeCodec.writeTo(childPage, childNode) + bufferPool.unpinPage(childNode.pageId, true) + } + + private fun borrowFromRight(parent: BTreeNode.Internal, idx: Int, rightPage: Page, rightNode: BTreeNode) { + val (childPage, childNode) = fetch(parent.children[idx]) + when { + rightNode is BTreeNode.Leaf && childNode is BTreeNode.Leaf -> { + val borrowedKey = rightNode.keys.removeAt(0) + val borrowedValue = rightNode.values.removeAt(0) + childNode.keys.add(borrowedKey) + childNode.values.add(borrowedValue) + parent.keys[idx] = rightNode.keys[0] + } + rightNode is BTreeNode.Internal && childNode is BTreeNode.Internal -> { + val movedChild = rightNode.children.removeAt(0) + val separatorDown = rightNode.keys.removeAt(0) + childNode.children.add(movedChild) + childNode.keys.add(parent.keys[idx]) + parent.keys[idx] = separatorDown + } + else -> throw BTreeCorruptionException("sibling type mismatch during redistribution") + } + NodeCodec.writeTo(rightPage, rightNode) + NodeCodec.writeTo(childPage, childNode) + bufferPool.unpinPage(childNode.pageId, true) + } + + private fun mergeChildren(parent: BTreeNode.Internal, leftIdx: Int) { + val leftId = parent.children[leftIdx] + val rightId = parent.children[leftIdx + 1] + val (leftPage, leftNode) = fetch(leftId) + val (_, rightNode) = fetch(rightId) + when { + leftNode is BTreeNode.Leaf && rightNode is BTreeNode.Leaf -> { + leftNode.keys.addAll(rightNode.keys) + leftNode.values.addAll(rightNode.values) + leftNode.nextLeaf = rightNode.nextLeaf + } + leftNode is BTreeNode.Internal && rightNode is BTreeNode.Internal -> { + leftNode.keys.add(parent.keys[leftIdx]) + leftNode.keys.addAll(rightNode.keys) + leftNode.children.addAll(rightNode.children) + } + else -> throw BTreeCorruptionException("sibling type mismatch during merge") + } + NodeCodec.writeTo(leftPage, leftNode) + bufferPool.unpinPage(leftId, true) + bufferPool.unpinPage(rightId, false) + bufferPool.deletePage(rightId) + parent.keys.removeAt(leftIdx) + parent.children.removeAt(leftIdx + 1) + } + + private fun isUnderflowNode(node: BTreeNode): Boolean = NodeCodec.serialize(node).size < UNDERFLOW_THRESHOLD + + private fun fetch(pageId: Long): Pair { + val page = bufferPool.fetchPage(pageId) + return page to NodeCodec.readFrom(page) + } + + private fun lowerBound(keys: List, key: ByteArray): Int { + var lo = 0 + var hi = keys.size + while (lo < hi) { + val mid = (lo + hi) ushr 1 + if (KeyCodec.compare(keys[mid], key) < 0) lo = mid + 1 else hi = mid + } + return lo + } + + private fun childIndexFor(keys: List, key: ByteArray): Int { + var lo = 0 + var hi = keys.size + while (lo < hi) { + val mid = (lo + hi) ushr 1 + if (KeyCodec.compare(keys[mid], key) <= 0) lo = mid + 1 else hi = mid + } + return lo + } +} diff --git a/src/main/kotlin/kestrel/store/btree/BTreeNode.kt b/src/main/kotlin/kestrel/store/btree/BTreeNode.kt new file mode 100644 index 0000000..17ae210 --- /dev/null +++ b/src/main/kotlin/kestrel/store/btree/BTreeNode.kt @@ -0,0 +1,107 @@ +package kestrel.store.btree + +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter +import kestrel.store.codec.CodecException +import kestrel.store.page.Page + +sealed class BTreeNode { + abstract val pageId: Long + + class Leaf( + override val pageId: Long, + val keys: MutableList, + val values: MutableList, + var nextLeaf: Long + ) : BTreeNode() + + /** `children.size` is always `keys.size + 1`: `children[0]` holds keys below `keys[0]`. */ + class Internal( + override val pageId: Long, + val keys: MutableList, + val children: MutableList + ) : BTreeNode() +} + +/** Serializes [BTreeNode]s onto/from a fixed-size [Page]. */ +object NodeCodec { + private const val TYPE_LEAF = 0 + private const val TYPE_INTERNAL = 1 + const val NO_PAGE = -1L + + fun serialize(node: BTreeNode): ByteArray { + val writer = ByteWriter(Page.PAGE_SIZE) + when (node) { + is BTreeNode.Leaf -> { + writer.writeByte(TYPE_LEAF) + writeCount(writer, node.keys.size) + writer.writeFixedLong(node.nextLeaf) + for (i in node.keys.indices) { + writer.writeLengthPrefixed(node.keys[i]) + writer.writeFixedLong(node.values[i]) + } + } + is BTreeNode.Internal -> { + writer.writeByte(TYPE_INTERNAL) + writeCount(writer, node.keys.size) + writer.writeFixedLong(node.children[0]) + for (i in node.keys.indices) { + writer.writeLengthPrefixed(node.keys[i]) + writer.writeFixedLong(node.children[i + 1]) + } + } + } + return writer.toByteArray() + } + + fun writeTo(page: Page, node: BTreeNode) { + val bytes = serialize(node) + if (bytes.size > Page.PAGE_SIZE) { + throw CodecException("btree node ${node.pageId} serializes to ${bytes.size} bytes, exceeds page size") + } + page.data.fill(0) + System.arraycopy(bytes, 0, page.data, 0, bytes.size) + page.isDirty = true + } + + fun deserialize(pageId: Long, data: ByteArray): BTreeNode { + val reader = ByteReader(data) + val type = reader.readByte() + val count = readCount(reader) + return when (type) { + TYPE_LEAF -> { + val nextLeaf = reader.readFixedLong() + val keys = ArrayList(count) + val values = ArrayList(count) + repeat(count) { + keys.add(reader.readLengthPrefixed()) + values.add(reader.readFixedLong()) + } + BTreeNode.Leaf(pageId, keys, values, nextLeaf) + } + TYPE_INTERNAL -> { + val firstChild = reader.readFixedLong() + val keys = ArrayList(count) + val children = ArrayList(count + 1) + children.add(firstChild) + repeat(count) { + keys.add(reader.readLengthPrefixed()) + children.add(reader.readFixedLong()) + } + BTreeNode.Internal(pageId, keys, children) + } + else -> throw CodecException("unknown btree node type $type") + } + } + + fun readFrom(page: Page): BTreeNode = deserialize(page.id, page.data) + + fun fitsInPage(node: BTreeNode): Boolean = serialize(node).size <= Page.PAGE_SIZE + + private fun writeCount(writer: ByteWriter, count: Int) { + writer.writeByte((count ushr 8) and 0xFF) + writer.writeByte(count and 0xFF) + } + + private fun readCount(reader: ByteReader): Int = (reader.readByte() shl 8) or reader.readByte() +} diff --git a/src/main/kotlin/kestrel/store/codec/ByteCursor.kt b/src/main/kotlin/kestrel/store/codec/ByteCursor.kt new file mode 100644 index 0000000..0aa17aa --- /dev/null +++ b/src/main/kotlin/kestrel/store/codec/ByteCursor.kt @@ -0,0 +1,113 @@ +package kestrel.store.codec + +import kestrel.KestrelException + +class CodecException(message: String) : KestrelException(message) + +/** Growable little-endian-free byte sink. Multi-byte integers are written big-endian. */ +class ByteWriter(initialCapacity: Int = 32) { + private var buffer = ByteArray(initialCapacity) + private var size = 0 + + val length: Int get() = size + + private fun ensure(extra: Int) { + if (size + extra <= buffer.size) return + var newCapacity = buffer.size * 2 + while (newCapacity < size + extra) newCapacity *= 2 + buffer = buffer.copyOf(newCapacity) + } + + fun writeByte(value: Int): ByteWriter { + ensure(1) + buffer[size++] = value.toByte() + return this + } + + fun writeBytes(bytes: ByteArray): ByteWriter { + ensure(bytes.size) + System.arraycopy(bytes, 0, buffer, size, bytes.size) + size += bytes.size + return this + } + + fun writeFixedLong(value: Long): ByteWriter { + ensure(8) + for (i in 7 downTo 0) { + buffer[size++] = ((value ushr (i * 8)) and 0xFF).toByte() + } + return this + } + + fun writeFixedDouble(value: Double): ByteWriter = writeFixedLong(java.lang.Double.doubleToLongBits(value)) + + /** Unsigned LEB128 varint. */ + fun writeVarInt(value: Long): ByteWriter { + var remaining = value + while (true) { + val byte = (remaining and 0x7F).toInt() + remaining = remaining ushr 7 + if (remaining == 0L) { + writeByte(byte) + return this + } + writeByte(byte or 0x80) + } + } + + /** Zigzag-encoded signed varint, safe for negative numbers. */ + fun writeZigZag(value: Long): ByteWriter = writeVarInt((value shl 1) xor (value shr 63)) + + fun writeLengthPrefixed(bytes: ByteArray): ByteWriter { + writeVarInt(bytes.size.toLong()) + writeBytes(bytes) + return this + } + + fun toByteArray(): ByteArray = buffer.copyOf(size) +} + +class ByteReader(private val data: ByteArray, private var position: Int = 0) { + val remaining: Int get() = data.size - position + + fun hasRemaining(): Boolean = position < data.size + + fun readByte(): Int { + if (position >= data.size) throw CodecException("unexpected end of buffer") + return data[position++].toInt() and 0xFF + } + + fun readBytes(count: Int): ByteArray { + if (position + count > data.size) throw CodecException("unexpected end of buffer") + val slice = data.copyOfRange(position, position + count) + position += count + return slice + } + + fun readFixedLong(): Long { + var result = 0L + repeat(8) { result = (result shl 8) or readByte().toLong() } + return result + } + + fun readFixedDouble(): Double = java.lang.Double.longBitsToDouble(readFixedLong()) + + fun readVarInt(): Long { + var result = 0L + var shift = 0 + while (true) { + val byte = readByte() + result = result or ((byte.toLong() and 0x7F) shl shift) + if (byte and 0x80 == 0) return result + shift += 7 + if (shift > 63) throw CodecException("varint too long") + } + } + + fun readZigZag(): Long { + val encoded = readVarInt() + return (encoded ushr 1) xor -(encoded and 1) + } + + fun readLengthPrefixed(): ByteArray = readBytes(readVarInt().toInt()) +} diff --git a/src/main/kotlin/kestrel/store/codec/Crc32C.kt b/src/main/kotlin/kestrel/store/codec/Crc32C.kt new file mode 100644 index 0000000..3615b47 --- /dev/null +++ b/src/main/kotlin/kestrel/store/codec/Crc32C.kt @@ -0,0 +1,14 @@ +package kestrel.store.codec + +import java.util.zip.CRC32C + +/** Castagnoli CRC-32 checksums used to detect corruption of on-disk pages and records. */ +object Crc32C { + fun of(bytes: ByteArray): Int = of(bytes, 0, bytes.size) + + fun of(bytes: ByteArray, offset: Int, length: Int): Int { + val crc = CRC32C() + crc.update(bytes, offset, length) + return crc.value.toInt() + } +} diff --git a/src/main/kotlin/kestrel/store/codec/KeyCodec.kt b/src/main/kotlin/kestrel/store/codec/KeyCodec.kt new file mode 100644 index 0000000..89f81d3 --- /dev/null +++ b/src/main/kotlin/kestrel/store/codec/KeyCodec.kt @@ -0,0 +1,214 @@ +package kestrel.store.codec + +import java.math.BigDecimal +import java.math.BigInteger +import java.time.LocalDate +import kestrel.storage.Value + +/** + * Order-preserving byte encoding for index keys: unsigned lexicographic + * comparison of the encoded bytes agrees with SQL value ordering, so a + * B+Tree can use plain `ByteArray` comparison instead of decoding values. + * + * Numeric values sort by type tag first, then by magnitude: same-type + * comparisons (INTEGER-INTEGER, REAL-REAL, DECIMAL-DECIMAL) are exact, but a + * mixed INTEGER/REAL/DECIMAL comparison is ordered by tag rather than by + * numeric value. Index columns are schema-typed in practice, so this does not + * affect single-column indexes; it is a deliberate scope cut versus the full + * cross-type semantics of [kestrel.storage.ValueOrdering]. + * + * DECIMAL uses a normalized "scientific notation" encoding: a sign byte, an + * offset-binary exponent (so larger magnitude sorts later), then the + * significant digits (trailing zeros stripped, one mapped byte per digit, + * terminated by a byte no digit can produce). Two decimals compare equal + * under this scheme exactly when [BigDecimal.compareTo] says they are equal + * (`1.5` and `1.50` included), and it holds arbitrary precision - unlike + * REAL's key encoding this never rounds through a `Double`. Negative values + * bit-complement the whole magnitude encoding, which reverses byte-order + * comparison so a more-negative number still sorts first. + */ +object KeyCodec { + private const val TAG_NULL = 0 + private const val TAG_BOOLEAN = 1 + private const val TAG_INTEGER = 2 + private const val TAG_REAL = 3 + private const val TAG_TEXT = 4 + private const val TAG_DECIMAL = 5 + private const val TAG_DATE = 6 + + private const val TEXT_ESCAPE_NUL = 0xFF + private const val TEXT_TERMINATOR_HIGH = 0x00 + private const val TEXT_TERMINATOR_LOW = 0x00 + + private const val DECIMAL_SIGN_NEGATIVE = 0 + private const val DECIMAL_SIGN_ZERO = 1 + private const val DECIMAL_SIGN_POSITIVE = 2 + private const val DECIMAL_DIGIT_TERMINATOR = 0 + + fun encodeKey(values: List): ByteArray { + val writer = ByteWriter() + for (value in values) encodeInto(writer, value) + return writer.toByteArray() + } + + fun encode(value: Value): ByteArray { + val writer = ByteWriter() + encodeInto(writer, value) + return writer.toByteArray() + } + + fun decodeKey(bytes: ByteArray): List { + val reader = ByteReader(bytes) + val result = ArrayList() + while (reader.hasRemaining()) { + result.add(decodeOne(reader)) + } + return result + } + + /** Unsigned lexicographic comparison, matching how the bytes are compared on disk. */ + fun compare(a: ByteArray, b: ByteArray): Int { + val n = minOf(a.size, b.size) + for (i in 0 until n) { + val diff = (a[i].toInt() and 0xFF) - (b[i].toInt() and 0xFF) + if (diff != 0) return diff + } + return a.size - b.size + } + + private fun encodeInto(writer: ByteWriter, value: Value) { + when (value) { + Value.NullValue -> writer.writeByte(TAG_NULL) + is Value.BooleanValue -> { + writer.writeByte(TAG_BOOLEAN) + writer.writeByte(if (value.value) 1 else 0) + } + is Value.IntegerValue -> { + writer.writeByte(TAG_INTEGER) + writer.writeFixedLong(value.value xor Long.MIN_VALUE) + } + is Value.RealValue -> { + writer.writeByte(TAG_REAL) + writer.writeFixedLong(encodeOrderedDouble(value.value)) + } + is Value.TextValue -> { + writer.writeByte(TAG_TEXT) + writeEscapedText(writer, value.value) + } + is Value.DecimalValue -> { + writer.writeByte(TAG_DECIMAL) + writeDecimalMagnitudeSign(writer, value.value) + } + is Value.DateValue -> { + writer.writeByte(TAG_DATE) + writer.writeFixedLong(value.value.toEpochDay() xor Long.MIN_VALUE) + } + } + } + + private fun decodeOne(reader: ByteReader): Value = when (val tag = reader.readByte()) { + TAG_NULL -> Value.NullValue + TAG_BOOLEAN -> Value.BooleanValue(reader.readByte() != 0) + TAG_INTEGER -> Value.IntegerValue(reader.readFixedLong() xor Long.MIN_VALUE) + TAG_REAL -> Value.RealValue(decodeOrderedDouble(reader.readFixedLong())) + TAG_TEXT -> Value.TextValue(readEscapedText(reader)) + TAG_DECIMAL -> Value.DecimalValue(readDecimalMagnitudeSign(reader)) + TAG_DATE -> Value.DateValue(LocalDate.ofEpochDay(reader.readFixedLong() xor Long.MIN_VALUE)) + else -> throw CodecException("unknown key tag $tag") + } + + private fun writeDecimalMagnitudeSign(writer: ByteWriter, decimal: BigDecimal) { + when { + decimal.signum() == 0 -> writer.writeByte(DECIMAL_SIGN_ZERO) + decimal.signum() > 0 -> { + writer.writeByte(DECIMAL_SIGN_POSITIVE) + writeDecimalMagnitude(writer, decimal, invert = false) + } + else -> { + writer.writeByte(DECIMAL_SIGN_NEGATIVE) + writeDecimalMagnitude(writer, decimal.negate(), invert = true) + } + } + } + + private fun writeDecimalMagnitude(writer: ByteWriter, magnitude: BigDecimal, invert: Boolean) { + val canonical = magnitude.stripTrailingZeros() + val digits = canonical.unscaledValue().toString() + val exponent = (digits.length - canonical.scale()).toLong() + val body = ByteWriter() + body.writeFixedLong(exponent xor Long.MIN_VALUE) + for (digit in digits) body.writeByte((digit - '0') + 1) + body.writeByte(DECIMAL_DIGIT_TERMINATOR) + val bytes = body.toByteArray() + writer.writeBytes(if (invert) ByteArray(bytes.size) { (bytes[it].toInt() xor 0xFF).toByte() } else bytes) + } + + private fun readDecimalMagnitudeSign(reader: ByteReader): BigDecimal = when (val sign = reader.readByte()) { + DECIMAL_SIGN_ZERO -> BigDecimal.ZERO + DECIMAL_SIGN_POSITIVE -> readDecimalMagnitude(reader, invert = false) + DECIMAL_SIGN_NEGATIVE -> readDecimalMagnitude(reader, invert = true).negate() + else -> throw CodecException("unknown decimal sign tag $sign") + } + + private fun readDecimalMagnitude(reader: ByteReader, invert: Boolean): BigDecimal { + fun nextByte(): Int { + val raw = reader.readByte() + return if (invert) raw xor 0xFF else raw + } + var exponentBits = 0L + repeat(8) { exponentBits = (exponentBits shl 8) or (nextByte().toLong() and 0xFF) } + val exponent = exponentBits xor Long.MIN_VALUE + val digits = StringBuilder() + while (true) { + val next = nextByte() + if (next == DECIMAL_DIGIT_TERMINATOR) break + digits.append('0' + (next - 1)) + } + if (digits.isEmpty()) return BigDecimal.ZERO + val unscaled = BigInteger(digits.toString()) + val scale = (digits.length - exponent).toInt() + return BigDecimal(unscaled, scale) + } + + private fun encodeOrderedDouble(value: Double): Long { + val bits = java.lang.Double.doubleToLongBits(value) + return if (bits < 0) bits.inv() else bits xor Long.MIN_VALUE + } + + private fun decodeOrderedDouble(transformed: Long): Double { + val bits = if (transformed < 0) transformed xor Long.MIN_VALUE else transformed.inv() + return java.lang.Double.longBitsToDouble(bits) + } + + private fun writeEscapedText(writer: ByteWriter, text: String) { + for (byte in text.toByteArray(Charsets.UTF_8)) { + val unsigned = byte.toInt() and 0xFF + if (unsigned == 0x00) { + writer.writeByte(0x00) + writer.writeByte(TEXT_ESCAPE_NUL) + } else { + writer.writeByte(unsigned) + } + } + writer.writeByte(TEXT_TERMINATOR_HIGH) + writer.writeByte(TEXT_TERMINATOR_LOW) + } + + private fun readEscapedText(reader: ByteReader): String { + val out = ByteWriter() + while (true) { + val byte = reader.readByte() + if (byte != 0x00) { + out.writeByte(byte) + continue + } + val next = reader.readByte() + if (next == TEXT_ESCAPE_NUL) { + out.writeByte(0x00) + } else { + break + } + } + return String(out.toByteArray(), Charsets.UTF_8) + } +} diff --git a/src/main/kotlin/kestrel/store/codec/TupleCodec.kt b/src/main/kotlin/kestrel/store/codec/TupleCodec.kt new file mode 100644 index 0000000..4a3f503 --- /dev/null +++ b/src/main/kotlin/kestrel/store/codec/TupleCodec.kt @@ -0,0 +1,103 @@ +package kestrel.store.codec + +import java.math.BigDecimal +import java.math.BigInteger +import java.time.LocalDate +import kestrel.storage.Value + +/** + * Encodes a row (list of [Value]) as a self-describing byte payload: a leading + * null bitmap followed by a type-tagged, variable-length payload for every + * non-null column. Used to store row bodies on disk pages, independent of the + * order-preserving key encoding in [KeyCodec]. + */ +object TupleCodec { + private const val TAG_BOOLEAN = 1 + private const val TAG_INTEGER = 2 + private const val TAG_REAL = 3 + private const val TAG_TEXT = 4 + private const val TAG_DECIMAL = 5 + private const val TAG_DATE = 6 + + fun encode(values: List): ByteArray { + val writer = ByteWriter() + writer.writeVarInt(values.size.toLong()) + writer.writeBytes(nullBitmap(values)) + for (value in values) { + if (value.isNull) continue + when (value) { + is Value.BooleanValue -> { + writer.writeByte(TAG_BOOLEAN) + writer.writeByte(if (value.value) 1 else 0) + } + is Value.IntegerValue -> { + writer.writeByte(TAG_INTEGER) + writer.writeZigZag(value.value) + } + is Value.RealValue -> { + writer.writeByte(TAG_REAL) + writer.writeFixedDouble(value.value) + } + is Value.TextValue -> { + writer.writeByte(TAG_TEXT) + writer.writeLengthPrefixed(value.value.toByteArray(Charsets.UTF_8)) + } + is Value.DecimalValue -> { + writer.writeByte(TAG_DECIMAL) + writer.writeZigZag(value.value.scale().toLong()) + writer.writeLengthPrefixed(value.value.unscaledValue().toByteArray()) + } + is Value.DateValue -> { + writer.writeByte(TAG_DATE) + writer.writeZigZag(value.value.toEpochDay()) + } + Value.NullValue -> Unit + } + } + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): List { + val reader = ByteReader(bytes) + val columnCount = reader.readVarInt().toInt() + val bitmapBytes = reader.readBytes(bitmapSize(columnCount)) + val result = ArrayList(columnCount) + for (index in 0 until columnCount) { + if (isNullBit(bitmapBytes, index)) { + result.add(Value.NullValue) + continue + } + result.add( + when (val tag = reader.readByte()) { + TAG_BOOLEAN -> Value.BooleanValue(reader.readByte() != 0) + TAG_INTEGER -> Value.IntegerValue(reader.readZigZag()) + TAG_REAL -> Value.RealValue(reader.readFixedDouble()) + TAG_TEXT -> Value.TextValue(String(reader.readLengthPrefixed(), Charsets.UTF_8)) + TAG_DECIMAL -> { + val scale = reader.readZigZag().toInt() + val unscaled = BigInteger(reader.readLengthPrefixed()) + Value.DecimalValue(BigDecimal(unscaled, scale)) + } + TAG_DATE -> Value.DateValue(LocalDate.ofEpochDay(reader.readZigZag())) + else -> throw CodecException("unknown tuple tag $tag") + } + ) + } + return result + } + + private fun bitmapSize(columnCount: Int): Int = (columnCount + 7) / 8 + + private fun nullBitmap(values: List): ByteArray { + val bitmap = ByteArray(bitmapSize(values.size)) + for (index in values.indices) { + if (values[index].isNull) { + bitmap[index / 8] = (bitmap[index / 8].toInt() or (1 shl (index % 8))).toByte() + } + } + return bitmap + } + + private fun isNullBit(bitmap: ByteArray, index: Int): Boolean = + (bitmap[index / 8].toInt() shr (index % 8)) and 1 == 1 +} diff --git a/src/main/kotlin/kestrel/store/engine/LoggingPageStore.kt b/src/main/kotlin/kestrel/store/engine/LoggingPageStore.kt new file mode 100644 index 0000000..21ce720 --- /dev/null +++ b/src/main/kotlin/kestrel/store/engine/LoggingPageStore.kt @@ -0,0 +1,71 @@ +package kestrel.store.engine + +import kestrel.store.page.Page +import kestrel.store.page.PageStore +import kestrel.store.txn.TransactionManager + +/** + * A [PageStore] decorator that watches every page a single logical operation + * touches and turns whatever ends up dirty into [TransactionManager.logUpdate] + * calls, without [kestrel.store.heap.HeapFile] or [kestrel.store.btree.BPlusTree] + * knowing they are being logged - they were written against [PageStore] with + * no notion of a transaction. + * + * One instance is meant to wrap exactly one logical operation (an insert, a + * delete, ...): the first [fetchPage] or [newPage] for a given page id + * captures its "before" image (an all-zero page for a freshly allocated one, + * since that is the state [kestrel.store.wal.RecoveryManager] would undo + * back to), and [finish] compares that snapshot against whatever the + * delegate's live [Page] holds once the operation is done, logging one WAL + * record per page whose bytes actually changed. [onTouch] is called only for + * those actually-logged pages, once [finish] has decided which ones they + * are: a page a caller merely read must never be reported, because its + * cached bytes can be dirty with an earlier, already-committed transaction's + * write that just has not been flushed yet, and [kestrel.store.wal.RecoveryManager] + * never touches such a page on disk during rollback, so invalidating it in + * the buffer pool would silently discard real data instead of undoing + * anything. + */ +class LoggingPageStore( + private val delegate: PageStore, + private val txnManager: TransactionManager, + private val txnId: Long, + private val onTouch: (Long) -> Unit = {} +) : PageStore { + private val liveByPageId = LinkedHashMap() + private val beforeImages = HashMap() + + override fun fetchPage(pageId: Long): Page { + val page = delegate.fetchPage(pageId) + track(pageId, page) + return page + } + + override fun newPage(): Page { + val page = delegate.newPage() + beforeImages[page.id] = ByteArray(Page.PAGE_SIZE) + track(page.id, page) + return page + } + + override fun unpinPage(pageId: Long, markDirty: Boolean) = delegate.unpinPage(pageId, markDirty) + + override fun deletePage(pageId: Long) = delegate.deletePage(pageId) + + /** Logs one update record per page whose bytes differ from when it was first touched. */ + fun finish() { + for ((pageId, page) in liveByPageId) { + val before = beforeImages.getValue(pageId) + val after = page.data + if (!before.contentEquals(after)) { + txnManager.logUpdate(txnId, pageId, before, after.copyOf()) + onTouch(pageId) + } + } + } + + private fun track(pageId: Long, page: Page) { + liveByPageId.putIfAbsent(pageId, page) + beforeImages.putIfAbsent(pageId, page.data.copyOf()) + } +} diff --git a/src/main/kotlin/kestrel/store/engine/StorageEngine.kt b/src/main/kotlin/kestrel/store/engine/StorageEngine.kt new file mode 100644 index 0000000..469c7b9 --- /dev/null +++ b/src/main/kotlin/kestrel/store/engine/StorageEngine.kt @@ -0,0 +1,168 @@ +package kestrel.store.engine + +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import kestrel.store.page.Page +import kestrel.store.txn.TransactionManager +import kestrel.store.wal.RecoveryManager +import kestrel.store.wal.WriteAheadLog +import java.io.File + +/** + * Top-level facade tying the disk manager, buffer pool, write-ahead log and + * transaction manager into one crash-safe engine: [open] always runs + * [RecoveryManager.recover] first, so any table or index reopened through it + * sees the post-recovery, consistent state whether the previous process + * committed cleanly or crashed mid-transaction. + * + * [commit] and [abort] are the only way transactions started here should be + * resolved: besides delegating to [TransactionManager], they keep every + * [TransactionalIndex]'s in-memory root pointer and the [TouchedPageTracker] + * in sync, which a caller driving [TransactionManager] directly would have + * to reproduce by hand. + */ +class StorageEngine private constructor( + private val diskManager: DiskManager, + private val bufferPool: BufferPool, + private val wal: WriteAheadLog, + private val txnManager: TransactionManager +) : AutoCloseable { + companion object { + fun open(dataFile: File, walFile: File, poolCapacity: Int = 64): StorageEngine { + val diskManager = DiskManager(dataFile) + val wal = WriteAheadLog(walFile) + RecoveryManager(wal, diskManager).recover() + val bufferPool = BufferPool(diskManager, poolCapacity) + val txnManager = TransactionManager(wal, diskManager) + return StorageEngine(diskManager, bufferPool, wal, txnManager) + } + } + + private val tracker = TouchedPageTracker() + private val indexes = mutableListOf() + private var nextResourceId = 0L + private val savepointLsnsByTxn = HashMap>() + + fun beginTransaction(): Long = txnManager.begin() + + fun commit(txnId: Long) { + txnManager.commit(txnId) + for (index in indexes) index.forgetTransaction(txnId) + tracker.forget(txnId) + forgetSavepoints(txnId) + } + + fun abort(txnId: Long) { + txnManager.abort(txnId) + for (index in indexes) index.rollback(txnId) + for (pageId in tracker.forget(txnId)) bufferPool.invalidate(pageId) + forgetSavepoints(txnId) + } + + /** + * Establishes a savepoint on [txnId]: returns a token [rollbackToSavepoint] can later undo back to, + * capturing every open index's current root pointer alongside the write-ahead log position so a + * later rollback can undo B+Tree structural changes (splits, merges, shrinks) too, not just page + * bytes. + */ + fun savepoint(txnId: Long): Long { + val lsn = txnManager.currentLsn(txnId) + for (index in indexes) index.snapshotRootAt(lsn) + savepointLsnsByTxn.getOrPut(txnId) { mutableListOf() }.add(lsn) + return lsn + } + + /** + * Undoes every write [txnId] made after [savepointLsn] (from [savepoint]) without resolving the + * transaction - it remains open, holds every lock it already had, and can be written to, committed, + * fully rolled back, or rolled back to another savepoint afterward. Unlike [abort], the savepoint + * itself is not forgotten: [restoreRootAt]-style rollback to the same savepoint again later is safe + * because [kestrel.store.wal.RecoveryManager.rollbackTo] rewrites the undo chain with compensation + * records rather than physically deleting anything. + */ + fun rollbackToSavepoint(txnId: Long, savepointLsn: Long) { + txnManager.rollbackTo(txnId, savepointLsn) + for (index in indexes) index.restoreRootAt(savepointLsn) + for (pageId in tracker.peek(txnId)) bufferPool.invalidate(pageId) + } + + /** Drops a savepoint that can no longer be rolled back to (RELEASE, or superseded by an earlier rollback). */ + fun discardSavepoint(txnId: Long, savepointLsn: Long) { + for (index in indexes) index.discardSavepoint(savepointLsn) + savepointLsnsByTxn[txnId]?.remove(savepointLsn) + } + + private fun forgetSavepoints(txnId: Long) { + val lsns = savepointLsnsByTxn.remove(txnId) ?: return + for (lsn in lsns) for (index in indexes) index.discardSavepoint(lsn) + } + + fun createHeap(): TransactionalHeap = TransactionalHeap.create(bufferPool, txnManager, tracker) + + fun openHeap(headerPageId: Long): TransactionalHeap = TransactionalHeap.open(bufferPool, txnManager, tracker, headerPageId) + + fun createIndex(): TransactionalIndex { + val index = TransactionalIndex.create(bufferPool, txnManager, tracker, allocateResourceId()) + indexes.add(index) + return index + } + + fun openIndex(rootPageId: Long): TransactionalIndex { + val index = TransactionalIndex.open(bufferPool, txnManager, tracker, allocateResourceId(), rootPageId) + indexes.add(index) + return index + } + + /** Writes every dirty cached page back to the data file without closing the engine. */ + fun flush() = bufferPool.flushAll() + + /** Total pages ever allocated in the backing data file, including ones now sitting on the free list. */ + fun allocatedPageCount(): Long = diskManager.allocatedPageCount() + + /** + * Allocates a page dedicated to content that is never covered by [beginTransaction]/[commit]/ + * [abort] - see [writeRawPage]'s own doc for why [kestrel.store.sql.DiskCatalog] needs that for + * a SEQUENCE's live counter. Never reused as a heap or index page: [DiskManager] hands out page + * ids from one shared counter, so a raw page can never collide with one [createHeap]/[createIndex] + * allocates. + */ + fun allocateRawPage(): Long = diskManager.allocatePage() + + /** + * Writes [bytes] (padded with zeros to a full page) straight to [diskManager], bypassing the + * buffer pool, write-ahead log and lock manager entirely - deliberately not participatory in + * [commit]/[abort] undo. [kestrel.store.sql.DiskCatalog] uses this exclusively for a SEQUENCE's + * NEXTVAL counter, whose whole point - like Postgres' own sequences - is to keep handing out + * values that never repeat even across an aborted transaction, so it must survive a ROLLBACK + * that would otherwise undo it if it were written through the normal transactional path. A page + * allocated by [allocateRawPage] is never fetched through [bufferPool] by anything else, so there + * is no cache to invalidate here the way [BufferPool.invalidate] does for [RecoveryManager]'s own + * direct writes. + */ + fun writeRawPage(pageId: Long, bytes: ByteArray) { + require(bytes.size <= Page.PAGE_SIZE) { "raw page payload of ${bytes.size} bytes exceeds the ${Page.PAGE_SIZE}-byte page size" } + val page = Page(pageId) + System.arraycopy(bytes, 0, page.data, 0, bytes.size) + diskManager.writePage(page) + } + + /** Reads back a page written by [writeRawPage], straight from [diskManager]. */ + fun readRawPage(pageId: Long): ByteArray = diskManager.readPage(pageId).data + + /** + * Frees a page allocated by [allocateRawPage] back to the shared [DiskManager] free list - used + * by `DROP SEQUENCE` once its counter page will never be read again. Bypasses the buffer pool + * like every other raw-page operation: nothing ever caches a raw page through [bufferPool], so + * there is no frame to invalidate the way [BufferPool.deletePage] would for a heap or index page. + */ + fun freeRawPage(pageId: Long) = diskManager.freePage(pageId) + + override fun close() { + bufferPool.flushAll() + wal.close() + diskManager.close() + } + + /** Negative so it never collides with a page id, which [DiskManager] hands out starting at zero. */ + private fun allocateResourceId(): Long = -(++nextResourceId) +} diff --git a/src/main/kotlin/kestrel/store/engine/TouchedPageTracker.kt b/src/main/kotlin/kestrel/store/engine/TouchedPageTracker.kt new file mode 100644 index 0000000..4bd9fc8 --- /dev/null +++ b/src/main/kotlin/kestrel/store/engine/TouchedPageTracker.kt @@ -0,0 +1,25 @@ +package kestrel.store.engine + +/** + * Remembers every page id a still-active transaction has touched across + * however many [TransactionalHeap] or [TransactionalIndex] operations it + * performs, so [StorageEngine.abort] can invalidate the buffer pool's cached + * copies of exactly those pages once the underlying rollback has restored + * their before-images on disk. + */ +class TouchedPageTracker { + private val pagesByTxn = HashMap>() + + fun record(txnId: Long, pageId: Long) { + pagesByTxn.getOrPut(txnId) { LinkedHashSet() }.add(pageId) + } + + fun forget(txnId: Long): Set = pagesByTxn.remove(txnId) ?: emptySet() + + /** + * Every page [txnId] has touched so far, without forgetting them - used after a savepoint + * rollback, which (unlike [forget]'s callers) does not resolve the transaction, so pages touched + * before the savepoint may still be written again later in the same transaction. + */ + fun peek(txnId: Long): Set = pagesByTxn[txnId] ?: emptySet() +} diff --git a/src/main/kotlin/kestrel/store/engine/TransactionalHeap.kt b/src/main/kotlin/kestrel/store/engine/TransactionalHeap.kt new file mode 100644 index 0000000..90cd1c9 --- /dev/null +++ b/src/main/kotlin/kestrel/store/engine/TransactionalHeap.kt @@ -0,0 +1,87 @@ +package kestrel.store.engine + +import kestrel.store.heap.HeapFile +import kestrel.store.heap.RecordId +import kestrel.store.page.PageStore +import kestrel.store.txn.TransactionManager + +/** + * A [HeapFile] made crash-safe and transactional: every write acquires an + * exclusive lock on [headerPageId] (the whole table is one lock resource - + * a deliberately coarse granularity, ahead of page-level locking) and is + * wrapped in a fresh [LoggingPageStore] per call so its page mutations reach + * the write-ahead log before [TransactionManager.commit] makes them durable. + * Reads take a shared lock on the same resource, so a reader never observes + * a partially applied write from another transaction under strict 2PL. + */ +class TransactionalHeap private constructor( + private val bufferPool: PageStore, + private val txnManager: TransactionManager, + private val tracker: TouchedPageTracker, + val headerPageId: Long +) { + companion object { + fun create(bufferPool: PageStore, txnManager: TransactionManager, tracker: TouchedPageTracker): TransactionalHeap { + val heap = HeapFile.create(bufferPool) + return TransactionalHeap(bufferPool, txnManager, tracker, heap.headerPageId) + } + + fun open( + bufferPool: PageStore, + txnManager: TransactionManager, + tracker: TouchedPageTracker, + headerPageId: Long + ): TransactionalHeap = TransactionalHeap(bufferPool, txnManager, tracker, headerPageId) + } + + fun insert(txnId: Long, bytes: ByteArray): RecordId { + txnManager.acquireExclusive(txnId, headerPageId) + val logging = loggingStoreFor(txnId) + val rid = HeapFile.open(logging, headerPageId).insert(bytes) + logging.finish() + return rid + } + + fun delete(txnId: Long, rid: RecordId) { + txnManager.acquireExclusive(txnId, headerPageId) + val logging = loggingStoreFor(txnId) + HeapFile.open(logging, headerPageId).delete(rid) + logging.finish() + } + + fun update(txnId: Long, rid: RecordId, bytes: ByteArray): RecordId { + txnManager.acquireExclusive(txnId, headerPageId) + val logging = loggingStoreFor(txnId) + val newRid = HeapFile.open(logging, headerPageId).update(rid, bytes) + logging.finish() + return newRid + } + + /** + * Frees every page this heap owns (header and every data page) back to the shared + * [kestrel.store.page.DiskManager] free list - used when the owning table is dropped, since + * nothing will ever open this [headerPageId] again. Not part of the crash-recoverable log the + * way [insert]/[delete]/[update] are: like [kestrel.store.btree.BPlusTree]'s own page frees on + * merge/root-shrink (see [kestrel.store.engine.LoggingPageStore.deletePage]), an abort after + * this runs cannot un-free the pages - acceptable because the sole caller ([DiskCatalog][kestrel.store.sql.DiskCatalog]'s + * `dropTable`) only calls this once the directory row removal in the same transaction has + * already succeeded, with nothing left afterward that could still fail. + */ + fun destroy(txnId: Long) { + txnManager.acquireExclusive(txnId, headerPageId) + for (pageId in HeapFile.open(bufferPool, headerPageId).allPageIds()) bufferPool.deletePage(pageId) + } + + fun get(txnId: Long, rid: RecordId): ByteArray { + txnManager.acquireShared(txnId, headerPageId) + return HeapFile.open(bufferPool, headerPageId).get(rid) + } + + fun scan(txnId: Long): List> { + txnManager.acquireShared(txnId, headerPageId) + return HeapFile.open(bufferPool, headerPageId).scan().toList() + } + + private fun loggingStoreFor(txnId: Long): LoggingPageStore = + LoggingPageStore(bufferPool, txnManager, txnId) { pageId -> tracker.record(txnId, pageId) } +} diff --git a/src/main/kotlin/kestrel/store/engine/TransactionalIndex.kt b/src/main/kotlin/kestrel/store/engine/TransactionalIndex.kt new file mode 100644 index 0000000..e3070b7 --- /dev/null +++ b/src/main/kotlin/kestrel/store/engine/TransactionalIndex.kt @@ -0,0 +1,137 @@ +package kestrel.store.engine + +import kestrel.store.btree.BPlusTree +import kestrel.store.page.PageStore +import kestrel.store.txn.TransactionManager + +/** + * A [BPlusTree] made crash-safe and transactional, following the same + * per-call [LoggingPageStore] pattern as [TransactionalHeap]. Locking uses + * [lockResourceId] rather than the tree's own root page id: [BPlusTree.delete] + * can shrink the tree and free its former root back to the shared page free + * list, so that id can later be reallocated to an unrelated page - using it + * as a lock resource would risk two unrelated structures serializing against + * each other by coincidence. [StorageEngine] hands out a disjoint id (see + * `nextResourceId`) precisely to avoid that. + * + * A root split or shrink also moves the tree's root pointer itself, which is + * plain in-memory engine state, not something [kestrel.store.wal.WriteAheadLog] + * records - rolling a transaction back restores page *bytes* but has no way + * to know it must also restore [rootPageId]. [rollback] does that: it + * remembers, on a transaction's first write, the root as of just before that + * transaction touched the tree, and restores it on abort. + */ +class TransactionalIndex private constructor( + private val bufferPool: PageStore, + private val txnManager: TransactionManager, + private val tracker: TouchedPageTracker, + private val lockResourceId: Long, + initialRootPageId: Long +) { + companion object { + fun create( + bufferPool: PageStore, + txnManager: TransactionManager, + tracker: TouchedPageTracker, + lockResourceId: Long + ): TransactionalIndex { + val tree = BPlusTree.create(bufferPool) + return TransactionalIndex(bufferPool, txnManager, tracker, lockResourceId, tree.rootPageId) + } + + fun open( + bufferPool: PageStore, + txnManager: TransactionManager, + tracker: TouchedPageTracker, + lockResourceId: Long, + rootPageId: Long + ): TransactionalIndex = TransactionalIndex(bufferPool, txnManager, tracker, lockResourceId, rootPageId) + } + + var rootPageId: Long = initialRootPageId + private set + + private val rootBeforeTxn = HashMap() + private val rootAtSavepoint = HashMap() + + fun insert(txnId: Long, key: ByteArray, value: Long) { + txnManager.acquireExclusive(txnId, lockResourceId) + rootBeforeTxn.putIfAbsent(txnId, rootPageId) + val logging = loggingStoreFor(txnId) + val tree = BPlusTree.open(logging, rootPageId) + tree.insert(key, value) + rootPageId = tree.rootPageId + logging.finish() + } + + fun delete(txnId: Long, key: ByteArray): Boolean { + txnManager.acquireExclusive(txnId, lockResourceId) + rootBeforeTxn.putIfAbsent(txnId, rootPageId) + val logging = loggingStoreFor(txnId) + val tree = BPlusTree.open(logging, rootPageId) + val found = tree.delete(key) + rootPageId = tree.rootPageId + logging.finish() + return found + } + + /** + * Frees every page this tree owns back to the shared [kestrel.store.page.DiskManager] free + * list - used when the owning index (or its whole table) is dropped. See + * [TransactionalHeap.destroy]'s own doc for why this is deliberately not undo-safe: the same + * accepted tradeoff [kestrel.store.btree.BPlusTree] already makes for a merge or root-shrink's + * own page frees. + */ + fun destroy(txnId: Long) { + txnManager.acquireExclusive(txnId, lockResourceId) + for (pageId in BPlusTree.open(bufferPool, rootPageId).allPageIds()) bufferPool.deletePage(pageId) + } + + fun search(txnId: Long, key: ByteArray): Long? { + txnManager.acquireShared(txnId, lockResourceId) + return BPlusTree.open(bufferPool, rootPageId).search(key) + } + + fun rangeScan( + txnId: Long, + from: ByteArray? = null, + to: ByteArray? = null, + toInclusive: Boolean = false + ): List> { + txnManager.acquireShared(txnId, lockResourceId) + return BPlusTree.open(bufferPool, rootPageId).rangeScan(from, to, toInclusive).asSequence().toList() + } + + /** Restores the root pointer as of before [txnId]'s first write. Called by [StorageEngine.abort]. */ + fun rollback(txnId: Long) { + rootBeforeTxn.remove(txnId)?.let { rootPageId = it } + } + + /** Drops the pending rollback snapshot once [txnId] is durably resolved. Called by [StorageEngine.commit]. */ + fun forgetTransaction(txnId: Long) { + rootBeforeTxn.remove(txnId) + } + + /** + * Snapshots the root pointer under [savepointLsn] (a transaction's [TransactionManager.currentLsn] + * at the moment a SAVEPOINT is established), so a later [restoreRootAt] can undo whatever root + * moves a split, merge or shrink made afterward - the same problem [rootBeforeTxn] solves for a + * whole transaction, scoped to a savepoint instead. Called by [StorageEngine.savepoint]. + */ + fun snapshotRootAt(savepointLsn: Long) { + rootAtSavepoint[savepointLsn] = rootPageId + } + + /** Restores the root pointer captured by [snapshotRootAt]. Called by [StorageEngine.rollbackToSavepoint]. */ + fun restoreRootAt(savepointLsn: Long) { + rootPageId = rootAtSavepoint.getValue(savepointLsn) + } + + /** Drops a snapshot once its savepoint can no longer be rolled back to. Called by [StorageEngine.discardSavepoint]. */ + fun discardSavepoint(savepointLsn: Long) { + rootAtSavepoint.remove(savepointLsn) + } + + private fun loggingStoreFor(txnId: Long): LoggingPageStore = + LoggingPageStore(bufferPool, txnManager, txnId) { pageId -> tracker.record(txnId, pageId) } +} diff --git a/src/main/kotlin/kestrel/store/heap/HeapFile.kt b/src/main/kotlin/kestrel/store/heap/HeapFile.kt new file mode 100644 index 0000000..184ccf4 --- /dev/null +++ b/src/main/kotlin/kestrel/store/heap/HeapFile.kt @@ -0,0 +1,139 @@ +package kestrel.store.heap + +import kestrel.KestrelException +import kestrel.store.page.Page +import kestrel.store.page.PageStore + +class HeapFileException(message: String) : KestrelException(message) + +/** + * A heap-organized collection of variable-length records spanning a singly + * linked chain of [HeapPage]s. The chain head lives in a dedicated header + * page whose id ([headerPageId]) callers persist to reopen the same heap + * later, mirroring how [kestrel.store.btree.BPlusTree] exposes its root page + * id. Insert walks the chain looking for a page with enough free space, + * newest first, before allocating and linking a fresh page; this reuses + * space freed by deletes ahead of growing the file. + */ +class HeapFile private constructor(private val bufferPool: PageStore, val headerPageId: Long) { + companion object { + fun create(bufferPool: PageStore): HeapFile { + val header = bufferPool.newPage() + writeFirstDataPage(header, HeapPage.NO_PAGE) + bufferPool.unpinPage(header.id, true) + return HeapFile(bufferPool, header.id) + } + + fun open(bufferPool: PageStore, headerPageId: Long): HeapFile = HeapFile(bufferPool, headerPageId) + + private fun writeFirstDataPage(header: Page, pageId: Long) { + for (i in 0 until 8) header.data[i] = (pageId ushr (8 * (7 - i))).toByte() + header.isDirty = true + } + + private fun readFirstDataPage(header: Page): Long { + var result = 0L + for (i in 0 until 8) result = (result shl 8) or (header.data[i].toLong() and 0xFF) + return result + } + } + + fun insert(bytes: ByteArray): RecordId { + if (bytes.size > HeapPage.MAX_RECORD_SIZE) { + throw HeapFileException("record of ${bytes.size} bytes exceeds max ${HeapPage.MAX_RECORD_SIZE}") + } + val header = bufferPool.fetchPage(headerPageId) + val firstPageId = readFirstDataPage(header) + var scanId = firstPageId + while (scanId != HeapPage.NO_PAGE) { + val page = bufferPool.fetchPage(scanId) + val slotId = HeapPage.insert(page, bytes) + if (slotId != null) { + bufferPool.unpinPage(scanId, true) + bufferPool.unpinPage(headerPageId, false) + return RecordId(scanId, slotId) + } + val next = HeapPage.nextPageId(page) + bufferPool.unpinPage(scanId, false) + scanId = next + } + val newPage = bufferPool.newPage() + HeapPage.init(newPage, firstPageId) + val slotId = HeapPage.insert(newPage, bytes) + ?: throw HeapFileException("record of ${bytes.size} bytes does not fit in an empty page") + bufferPool.unpinPage(newPage.id, true) + writeFirstDataPage(header, newPage.id) + bufferPool.unpinPage(headerPageId, true) + return RecordId(newPage.id, slotId) + } + + fun get(rid: RecordId): ByteArray { + val page = bufferPool.fetchPage(rid.pageId) + try { + return HeapPage.get(page, rid.slotId) + } finally { + bufferPool.unpinPage(rid.pageId, false) + } + } + + fun delete(rid: RecordId) { + val page = bufferPool.fetchPage(rid.pageId) + HeapPage.delete(page, rid.slotId) + bufferPool.unpinPage(rid.pageId, true) + } + + /** + * Replaces the record at [rid]. When [bytes] fits within the slot's + * current capacity it is overwritten in place and [rid] is returned + * unchanged; otherwise the old slot is deleted and the record is + * re-inserted, possibly moving to a different page, and the new + * [RecordId] is returned. Callers that index this heap must update any + * stored [RecordId] with the return value. + */ + fun update(rid: RecordId, bytes: ByteArray): RecordId { + val page = bufferPool.fetchPage(rid.pageId) + val capacity = HeapPage.lengthOf(page, rid.slotId) + if (bytes.size <= capacity) { + HeapPage.overwrite(page, rid.slotId, bytes) + bufferPool.unpinPage(rid.pageId, true) + return rid + } + HeapPage.delete(page, rid.slotId) + bufferPool.unpinPage(rid.pageId, true) + return insert(bytes) + } + + /** Every page belonging to this heap: [headerPageId] plus every page in the data-page chain. */ + fun allPageIds(): List { + val ids = mutableListOf(headerPageId) + val header = bufferPool.fetchPage(headerPageId) + var pageId = readFirstDataPage(header) + bufferPool.unpinPage(headerPageId, false) + while (pageId != HeapPage.NO_PAGE) { + val page = bufferPool.fetchPage(pageId) + val next = HeapPage.nextPageId(page) + bufferPool.unpinPage(pageId, false) + ids.add(pageId) + pageId = next + } + return ids + } + + /** Iterates every live record in page-chain order. Deleted slots are skipped. */ + fun scan(): Sequence> = sequence { + val header = bufferPool.fetchPage(headerPageId) + var pageId = readFirstDataPage(header) + bufferPool.unpinPage(headerPageId, false) + while (pageId != HeapPage.NO_PAGE) { + val page = bufferPool.fetchPage(pageId) + val count = HeapPage.slotCount(page) + val next = HeapPage.nextPageId(page) + val liveBytes = (0 until count) + .filterNot { HeapPage.isDeleted(page, it) } + .map { RecordId(pageId, it) to HeapPage.get(page, it) } + bufferPool.unpinPage(pageId, false) + for (entry in liveBytes) yield(entry) + pageId = next + } + } +} diff --git a/src/main/kotlin/kestrel/store/heap/HeapPage.kt b/src/main/kotlin/kestrel/store/heap/HeapPage.kt new file mode 100644 index 0000000..d39c3de --- /dev/null +++ b/src/main/kotlin/kestrel/store/heap/HeapPage.kt @@ -0,0 +1,159 @@ +package kestrel.store.heap + +import kestrel.KestrelException +import kestrel.store.page.Page + +class HeapPageException(message: String) : KestrelException(message) + +/** + * Slotted-page layout for heap-organized storage. + * + * ``` + * [ header (12B) | slot directory, grows down | free space | tuple bytes, grow up ] + * ``` + * + * The header holds the id of the next page in the heap's page chain, the + * slot count, and the offset where tuple bytes currently begin. Each slot + * directory entry is `[offset:u16][length:u16]`; a length of + * [TOMBSTONE_LENGTH] marks a deleted slot. Deleting a record only tombstones + * its slot so the [RecordId] stays stable; [compact] later reclaims the + * space by repacking live tuples against the end of the page. + */ +object HeapPage { + private const val HEADER_SIZE = 12 + private const val SLOT_SIZE = 4 + private const val TOMBSTONE_LENGTH = 0xFFFF + const val NO_PAGE = -1L + + val MAX_RECORD_SIZE: Int = Page.PAGE_SIZE - HEADER_SIZE - SLOT_SIZE + + fun init(page: Page, nextPageId: Long = NO_PAGE) { + page.data.fill(0) + writeLong(page, 0, nextPageId) + writeShort(page, 8, 0) + writeShort(page, 10, Page.PAGE_SIZE) + page.isDirty = true + } + + fun nextPageId(page: Page): Long = readLong(page, 0) + + fun setNextPageId(page: Page, pageId: Long) { + writeLong(page, 0, pageId) + page.isDirty = true + } + + fun slotCount(page: Page): Int = readShort(page, 8) + + fun freeSpaceOffset(page: Page): Int = readShort(page, 10) + + fun freeBytes(page: Page): Int = freeSpaceOffset(page) - directoryEnd(page) + + fun isDeleted(page: Page, slotId: Int): Boolean = lengthAt(page, slotId) == TOMBSTONE_LENGTH + + fun lengthOf(page: Page, slotId: Int): Int { + val length = lengthAt(page, slotId) + if (length == TOMBSTONE_LENGTH) throw HeapPageException("slot $slotId is deleted") + return length + } + + fun get(page: Page, slotId: Int): ByteArray { + val entry = slotEntryOffset(page, slotId) + val length = readShort(page, entry + 2) + if (length == TOMBSTONE_LENGTH) throw HeapPageException("slot $slotId is deleted") + val offset = readShort(page, entry) + return page.data.copyOfRange(offset, offset + length) + } + + /** Returns the new slot id, or null if [bytes] does not fit even after [compact]. */ + fun insert(page: Page, bytes: ByteArray): Int? { + require(bytes.size <= MAX_RECORD_SIZE) { "record of ${bytes.size} bytes exceeds page capacity" } + if (!fits(page, bytes.size)) { + compact(page) + if (!fits(page, bytes.size)) return null + } + val slotId = slotCount(page) + val newOffset = freeSpaceOffset(page) - bytes.size + System.arraycopy(bytes, 0, page.data, newOffset, bytes.size) + writeSlotEntry(page, slotId, newOffset, bytes.size) + writeShort(page, 8, slotId + 1) + writeShort(page, 10, newOffset) + page.isDirty = true + return slotId + } + + fun delete(page: Page, slotId: Int) { + val entry = slotEntryOffset(page, slotId) + if (readShort(page, entry + 2) == TOMBSTONE_LENGTH) throw HeapPageException("slot $slotId is already deleted") + writeShort(page, entry, 0) + writeShort(page, entry + 2, TOMBSTONE_LENGTH) + page.isDirty = true + } + + /** Overwrites a slot in place. [bytes] must be no larger than the slot's current length. */ + fun overwrite(page: Page, slotId: Int, bytes: ByteArray) { + val entry = slotEntryOffset(page, slotId) + val length = readShort(page, entry + 2) + if (length == TOMBSTONE_LENGTH) throw HeapPageException("slot $slotId is deleted") + require(bytes.size <= length) { "overwrite of ${bytes.size} bytes must not grow slot $slotId (capacity $length)" } + val offset = readShort(page, entry) + System.arraycopy(bytes, 0, page.data, offset, bytes.size) + writeShort(page, entry + 2, bytes.size) + page.isDirty = true + } + + /** Repacks live tuples against the end of the page, discarding tombstoned bytes. */ + fun compact(page: Page) { + val count = slotCount(page) + val live = ArrayList>(count) + for (slotId in 0 until count) { + val entry = slotEntryOffset(page, slotId) + val length = readShort(page, entry + 2) + if (length == TOMBSTONE_LENGTH) continue + val offset = readShort(page, entry) + live.add(slotId to page.data.copyOfRange(offset, offset + length)) + } + var writeOffset = Page.PAGE_SIZE + for ((slotId, bytes) in live) { + writeOffset -= bytes.size + System.arraycopy(bytes, 0, page.data, writeOffset, bytes.size) + writeSlotEntry(page, slotId, writeOffset, bytes.size) + } + writeShort(page, 10, writeOffset) + page.isDirty = true + } + + private fun fits(page: Page, size: Int): Boolean = directoryEnd(page) + SLOT_SIZE + size <= freeSpaceOffset(page) + + private fun directoryEnd(page: Page): Int = HEADER_SIZE + slotCount(page) * SLOT_SIZE + + private fun lengthAt(page: Page, slotId: Int): Int = readShort(page, slotEntryOffset(page, slotId) + 2) + + private fun slotEntryOffset(page: Page, slotId: Int): Int { + if (slotId !in 0 until slotCount(page)) throw HeapPageException("slot $slotId out of range") + return HEADER_SIZE + slotId * SLOT_SIZE + } + + private fun writeSlotEntry(page: Page, slotId: Int, offset: Int, length: Int) { + val entry = HEADER_SIZE + slotId * SLOT_SIZE + writeShort(page, entry, offset) + writeShort(page, entry + 2, length) + } + + private fun readShort(page: Page, at: Int): Int = + ((page.data[at].toInt() and 0xFF) shl 8) or (page.data[at + 1].toInt() and 0xFF) + + private fun writeShort(page: Page, at: Int, value: Int) { + page.data[at] = (value ushr 8).toByte() + page.data[at + 1] = value.toByte() + } + + private fun readLong(page: Page, at: Int): Long { + var result = 0L + for (i in 0 until 8) result = (result shl 8) or (page.data[at + i].toLong() and 0xFF) + return result + } + + private fun writeLong(page: Page, at: Int, value: Long) { + for (i in 0 until 8) page.data[at + i] = (value ushr (8 * (7 - i))).toByte() + } +} diff --git a/src/main/kotlin/kestrel/store/heap/RecordId.kt b/src/main/kotlin/kestrel/store/heap/RecordId.kt new file mode 100644 index 0000000..acbb44d --- /dev/null +++ b/src/main/kotlin/kestrel/store/heap/RecordId.kt @@ -0,0 +1,16 @@ +package kestrel.store.heap + +/** + * Identifies a record's slot within a heap page. Stable across compaction: + * only the slot directory entry moves, never the (pageId, slotId) pair, so a + * B+Tree entry can point at a [RecordId] without tracking page-internal + * offsets. + */ +data class RecordId(val pageId: Long, val slotId: Int) { + /** Packs into a single [Long] so it can be stored as a B+Tree leaf value. */ + fun encode(): Long = (pageId shl 16) or (slotId.toLong() and 0xFFFF) + + companion object { + fun decode(encoded: Long): RecordId = RecordId(encoded ushr 16, (encoded and 0xFFFF).toInt()) + } +} diff --git a/src/main/kotlin/kestrel/store/page/BufferPool.kt b/src/main/kotlin/kestrel/store/page/BufferPool.kt new file mode 100644 index 0000000..ce801af --- /dev/null +++ b/src/main/kotlin/kestrel/store/page/BufferPool.kt @@ -0,0 +1,133 @@ +package kestrel.store.page + +import kestrel.KestrelException + +class BufferPoolExhaustedException(message: String) : KestrelException(message) + +/** + * Fixed-capacity page cache in front of a [DiskManager], using clock + * (second-chance) replacement: each frame has a reference bit set on access, + * and eviction sweeps frames clearing bits before reclaiming an unpinned, + * unreferenced frame. Dirty pages are written back on eviction or explicit + * flush. + */ +class BufferPool(private val diskManager: DiskManager, private val capacity: Int) : PageStore { + init { + require(capacity > 0) { "buffer pool capacity must be positive" } + } + + private val frames = arrayOfNulls(capacity) + private val referenceBit = BooleanArray(capacity) + private val pageTable = HashMap() + private val freeFrames = ArrayDeque((0 until capacity).toList()) + private var clockHand = 0 + + override fun fetchPage(pageId: Long): Page { + pageTable[pageId]?.let { idx -> + val page = frames[idx]!! + page.pinCount++ + referenceBit[idx] = true + return page + } + val idx = allocateFrame() + val page = diskManager.readPage(pageId) + page.pinCount = 1 + frames[idx] = page + pageTable[pageId] = idx + referenceBit[idx] = true + return page + } + + override fun newPage(): Page { + val idx = allocateFrame() + val pageId = diskManager.allocatePage() + val page = Page(pageId) + page.pinCount = 1 + page.isDirty = true + frames[idx] = page + pageTable[pageId] = idx + referenceBit[idx] = true + return page + } + + override fun unpinPage(pageId: Long, markDirty: Boolean) { + val idx = pageTable[pageId] ?: return + val page = frames[idx]!! + if (markDirty) page.isDirty = true + if (page.pinCount > 0) page.pinCount-- + } + + fun flushPage(pageId: Long) { + val idx = pageTable[pageId] ?: return + flushFrame(idx) + } + + fun flushAll() { + for (idx in frames.indices) { + if (frames[idx] != null) flushFrame(idx) + } + } + + /** Removes a page from the cache and the backing file. Caller must not hold a pin on it. */ + override fun deletePage(pageId: Long) { + dropFromCache(pageId) + diskManager.freePage(pageId) + } + + /** + * Drops a page from the cache without writing it back or freeing it on + * disk, so the next [fetchPage] re-reads whatever is currently on disk. + * Used after [kestrel.store.wal.RecoveryManager] restores a before-image + * directly through the [DiskManager], bypassing this cache: without an + * invalidation the pool would keep serving the stale, rolled-back copy. + */ + fun invalidate(pageId: Long) { + dropFromCache(pageId) + } + + private fun dropFromCache(pageId: Long) { + val idx = pageTable[pageId] ?: return + val page = frames[idx]!! + check(page.pinCount == 0) { "cannot drop pinned page $pageId from the cache" } + pageTable.remove(pageId) + frames[idx] = null + referenceBit[idx] = false + freeFrames.addLast(idx) + } + + fun isPinned(pageId: Long): Boolean = pageTable[pageId]?.let { frames[it]?.pinCount ?: 0 > 0 } ?: false + + private fun flushFrame(idx: Int) { + val page = frames[idx] ?: return + if (page.isDirty) { + diskManager.writePage(page) + page.isDirty = false + } + } + + private fun allocateFrame(): Int { + if (freeFrames.isNotEmpty()) return freeFrames.removeFirst() + var sweeps = 0 + val maxSweeps = 2 * capacity + 1 + while (sweeps <= maxSweeps) { + val idx = clockHand + clockHand = (clockHand + 1) % capacity + val page = frames[idx] + if (page == null) return idx + if (page.pinCount > 0) { + sweeps++ + continue + } + if (referenceBit[idx]) { + referenceBit[idx] = false + sweeps++ + continue + } + if (page.isDirty) diskManager.writePage(page) + pageTable.remove(page.id) + frames[idx] = null + return idx + } + throw BufferPoolExhaustedException("no unpinned frame available to evict (capacity=$capacity)") + } +} diff --git a/src/main/kotlin/kestrel/store/page/DiskManager.kt b/src/main/kotlin/kestrel/store/page/DiskManager.kt new file mode 100644 index 0000000..e603b41 --- /dev/null +++ b/src/main/kotlin/kestrel/store/page/DiskManager.kt @@ -0,0 +1,132 @@ +package kestrel.store.page + +import kestrel.KestrelException +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter +import kestrel.store.codec.Crc32C +import java.io.File +import java.io.RandomAccessFile + +class CorruptPageException(message: String) : KestrelException(message) + +/** + * Owns a single heap file of fixed-size pages. Every physical slot is stored + * as `[4-byte CRC32C][PAGE_SIZE payload]` so torn or corrupted writes are + * detected on read. Slot 0 is reserved for a metadata header (page count and + * the head of the free list); logical page ids are 0-based and map to + * physical slot `id + 1`. Freed pages are threaded into a singly linked free + * list stored inline in the freed page's own bytes, so allocation reuses + * space instead of only ever growing the file. + */ +class DiskManager(private val file: File) : AutoCloseable { + companion object { + private const val MAGIC = 0x4B535452 // "KSTR" + private const val SLOT_SIZE = 4 + Page.PAGE_SIZE + private const val HEADER_SLOT = 0L + private const val NO_PAGE = -1L + } + + private val raf = RandomAccessFile(file, "rw") + private var pageCount: Long + private var freeListHead: Long + + init { + if (raf.length() == 0L) { + pageCount = 0 + freeListHead = NO_PAGE + writeHeader() + } else { + val header = readSlot(HEADER_SLOT) + val reader = ByteReader(header) + val magic = reader.readFixedLong().toInt() + if (magic != MAGIC) throw CorruptPageException("bad header magic in ${file.path}") + pageCount = reader.readFixedLong() + freeListHead = reader.readFixedLong() + } + } + + fun allocatedPageCount(): Long = pageCount + + fun allocatePage(): Long { + if (freeListHead != NO_PAGE) { + val pageId = freeListHead + val payload = readSlot(physicalSlot(pageId)) + freeListHead = ByteReader(payload).readFixedLong() + writeHeader() + return pageId + } + val pageId = pageCount + pageCount += 1 + writeHeader() + return pageId + } + + fun freePage(pageId: Long) { + val payload = ByteWriter(Page.PAGE_SIZE) + payload.writeFixedLong(freeListHead) + val padded = payload.toByteArray().copyOf(Page.PAGE_SIZE) + writeSlot(physicalSlot(pageId), padded) + freeListHead = pageId + writeHeader() + } + + fun readPage(pageId: Long): Page { + require(pageId in 0 until pageCount) { "page $pageId is not allocated" } + val page = Page(pageId) + val payload = readSlot(physicalSlot(pageId)) + System.arraycopy(payload, 0, page.data, 0, Page.PAGE_SIZE) + return page + } + + fun writePage(page: Page) { + writeSlot(physicalSlot(page.id), page.data) + } + + fun fsync() { + raf.fd.sync() + } + + override fun close() { + fsync() + raf.close() + } + + private fun physicalSlot(pageId: Long): Long = pageId + 1 + + private fun writeHeader() { + val writer = ByteWriter(Page.PAGE_SIZE) + writer.writeFixedLong(MAGIC.toLong()) + writer.writeFixedLong(pageCount) + writer.writeFixedLong(freeListHead) + val padded = writer.toByteArray().copyOf(Page.PAGE_SIZE) + writeSlot(HEADER_SLOT, padded) + } + + private fun writeSlot(slot: Long, payload: ByteArray) { + require(payload.size == Page.PAGE_SIZE) { "payload must be exactly ${Page.PAGE_SIZE} bytes" } + val checksum = Crc32C.of(payload) + raf.seek(slot * SLOT_SIZE) + raf.write(checksum ushr 24) + raf.write(checksum ushr 16) + raf.write(checksum ushr 8) + raf.write(checksum) + raf.write(payload) + } + + private fun readSlot(slot: Long): ByteArray { + raf.seek(slot * SLOT_SIZE) + val checksumBytes = ByteArray(4) + raf.readFully(checksumBytes) + val expectedChecksum = ((checksumBytes[0].toInt() and 0xFF) shl 24) or + ((checksumBytes[1].toInt() and 0xFF) shl 16) or + ((checksumBytes[2].toInt() and 0xFF) shl 8) or + (checksumBytes[3].toInt() and 0xFF) + val payload = ByteArray(Page.PAGE_SIZE) + raf.readFully(payload) + val actualChecksum = Crc32C.of(payload) + if (actualChecksum != expectedChecksum) { + throw CorruptPageException("checksum mismatch at slot $slot in ${file.path}") + } + return payload + } +} diff --git a/src/main/kotlin/kestrel/store/page/Page.kt b/src/main/kotlin/kestrel/store/page/Page.kt new file mode 100644 index 0000000..e6967c7 --- /dev/null +++ b/src/main/kotlin/kestrel/store/page/Page.kt @@ -0,0 +1,17 @@ +package kestrel.store.page + +/** A fixed-size in-memory buffer for one on-disk page, tracked by the buffer pool. */ +class Page(val id: Long) { + companion object { + const val PAGE_SIZE = 4096 + } + + val data: ByteArray = ByteArray(PAGE_SIZE) + var isDirty: Boolean = false + var pinCount: Int = 0 + + fun reset() { + data.fill(0) + isDirty = false + } +} diff --git a/src/main/kotlin/kestrel/store/page/PageStore.kt b/src/main/kotlin/kestrel/store/page/PageStore.kt new file mode 100644 index 0000000..c08d517 --- /dev/null +++ b/src/main/kotlin/kestrel/store/page/PageStore.kt @@ -0,0 +1,16 @@ +package kestrel.store.page + +/** + * The page-level surface [kestrel.store.heap.HeapFile] and + * [kestrel.store.btree.BPlusTree] depend on, extracted from [BufferPool] so a + * transactional caller can substitute a decorator (see + * [kestrel.store.engine.LoggingPageStore]) that observes every page touched + * during an operation without either data structure knowing it is being + * logged. + */ +interface PageStore { + fun fetchPage(pageId: Long): Page + fun newPage(): Page + fun unpinPage(pageId: Long, markDirty: Boolean) + fun deletePage(pageId: Long) +} diff --git a/src/main/kotlin/kestrel/store/sql/DirectoryEntryCodec.kt b/src/main/kotlin/kestrel/store/sql/DirectoryEntryCodec.kt new file mode 100644 index 0000000..7652c97 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DirectoryEntryCodec.kt @@ -0,0 +1,187 @@ +package kestrel.store.sql + +import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.UniqueConstraint +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.SelectStatement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter +import kestrel.store.codec.CodecException +import kestrel.storage.Collation +import kestrel.storage.ColumnType + +/** + * Encodes a [TableDirectoryEntry] as a self-contained byte payload so it can + * live as a single row in [DiskCatalog]'s directory heap. Unlike + * [kestrel.store.codec.TupleCodec] this is not a generic row format: it is + * shaped exactly around the fields a directory entry has, including nested + * column and index-definition lists. A column's [Column.default] and + * [Column.generated] are each stored as rendered SQL text and re-parsed through + * a throwaway `SELECT () FROM ...` (via the same [writeDefault]/[readDefault] + * helpers - the wire shape is identical, an optional expression rendered as text) + * the same way [DiskCatalog.parseCheckExpressionText] resolves a + * [CheckConstraintEntry] - the alternative, a bespoke binary codec for + * [Expression], would have to mirror the whole AST hierarchy for what is + * always a constant-only (DEFAULT) or own-table-column-only (GENERATED) + * expression in practice. [Column.collation] itself needs no such trick - a plain byte, its + * [kestrel.storage.Collation] enum ordinal - since it is never anything but one of the three + * built-in sequences. + */ +object DirectoryEntryCodec { + private const val NO_PAGE = -1L + private const val NO_PRECISION = -1L + + fun encode(entry: TableDirectoryEntry): ByteArray { + val writer = ByteWriter() + writeString(writer, entry.tableName) + writer.writeVarInt(entry.columns.size.toLong()) + for (column in entry.columns) { + writeString(writer, column.name) + writeString(writer, column.type.displayName) + writer.writeByte(if (column.notNull) 1 else 0) + writer.writeByte(if (column.primaryKey) 1 else 0) + writer.writeZigZag(column.precision?.toLong() ?: NO_PRECISION) + writer.writeZigZag(column.scale?.toLong() ?: NO_PRECISION) + writeOptionalExpression(writer, column.default) + writeOptionalExpression(writer, column.generated) + writer.writeByte(column.collation.ordinal) + } + writer.writeFixedLong(entry.dataHeaderPageId) + writer.writeFixedLong(entry.primaryKeyIndexRootPageId ?: NO_PAGE) + writer.writeVarInt(entry.secondaryIndexes.size.toLong()) + for (index in entry.secondaryIndexes) { + writeString(writer, index.name) + writer.writeVarInt(index.columnNames.size.toLong()) + for (columnName in index.columnNames) writeString(writer, columnName) + writer.writeFixedLong(index.rootPageId) + writer.writeByte(if (index.unique) 1 else 0) + } + writer.writeVarInt(entry.compositePrimaryKey.size.toLong()) + for (columnName in entry.compositePrimaryKey) writeString(writer, columnName) + writer.writeVarInt(entry.uniqueConstraints.size.toLong()) + for (unique in entry.uniqueConstraints) { + writer.writeByte(if (unique.name != null) 1 else 0) + unique.name?.let { writeString(writer, it) } + writer.writeVarInt(unique.columns.size.toLong()) + for (columnName in unique.columns) writeString(writer, columnName) + } + writer.writeVarInt(entry.checks.size.toLong()) + for (check in entry.checks) { + writer.writeByte(if (check.name != null) 1 else 0) + check.name?.let { writeString(writer, it) } + writeString(writer, check.expressionText) + } + writer.writeVarInt(entry.foreignKeys.size.toLong()) + for (fk in entry.foreignKeys) { + writer.writeByte(if (fk.name != null) 1 else 0) + fk.name?.let { writeString(writer, it) } + writer.writeVarInt(fk.columns.size.toLong()) + for (columnName in fk.columns) writeString(writer, columnName) + writeString(writer, fk.referencedTable) + writer.writeVarInt(fk.referencedColumns.size.toLong()) + for (columnName in fk.referencedColumns) writeString(writer, columnName) + writer.writeByte(fk.onDelete.ordinal) + writer.writeByte(fk.onUpdate.ordinal) + } + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): TableDirectoryEntry { + val reader = ByteReader(bytes) + val tableName = readString(reader) + val columnCount = reader.readVarInt().toInt() + val columns = ArrayList(columnCount) + repeat(columnCount) { + val name = readString(reader) + val type = ColumnType.fromKeyword(readString(reader)) + ?: throw CodecException("unknown column type in directory entry for table $tableName") + val notNull = reader.readByte() != 0 + val primaryKey = reader.readByte() != 0 + val precision = reader.readZigZag().takeIf { it != NO_PRECISION }?.toInt() + val scale = reader.readZigZag().takeIf { it != NO_PRECISION }?.toInt() + val default = readOptionalExpression(reader, name, "DEFAULT") + val generated = readOptionalExpression(reader, name, "GENERATED") + val collation = Collation.entries[reader.readByte()] + columns.add(Column(name, type, notNull, primaryKey, precision, scale, default, generated, collation)) + } + val dataHeaderPageId = reader.readFixedLong() + val primaryKeyIndexRootPageId = reader.readFixedLong().takeIf { it != NO_PAGE } + val indexCount = reader.readVarInt().toInt() + val secondaryIndexes = ArrayList(indexCount) + repeat(indexCount) { + val name = readString(reader) + val columnNameCount = reader.readVarInt().toInt() + val columnNames = ArrayList(columnNameCount) + repeat(columnNameCount) { columnNames.add(readString(reader)) } + val rootPageId = reader.readFixedLong() + val unique = reader.readByte() != 0 + secondaryIndexes.add(SecondaryIndexEntry(name, columnNames, rootPageId, unique)) + } + val compositePrimaryKeyCount = reader.readVarInt().toInt() + val compositePrimaryKey = ArrayList(compositePrimaryKeyCount) + repeat(compositePrimaryKeyCount) { compositePrimaryKey.add(readString(reader)) } + val uniqueConstraintCount = reader.readVarInt().toInt() + val uniqueConstraints = ArrayList(uniqueConstraintCount) + repeat(uniqueConstraintCount) { + val name = if (reader.readByte() != 0) readString(reader) else null + val columnCount = reader.readVarInt().toInt() + val columns = ArrayList(columnCount) + repeat(columnCount) { columns.add(readString(reader)) } + uniqueConstraints.add(UniqueConstraint(name, columns)) + } + val checkCount = reader.readVarInt().toInt() + val checks = ArrayList(checkCount) + repeat(checkCount) { + val name = if (reader.readByte() != 0) readString(reader) else null + val expressionText = readString(reader) + checks.add(CheckConstraintEntry(name, expressionText)) + } + val foreignKeyCount = reader.readVarInt().toInt() + val foreignKeys = ArrayList(foreignKeyCount) + repeat(foreignKeyCount) { + val name = if (reader.readByte() != 0) readString(reader) else null + val columnCount = reader.readVarInt().toInt() + val columns = ArrayList(columnCount) + repeat(columnCount) { columns.add(readString(reader)) } + val referencedTable = readString(reader) + val referencedColumnCount = reader.readVarInt().toInt() + val referencedColumns = ArrayList(referencedColumnCount) + repeat(referencedColumnCount) { referencedColumns.add(readString(reader)) } + val onDelete = ForeignKeyAction.values()[reader.readByte()] + val onUpdate = ForeignKeyAction.values()[reader.readByte()] + foreignKeys.add(ForeignKeyConstraint(columns, referencedTable, referencedColumns, onDelete, onUpdate, name)) + } + return TableDirectoryEntry( + tableName, columns, dataHeaderPageId, primaryKeyIndexRootPageId, secondaryIndexes, + compositePrimaryKey, uniqueConstraints, checks, foreignKeys + ) + } + + private fun writeString(writer: ByteWriter, text: String) { + writer.writeLengthPrefixed(text.toByteArray(Charsets.UTF_8)) + } + + private fun readString(reader: ByteReader): String = String(reader.readLengthPrefixed(), Charsets.UTF_8) + + private fun writeOptionalExpression(writer: ByteWriter, expression: Expression?) { + writer.writeByte(if (expression != null) 1 else 0) + expression?.let { writeString(writer, SqlWriter.expressionText(it)) } + } + + private fun readOptionalExpression(reader: ByteReader, columnName: String, label: String): Expression? { + if (reader.readByte() == 0) return null + val text = readString(reader) + val statements = Parser(Lexer("SELECT ($text) FROM directory_entry_placeholder;").tokenize()).parseProgram() + val statement = statements.singleOrNull() as? SelectStatement + ?: throw CodecException("corrupt $label expression for column $columnName: $text") + val projection = statement.projection as? ExpressionProjection + ?: throw CodecException("corrupt $label expression for column $columnName: $text") + return projection.items.single().expression + } +} diff --git a/src/main/kotlin/kestrel/store/sql/DiskCatalog.kt b/src/main/kotlin/kestrel/store/sql/DiskCatalog.kt new file mode 100644 index 0000000..5a80606 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DiskCatalog.kt @@ -0,0 +1,1272 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.catalog.CheckConstraint +import kestrel.catalog.CheckExpressionColumns +import kestrel.catalog.Column +import kestrel.catalog.DependencyAnalysis +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.RelationRename +import kestrel.catalog.Sequence +import kestrel.catalog.SequenceDefinition +import kestrel.catalog.TableSchema +import kestrel.catalog.TriggerDefinition +import kestrel.catalog.UniqueConstraint +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter +import kestrel.store.engine.StorageEngine +import kestrel.store.engine.TransactionalHeap +import kestrel.store.heap.RecordId +import kestrel.storage.Row +import kestrel.storage.Table +import kestrel.storage.Value +import java.io.File + +class DiskCatalogException(message: String) : KestrelException(message) + +/** + * A disk-backed system catalog: one [DiskTable] per user table, described by + * a [TableDirectoryEntry] row in a dedicated directory heap. The directory + * heap is always the very first heap allocated in a fresh database file, so + * it always lands on page 0 (see [DiskManager][kestrel.store.page.DiskManager]: + * page ids are handed out from 0 upward and nothing else runs before it) - + * that fixed, well-known id is what lets a fresh process find the directory + * again without another bootstrap record pointing at it. + * + * Every [DiskTable] this catalog opens is created with a callback that + * rewrites its directory row, in the same transaction as the write that + * triggered it, whenever a primary or secondary index's root page changes + * (see [DiskTable]'s documentation on why that matters). That keeps the + * directory heap's on-disk contents always in sync with the last committed + * root, so [open] can reconstruct every table and index purely from what is + * on disk after a full process restart - not just within one process's + * lifetime the way holding a live [kestrel.store.engine.TransactionalIndex] + * object would. + * + * The same directory heap also holds one row per view, tagged (see + * `ENTRY_TAG_TABLE`/`ENTRY_TAG_VIEW`) and encoded by [ViewDirectoryEntryCodec] instead of + * [DirectoryEntryCodec] - a view has no heap/index pages of its own to describe, only its + * `CREATE VIEW ... AS ...;` text (see [ViewDirectoryEntry]), so it rides along in the one heap + * that is already guaranteed to exist at a well-known page id, rather than needing a second + * fixed-page-id heap of its own. + * + * A third tag, `ENTRY_TAG_SEQUENCE`, holds one row per SEQUENCE, encoded by + * [SequenceDirectoryEntryCodec] - see [SequenceDirectoryEntry]'s own doc for why its live counter + * (unlike everything else in this heap) is deliberately NOT part of that transactional row. + * + * A fourth tag, `ENTRY_TAG_TRIGGER`, holds one row per TRIGGER, encoded by + * [TriggerDirectoryEntryCodec] - the same "store rendered SQL text, re-parse on open" trick as + * [ViewDirectoryEntry], since a trigger's WHEN clause and body are arbitrarily deep AST too. + */ +class DiskCatalog private constructor( + private val engine: StorageEngine, + private val directory: TransactionalHeap +) : AutoCloseable { + companion object { + private const val DIRECTORY_HEADER_PAGE_ID = 0L + + /** + * Leading byte on every row stored in the directory heap, distinguishing a + * [TableDirectoryEntry] from a [ViewDirectoryEntry] - both share the same heap rather than + * each getting a dedicated one, since a fresh heap's header page id is only deterministic + * (page 0) for the very first heap [open] allocates; a second fixed-page-id heap would need + * its own bootstrap record pointing at it, which this tag byte avoids entirely. + */ + private const val ENTRY_TAG_TABLE: Int = 0 + private const val ENTRY_TAG_VIEW: Int = 1 + private const val ENTRY_TAG_SEQUENCE: Int = 2 + private const val ENTRY_TAG_TRIGGER: Int = 3 + + fun open(dataFile: File, walFile: File, poolCapacity: Int = 64): DiskCatalog { + val isFresh = !dataFile.exists() || dataFile.length() == 0L + val engine = StorageEngine.open(dataFile, walFile, poolCapacity) + val directory = if (isFresh) { + val heap = engine.createHeap() + check(heap.headerPageId == DIRECTORY_HEADER_PAGE_ID) { + "expected the catalog directory to be the first page allocated in a fresh file, " + + "got page ${heap.headerPageId}" + } + heap + } else { + engine.openHeap(DIRECTORY_HEADER_PAGE_ID) + } + val catalog = DiskCatalog(engine, directory) + catalog.loadDirectory() + return catalog + } + } + + /** + * A mutable box for the [rowsByName] key a [DiskTable]'s `onRootsChanged` callback should + * persist under - captured by reference rather than by value so `ALTER TABLE ... RENAME TO` + * (see [renameTable]) can repoint an already-constructed [DiskTable]'s callback at its new key + * without having to reconstruct the table (which would mean reopening its heap/index pages). + */ + private class TableKeyBox(var value: String) + + private class DirectoryRow(var rid: RecordId, var entry: TableDirectoryEntry, val table: DiskTable, val keyBox: TableKeyBox) + private class ViewRow(var rid: RecordId, val entry: ViewDirectoryEntry, val query: QueryStatement) + private class SequenceRow(var rid: RecordId, var entry: SequenceDirectoryEntry, var sequence: Sequence) + private class TriggerRow(var rid: RecordId, var entry: TriggerDirectoryEntry, var definition: TriggerDefinition) + + private val rowsByName = LinkedHashMap() + private val viewsByName = LinkedHashMap() + private val sequencesByName = LinkedHashMap() + private val triggersByName = LinkedHashMap() + + /** + * `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE`: a plain in-memory [Table] - the exact same + * class [kestrel.catalog.Catalog] uses for every table on the in-memory engine - rather than a + * [DiskTable]. It never allocates a heap or index page and never gains a directory entry, so + * [loadDirectory] can never discover it again after a process restart: there is no "don't + * persist this one" flag to thread through the pager/WAL at all, unlike + * [kestrel.catalog.TableSchema.temporary] on the in-memory engine (which shares its `tables` map + * with every ordinary table and instead relies on [kestrel.persistence.SqlWriter] skipping it). + * See [DiskExecutor]'s temporary-table INSERT/UPDATE/DELETE/SELECT path for how it is read and + * mutated, and its BEGIN/SAVEPOINT/ROLLBACK handling for why a live [Table] reference (not just + * its rows) needs to be reachable from outside this class. + */ + private val temporaryTables = LinkedHashMap() + + private fun loadDirectory() { + val txn = engine.beginTransaction() + for ((rid, bytes) in directory.scan(txn)) { + val payload = bytes.copyOfRange(1, bytes.size) + when (val tag = bytes[0].toInt()) { + ENTRY_TAG_TABLE -> { + val entry = DirectoryEntryCodec.decode(payload) + val keyBox = TableKeyBox(entry.tableName.lowercase()) + val table = openTableFromEntry(entry, keyBox) + rowsByName[keyBox.value] = DirectoryRow(rid, entry, table, keyBox) + } + ENTRY_TAG_VIEW -> { + val entry = ViewDirectoryEntryCodec.decode(payload) + viewsByName[entry.viewName.lowercase()] = ViewRow(rid, entry, parseViewQuery(entry)) + } + ENTRY_TAG_SEQUENCE -> { + val entry = SequenceDirectoryEntryCodec.decode(payload) + sequencesByName[entry.name.lowercase()] = SequenceRow(rid, entry, sequenceFromEntry(entry)) + } + ENTRY_TAG_TRIGGER -> { + val entry = TriggerDirectoryEntryCodec.decode(payload) + triggersByName[entry.triggerName.lowercase()] = TriggerRow(rid, entry, parseTriggerDefinition(entry)) + } + else -> throw DiskCatalogException("unknown directory entry tag: $tag") + } + } + engine.commit(txn) + } + + /** + * Re-derives a view's [QueryStatement] from its stored `CREATE VIEW ... AS ...;` text using the + * same [Lexer]/[Parser] pair the REPL and every other SQL entry point already go through - no + * bespoke AST codec, see [ViewDirectoryEntry]'s own KDoc for why. + */ + private fun parseViewQuery(entry: ViewDirectoryEntry): QueryStatement { + val statements = Parser(Lexer(entry.statementText).tokenize()).parseProgram() + val statement = statements.singleOrNull() as? CreateViewStatement + ?: throw DiskCatalogException("corrupt view directory entry for ${entry.viewName}") + return statement.query + } + + /** + * Re-derives a trigger's [TriggerDefinition] from its stored `CREATE TRIGGER ...;` text - same + * "reuse the real Lexer/Parser instead of a bespoke codec" trick [parseViewQuery] uses for a + * whole query. + */ + private fun parseTriggerDefinition(entry: TriggerDirectoryEntry): TriggerDefinition { + val statements = Parser(Lexer(entry.statementText).tokenize()).parseProgram() + val statement = statements.singleOrNull() as? CreateTriggerStatement + ?: throw DiskCatalogException("corrupt trigger directory entry for ${entry.triggerName}") + return TriggerDefinition( + statement.triggerName, statement.timing, statement.event, statement.tableName, statement.whenClause, statement.body + ) + } + + /** + * Re-derives a CHECK constraint's [Expression] from its stored text (see [CheckConstraintEntry]) + * by wrapping it back into a throwaway, never-executed `SELECT () FROM ...` and pulling the + * expression back out of the parsed projection - the same "reuse the real Lexer/Parser instead of + * a bespoke codec" trick [parseViewQuery] uses for a whole query. The FROM target is never + * resolved against this or any catalog, so its name is arbitrary. + */ + private fun parseCheckExpressionText(text: String): Expression { + val statements = Parser(Lexer("SELECT ($text) FROM check_constraint_placeholder;").tokenize()).parseProgram() + val statement = statements.singleOrNull() as? SelectStatement + ?: throw DiskCatalogException("corrupt CHECK constraint expression: $text") + val projection = statement.projection as? ExpressionProjection + ?: throw DiskCatalogException("corrupt CHECK constraint expression: $text") + return projection.items.single().expression + } + + private fun openTableFromEntry(entry: TableDirectoryEntry, keyBox: TableKeyBox): DiskTable { + val heap = engine.openHeap(entry.dataHeaderPageId) + val primaryIndex = entry.primaryKeyIndexRootPageId?.let { engine.openIndex(it) } + val schema = TableSchema( + entry.tableName, entry.columns, + foreignKeys = entry.foreignKeys, + uniqueConstraints = entry.uniqueConstraints, + compositePrimaryKey = entry.compositePrimaryKey, + checks = entry.checks.map { CheckConstraint(it.name, parseCheckExpressionText(it.expressionText)) } + ) + val table = DiskTable(schema, heap, primaryIndex, mutableListOf()) { txnId -> + persistEntry(keyBox.value, txnId) + } + for (secondary in entry.secondaryIndexes) { + val columnIndices = secondary.columnNames.map { columnName -> + schema.columnIndex(columnName) + ?: throw DiskCatalogException("directory entry for ${entry.tableName} references unknown column $columnName") + } + val index = engine.openIndex(secondary.rootPageId) + table.attachSecondaryIndex(SecondaryIndexHandle(secondary.name, secondary.columnNames, columnIndices, index, secondary.unique)) + } + return table + } + + fun beginTransaction(): Long = engine.beginTransaction() + fun commit(txnId: Long) = engine.commit(txnId) + fun abort(txnId: Long) = engine.abort(txnId) + fun savepoint(txnId: Long): Long = engine.savepoint(txnId) + fun rollbackToSavepoint(txnId: Long, savepointLsn: Long) = engine.rollbackToSavepoint(txnId, savepointLsn) + fun discardSavepoint(txnId: Long, savepointLsn: Long) = engine.discardSavepoint(txnId, savepointLsn) + + val tableNames: List get() = rowsByName.values.map { it.entry.tableName } + temporaryTables.values.map { it.schema.name } + + /** + * Total pages ever allocated in the backing data file, including any now sitting on the free + * list after a `DROP TABLE`/`DROP INDEX` (see [DiskTable.destroyStorage]/[DiskTable.dropSecondaryIndexByName]) - + * exposed so a caller (chiefly a test) can confirm a later `CREATE TABLE`/`CREATE INDEX` reused + * that reclaimed space instead of only ever growing the file, the same proof-by-counter + * [DiskTable]'s own `scanCount`/`pointLookupCount`/etc. give for index pushdown. + */ + fun allocatedPageCount(): Long = engine.allocatedPageCount() + + fun table(name: String): DiskTable { + val key = name.lowercase() + rowsByName[key]?.let { return it.table } + if (viewsByName.containsKey(key)) throw DiskCatalogException("cannot modify view: $name") + throw DiskCatalogException("no such table: $name") + } + + fun tableOrNull(name: String): DiskTable? = rowsByName[name.lowercase()]?.table + + /** + * `compositePrimaryKey` mirrors [kestrel.catalog.TableSchema.compositePrimaryKey]: a + * table-level `PRIMARY KEY (a, b, ...)`, as an alternative to an inline `col TYPE PRIMARY KEY` + * tracked by [Column.primaryKey]. Marking the listed columns NOT NULL (like the in-memory + * [kestrel.engine.Executor] does for the same clause) is the caller's responsibility - this + * catalog only records schema/storage, it does not rewrite the SQL-level column definitions. + */ + fun createTable( + name: String, + columns: List, + compositePrimaryKey: List = emptyList(), + uniqueConstraints: List = emptyList(), + checks: List = emptyList(), + foreignKeys: List = emptyList() + ): DiskTable { + val key = name.lowercase() + if (rowsByName.containsKey(key)) throw DiskCatalogException("table '$name' already exists") + if (viewsByName.containsKey(key)) throw DiskCatalogException("a view named '$name' already exists") + if (temporaryTables.containsKey(key)) throw DiskCatalogException("a temporary table named '$name' already exists") + if (columns.count { it.primaryKey } > 1 || + (compositePrimaryKey.isNotEmpty() && columns.any { it.primaryKey }) + ) { + throw DiskCatalogException("table '$name' declares more than one primary key") + } + if (compositePrimaryKey.isNotEmpty()) { + val declared = compositePrimaryKey.map { it.lowercase() }.toSet() + if (declared.size != compositePrimaryKey.size) { + throw DiskCatalogException("table '$name' declares a duplicate column in its PRIMARY KEY") + } + for (columnName in compositePrimaryKey) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw DiskCatalogException("PRIMARY KEY refers to unknown column: $columnName") + } + } + } + for (unique in uniqueConstraints) { + for (columnName in unique.columns) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw DiskCatalogException("UNIQUE constraint refers to unknown column: $columnName") + } + } + } + val hasPrimaryKey = columns.any { it.primaryKey } || compositePrimaryKey.isNotEmpty() + + val heap = engine.createHeap() + val primaryIndex = if (hasPrimaryKey) engine.createIndex() else null + val entry = TableDirectoryEntry( + name, columns, heap.headerPageId, primaryIndex?.rootPageId, emptyList(), + compositePrimaryKey, uniqueConstraints, checks.map { CheckConstraintEntry.from(it) }, foreignKeys + ) + + val txn = engine.beginTransaction() + val rid = directory.insert(txn, taggedTable(entry)) + engine.commit(txn) + + val schema = TableSchema( + name, columns, + foreignKeys = foreignKeys, + uniqueConstraints = uniqueConstraints, + compositePrimaryKey = compositePrimaryKey, + checks = checks + ) + val keyBox = TableKeyBox(key) + val table = DiskTable(schema, heap, primaryIndex, mutableListOf()) { txnId -> persistEntry(keyBox.value, txnId) } + rowsByName[key] = DirectoryRow(rid, entry, table, keyBox) + return table + } + + /** + * Registers a brand new `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` - see [temporaryTables]' + * own KDoc for why this never touches [engine]/[directory] at all, unlike [createTable]. Name + * collisions are checked against every namespace a permanent table already checks against + * ([rowsByName]/[viewsByName]) plus every other temp table, deliberately NOT allowing a temp + * table to shadow a same-named permanent table/view the way SQLite does - see [DiskExecutor]'s + * own resolution order for the reasoning (a single, simpler, unambiguous namespace is a + * deliberate, narrower scope-cut for this executor). + */ + fun createTemporaryTable(schema: TableSchema): Table { + val key = schema.name.lowercase() + if (rowsByName.containsKey(key)) throw DiskCatalogException("table '${schema.name}' already exists") + if (viewsByName.containsKey(key)) throw DiskCatalogException("a view named '${schema.name}' already exists") + if (temporaryTables.containsKey(key)) throw DiskCatalogException("table '${schema.name}' already exists") + val table = Table(schema) + temporaryTables[key] = table + return table + } + + fun temporaryTableOrNull(name: String): Table? = temporaryTables[name.lowercase()] + + fun dropTemporaryTable(name: String) { + temporaryTables.remove(name.lowercase()) ?: throw DiskCatalogException("no such temporary table: $name") + } + + /** + * Every currently-open temp table, oldest first - used by [DiskExecutor]'s BEGIN/SAVEPOINT/ + * ROLLBACK handling to snapshot/restore live [Table.rows] lists around a disk transaction. Since + * `CREATE`/`DROP TEMPORARY TABLE` are DDL and already rejected inside an explicit transaction + * (see [DiskExecutor.isDdl]), the *set* of temp tables never changes mid-transaction - only their + * row contents can - so returning the live objects (not a name list) is enough for that caller to + * snapshot/restore without a second lookup. + */ + val temporaryTableValues: List
get() = temporaryTables.values.toList() + + /** + * Drops a table's directory entry so it is no longer visible, and frees every page its heap + * and indexes (primary and secondary) own back to the shared [kestrel.store.page.DiskManager] + * free list, via [DiskTable.destroyStorage] - so a later `CREATE TABLE`/`CREATE INDEX` reuses + * this space instead of only ever growing the data file. RESTRICT (default) is blocked by any view depending on it, directly or + * through another view, or by any other table's FOREIGN KEY referencing it (a self-referencing + * FK does not count - it disappears together with the table itself). [cascade] drops every + * dependent view (transitively, diamond-safe like [dropView]) and strips - never drops - every + * other table's FOREIGN KEY referencing this one, mirroring [kestrel.catalog.Catalog.dropTable]'s + * `dropDependentViews`/`dropDependentForeignKeys`: CASCADE never implies dropping a table that + * merely references the one being dropped, only the constraint pointing at it. Everything - the + * view deletions, the table's own row, and every stripped referencing schema - commits in one + * transaction so a mid-way failure leaves nothing half-dropped. A trigger defined on [name] is + * always dropped alongside it, CASCADE or not - mirrors [kestrel.catalog.Catalog.dropTable]: a + * trigger without its table makes no sense to keep around, unlike a FOREIGN KEY reference, so + * there is nothing to RESTRICT against. + */ + fun dropTable(name: String, cascade: Boolean = false) { + val key = name.lowercase() + val row = rowsByName[key] ?: run { + if (viewsByName.containsKey(key)) throw DiskCatalogException("'$name' is a view, use DROP VIEW") + throw DiskCatalogException("no such table: $name") + } + val viewClosure = dependentViewsOrThrow(name, cascade) + val referencingTables = dependentForeignKeysOrThrow(name, cascade) + val droppedTriggerKeys = triggersByName.filterValues { it.definition.tableName.equals(name, ignoreCase = true) }.keys.toList() + + val txn = engine.beginTransaction() + try { + for (viewKey in viewClosure) directory.delete(txn, viewsByName.getValue(viewKey).rid) + for (triggerKey in droppedTriggerKeys) directory.delete(txn, triggersByName.getValue(triggerKey).rid) + directory.delete(txn, row.rid) + row.table.destroyStorage(txn) + for ((otherKey, other) in referencingTables) { + val stripped = other.table.schema.foreignKeys.filterNot { it.referencedTable.equals(name, ignoreCase = true) } + other.table.replaceSchemaMetadata(withForeignKeys(other.table.schema, stripped)) + persistEntry(otherKey, txn) + } + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + for (viewKey in viewClosure) viewsByName.remove(viewKey) + for (triggerKey in droppedTriggerKeys) triggersByName.remove(triggerKey) + rowsByName.remove(key) + } + + /** + * RESTRICT check against every view depending on [name] - returns the transitive closure to + * delete instead when [cascade] is set. See [dropTable]'s KDoc. + */ + private fun dependentViewsOrThrow(name: String, cascade: Boolean): Set { + val closure = transitiveViewDependents(name.lowercase()) + if (closure.isEmpty() || cascade) return closure + val names = closure.mapNotNull { viewsByName[it]?.entry?.viewName } + throw DiskCatalogException("cannot drop '$name' because view(s) ${names.joinToString(", ")} depend on it") + } + + /** + * RESTRICT check against every other table's FOREIGN KEY referencing [name] - returns those + * rows instead when [cascade] is set, so the caller can strip the constraint from each. See + * [dropTable]'s KDoc. + */ + private fun dependentForeignKeysOrThrow(name: String, cascade: Boolean): List> { + val key = name.lowercase() + val referencing = rowsByName.entries.filter { (otherKey, other) -> + otherKey != key && other.table.schema.foreignKeys.any { fk -> fk.referencedTable.equals(name, ignoreCase = true) } + } + if (referencing.isEmpty() || cascade) return referencing + val names = referencing.map { it.value.table.schema.name } + throw DiskCatalogException( + "cannot drop '$name' because table(s) ${names.joinToString(", ")} have a FOREIGN KEY referencing it" + ) + } + + private fun rowOrThrow(tableName: String): DirectoryRow { + val key = tableName.lowercase() + rowsByName[key]?.let { return it } + if (viewsByName.containsKey(key)) throw DiskCatalogException("cannot modify view: $tableName") + throw DiskCatalogException("no such table: $tableName") + } + + /** + * Mirrors [kestrel.catalog.Catalog.truncateTables]: wiping every row of [tableNames] makes every + * row of any OTHER table with a FOREIGN KEY into one of them dangling at once, so RESTRICT + * (default) blocks on that regardless of the FK's own `onDelete` action - there is no per-row + * delete for it to act on - and [cascade] means "also truncate the whole referencing table", a + * transitive closure over FOREIGN KEY edges computed exactly like [dependentForeignKeysOrThrow]'s + * single-hop version, never a per-row [DiskForeignKeyEnforcer.cascadeDelete]-style plan. Every + * table in the closure is emptied against the SAME [txn] the caller already opened (see + * [DiskExecutor.executeTruncate]), so it shares that statement's commit/rollback exactly like a + * WHERE-less DELETE does - unlike RESTART IDENTITY's sequence reset, which + * [DiskExecutor.executeTruncate] deliberately runs outside any transaction, mirroring [nextValue]'s + * own already-non-transactional sequence position. + */ + fun truncateTables(txn: Long, tableNames: List, cascade: Boolean): List { + tableNames.forEach { rowOrThrow(it) } + val closure = truncateClosure(tableNames.map { it.lowercase() }.toCollection(linkedSetOf()), cascade, tableNames) + for (key in closure) { + val table = rowsByName.getValue(key).table + for ((rid, _) in table.scan(txn)) table.delete(txn, rid) + } + return closure.map { rowsByName.getValue(it).entry.tableName } + } + + private fun truncateClosure(found: LinkedHashSet, cascade: Boolean, requestedNames: List): LinkedHashSet { + var frontier: Set = found + while (true) { + val referencing = rowsByName.values.filter { row -> + row.entry.tableName.lowercase() !in found && + row.table.schema.foreignKeys.any { fk -> fk.referencedTable.lowercase() in frontier } + } + if (referencing.isEmpty()) return found + if (!cascade) { + val names = referencing.map { it.entry.tableName } + throw DiskCatalogException( + "cannot truncate ${requestedNames.joinToString(", ")} because table(s) " + + "${names.joinToString(", ")} have a FOREIGN KEY referencing them; use CASCADE" + ) + } + val next = referencing.map { it.entry.tableName.lowercase() }.toSet() + found.addAll(next) + frontier = next + } + } + + /** + * Appends a new column (already validated by [DiskExecutor]) to every existing row - mirrors + * [kestrel.catalog.Catalog.addColumn]. [valueForRow] backfills every pre-existing row (each + * called with that row's OLD, pre-append values, then [DiskTable.rewriteRows]-transformed to + * carry the result) - a constant lambda for a plain DEFAULT, or a fresh per-row expression + * evaluation for GENERATED ALWAYS AS; a table with zero rows just gets the new schema with + * nothing to rewrite. + */ + fun addColumn(tableName: String, column: Column, valueForRow: (Row) -> Value) { + val row = rowOrThrow(tableName) + val schema = row.table.schema + if (schema.columnIndex(column.name) != null) { + throw DiskCatalogException("duplicate column name: ${column.name}") + } + val newSchema = TableSchema( + schema.name, schema.columns + column, schema.foreignKeys, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.rewriteRows(txn, newSchema) { existingRow -> Row(existingRow.values + valueForRow(existingRow)) } + persistEntry(tableName.lowercase(), txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** + * `ALTER TABLE t ALTER COLUMN col TYPE ...` - mirrors [kestrel.catalog.Catalog.changeColumnType], + * built on [DiskTable.rewriteRows] exactly like [addColumn] rather than [addColumn]'s own + * `List -> Value` append shape, since this replaces an EXISTING position instead of + * appending a new one. Unlike the in-memory engine's [kestrel.engine.Executor], which validates + * every row up front and only ever calls its own `Catalog.changeColumnType` once every row is + * already known-good, [valueForRow] here is BOTH the value-computation AND the NOT NULL/CHECK + * validation step (see [DiskExecutor.executeChangeColumnType]) - a failure partway through + * throws out of [DiskTable.rewriteRows]'s loop, and the `catch` below aborts the transaction, + * unwinding every row already rewritten. Same "validate lazily, rely on abort to undo a partial + * per-row rewrite" contract [addColumn] already leans on for a NOT NULL `GENERATED ALWAYS AS` + * backfill. + */ + fun changeColumnType(tableName: String, columnName: String, newColumn: Column, valueForRow: (Row) -> Value) { + val row = rowOrThrow(tableName) + val schema = row.table.schema + val columnIndex = schema.columnIndex(columnName) + ?: throw DiskCatalogException("no such column: $columnName in table $tableName") + val newColumns = schema.columns.toMutableList().also { it[columnIndex] = newColumn } + val newSchema = TableSchema( + schema.name, newColumns, schema.foreignKeys, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.rewriteRows(txn, newSchema) { existingRow -> + val values = existingRow.values.toMutableList() + values[columnIndex] = valueForRow(existingRow) + Row(values) + } + persistEntry(tableName.lowercase(), txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** + * RESTRICT (default) rejects the drop if [columnName] is used by another table's FOREIGN KEY, + * this table's own FOREIGN KEY, a UNIQUE constraint, a CHECK constraint, or a secondary index - + * CASCADE removes all of those alongside the column itself. Mirrors + * [kestrel.catalog.Catalog.dropColumn], including that a PRIMARY KEY column can never be + * dropped, with or without CASCADE. + */ + fun dropColumn(tableName: String, columnName: String, cascade: Boolean) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + schema.columnIndex(columnName) ?: throw DiskCatalogException("no such column: $columnName in table $tableName") + if (schema.columns.size == 1) { + throw DiskCatalogException("cannot drop $columnName: table $tableName would be left with no columns") + } + if (schema.primaryKeyColumns.any { it.equals(columnName, ignoreCase = true) }) { + throw DiskCatalogException("cannot drop column $columnName because it is part of the PRIMARY KEY of $tableName") + } + + // Mirrors kestrel.catalog.Catalog.dropColumn: a GENERATED column can only ever read this + // table's own non-GENERATED columns, so dropping columnName orphans any GENERATED column + // depending on it exactly like a CHECK constraint referencing it - CASCADE drops it too, + // RESTRICT blocks the whole drop instead. GENERATED can never be a PRIMARY KEY column, so + // there is no "cascaded into dropping the PRIMARY KEY" case to guard against here. + val ownGeneratedDependents = schema.columns.filter { column -> + !column.name.equals(columnName, ignoreCase = true) && + column.generated?.let { CheckExpressionColumns.references(it, columnName) } == true + } + val droppedNames = (ownGeneratedDependents.map { it.name } + columnName).map { it.lowercase() }.toSet() + val droppedIndices = schema.columns.withIndex() + .filter { (_, column) -> column.name.lowercase() in droppedNames } + .map { it.index } + .toSet() + + val referencingForeignKeys = rowsByName.entries.filter { (otherKey, other) -> + otherKey != key && other.table.schema.foreignKeys.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && fk.referencedColumns.any { it.lowercase() in droppedNames } + } + } + val ownForeignKeys = schema.foreignKeys.filter { fk -> fk.columns.any { it.lowercase() in droppedNames } } + val ownUniqueConstraints = schema.uniqueConstraints.filter { unique -> unique.columns.any { it.lowercase() in droppedNames } } + val ownChecks = schema.checks.filter { check -> droppedNames.any { name -> CheckExpressionColumns.references(check.expression, name) } } + val dependentIndexes = row.table.secondaryIndexes.filter { it.columnNames.any { c -> c.lowercase() in droppedNames } } + + if (!cascade) { + val reasons = mutableListOf() + if (ownGeneratedDependents.isNotEmpty()) { + reasons.add( + "column(s) ${ownGeneratedDependents.joinToString(", ") { it.name }} are GENERATED columns that depend on it" + ) + } + if (referencingForeignKeys.isNotEmpty()) { + reasons.add( + "table(s) ${referencingForeignKeys.joinToString(", ") { it.value.table.schema.name }} have a FOREIGN KEY referencing it" + ) + } + if (ownForeignKeys.isNotEmpty()) reasons.add("it is used by one of its own FOREIGN KEY constraints") + if (ownUniqueConstraints.isNotEmpty()) reasons.add("it is used by a UNIQUE constraint") + if (ownChecks.isNotEmpty()) reasons.add("it is used by a CHECK constraint") + if (dependentIndexes.isNotEmpty()) { + reasons.add("index(es) ${dependentIndexes.joinToString(", ") { it.name }} are defined on it") + } + if (reasons.isNotEmpty()) { + throw DiskCatalogException( + "cannot drop column $columnName from $tableName because ${reasons.joinToString("; ")}; use CASCADE" + ) + } + } + + val newColumns = schema.columns.filterNot { it.name.lowercase() in droppedNames } + val newForeignKeys = schema.foreignKeys - ownForeignKeys.toSet() + val newUniqueConstraints = schema.uniqueConstraints - ownUniqueConstraints.toSet() + val newChecks = schema.checks - ownChecks.toSet() + val newSchema = TableSchema(schema.name, newColumns, newForeignKeys, newChecks, newUniqueConstraints, schema.compositePrimaryKey) + + val txn = engine.beginTransaction() + try { + for (indexName in dependentIndexes.map { it.name }) row.table.dropSecondaryIndexByName(indexName, txn) + row.table.rewriteRows(txn, newSchema) { existingRow -> + Row(existingRow.values.filterIndexed { valueIndex, _ -> valueIndex !in droppedIndices }) + } + persistEntry(key, txn) + for ((otherKey, other) in referencingForeignKeys) { + val updatedForeignKeys = other.table.schema.foreignKeys.filterNot { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && fk.referencedColumns.any { it.lowercase() in droppedNames } + } + other.table.replaceSchemaMetadata(withForeignKeys(other.table.schema, updatedForeignKeys)) + persistEntry(otherKey, txn) + } + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** + * Always fully propagates rather than being rejected the way [dropColumn] can be - mirrors + * [kestrel.catalog.Catalog.renameColumn]: this table's own PRIMARY KEY/FOREIGN KEY/UNIQUE/CHECK + * definitions, any other table's FOREIGN KEY referencing this column, and any secondary index + * defined on it (see [DiskTable.renameColumnAndIndexes]) all get the new name. + */ + fun renameColumn(tableName: String, oldName: String, newName: String) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + if (schema.columnIndex(oldName) == null) { + throw DiskCatalogException("no such column: $oldName in table $tableName") + } + if (!oldName.equals(newName, ignoreCase = true) && schema.columnIndex(newName) != null) { + throw DiskCatalogException("column $newName already exists in table $tableName") + } + + val newColumns = schema.columns.map { column -> + val renamed = if (column.name.equals(oldName, ignoreCase = true)) column.copy(name = newName) else column + // A GENERATED column can only ever reference this table's own columns, so renaming one + // of them must rewrite that reference too - mirrors kestrel.catalog.Catalog.renameColumn. + renamed.generated?.let { renamed.copy(generated = CheckExpressionColumns.rename(it, oldName, newName)) } ?: renamed + } + val newForeignKeys = schema.foreignKeys.map { fk -> + val selfReference = fk.referencedTable.equals(tableName, ignoreCase = true) + fk.copy( + columns = renameInList(fk.columns, oldName, newName), + referencedColumns = if (selfReference) renameInList(fk.referencedColumns, oldName, newName) else fk.referencedColumns + ) + } + val newUniqueConstraints = schema.uniqueConstraints.map { it.copy(columns = renameInList(it.columns, oldName, newName)) } + val newChecks = schema.checks.map { CheckConstraint(it.name, CheckExpressionColumns.rename(it.expression, oldName, newName)) } + val newCompositePrimaryKey = renameInList(schema.compositePrimaryKey, oldName, newName) + val newSchema = TableSchema(schema.name, newColumns, newForeignKeys, newChecks, newUniqueConstraints, newCompositePrimaryKey) + + val affectedOthers = rowsByName.entries.filter { (otherKey, other) -> + otherKey != key && other.table.schema.foreignKeys.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && fk.referencedColumns.any { it.equals(oldName, ignoreCase = true) } + } + } + + val txn = engine.beginTransaction() + try { + row.table.renameColumnAndIndexes(newSchema, oldName, newName) + persistEntry(key, txn) + for ((otherKey, other) in affectedOthers) { + val updatedForeignKeys = other.table.schema.foreignKeys.map { fk -> + if (fk.referencedTable.equals(tableName, ignoreCase = true)) { + fk.copy(referencedColumns = renameInList(fk.referencedColumns, oldName, newName)) + } else { + fk + } + } + other.table.replaceSchemaMetadata(withForeignKeys(other.table.schema, updatedForeignKeys)) + persistEntry(otherKey, txn) + } + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** + * Renames a table and propagates the identifier everywhere it is used by name - mirrors + * [kestrel.catalog.Catalog.renameTable]: this table's own FOREIGN KEY(s) that self-reference it, + * any other table's FOREIGN KEY pointing at it, every view whose defining query mentions it + * anywhere (see [RelationRename]), and every trigger defined ON it (just the [TriggerDefinition. + * tableName] field, re-rendered and re-persisted - trigger body statements are NOT rewritten, + * same documented scope-cut as [kestrel.catalog.Catalog.renameTable]). Unlike the in-memory + * engine there is no separate global index map to fix up - every secondary index lives inside its + * own [DiskTable], keyed by column name, which a table rename never touches. + */ + fun renameTable(oldName: String, newName: String) { + val oldKey = oldName.lowercase() + val row = rowOrThrow(oldName) + val newKey = newName.lowercase() + if (!oldName.equals(newName, ignoreCase = true)) { + if (rowsByName.containsKey(newKey)) throw DiskCatalogException("table '$newName' already exists") + if (viewsByName.containsKey(newKey)) throw DiskCatalogException("a view named '$newName' already exists") + } + + val schema = row.table.schema + val selfReferencing = schema.foreignKeys.any { it.referencedTable.equals(oldName, ignoreCase = true) } + val ownNewForeignKeys = if (selfReferencing) { + schema.foreignKeys.map { fk -> if (fk.referencedTable.equals(oldName, ignoreCase = true)) fk.copy(referencedTable = newName) else fk } + } else { + schema.foreignKeys + } + val newSchema = TableSchema(newName, schema.columns, ownNewForeignKeys, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey) + + val affectedOthers = rowsByName.entries.filter { (otherKey, other) -> + otherKey != oldKey && other.table.schema.foreignKeys.any { it.referencedTable.equals(oldName, ignoreCase = true) } + } + val affectedViews = viewsByName.entries.filter { (_, view) -> DependencyAnalysis.referencesRelation(view.query, oldName) } + val affectedTriggers = triggersByName.entries.filter { (_, trigger) -> trigger.definition.tableName.equals(oldName, ignoreCase = true) } + + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + if (oldKey != newKey) { + rowsByName.remove(oldKey) + row.keyBox.value = newKey + rowsByName[newKey] = row + } + persistEntry(newKey, txn) + + for ((otherKey, other) in affectedOthers) { + val updatedForeignKeys = other.table.schema.foreignKeys.map { fk -> + if (fk.referencedTable.equals(oldName, ignoreCase = true)) fk.copy(referencedTable = newName) else fk + } + other.table.replaceSchemaMetadata(withForeignKeys(other.table.schema, updatedForeignKeys)) + persistEntry(otherKey, txn) + } + + for ((viewKey, viewRow) in affectedViews) { + val renamedQuery = RelationRename.rename(viewRow.query, oldName, newName) + val newEntry = ViewDirectoryEntry( + viewRow.entry.viewName, SqlWriter.write(CreateViewStatement(viewRow.entry.viewName, renamedQuery, orReplace = false)) + ) + val rid = directory.update(txn, viewRow.rid, taggedView(newEntry)) + viewsByName[viewKey] = ViewRow(rid, newEntry, renamedQuery) + } + + for ((triggerKey, triggerRow) in affectedTriggers) { + val renamedDefinition = triggerRow.definition.copy(tableName = newName) + val newEntry = TriggerDirectoryEntry(triggerRow.entry.triggerName, SqlWriter.triggerText(renamedDefinition)) + val rid = directory.update(txn, triggerRow.rid, taggedTrigger(newEntry)) + triggersByName[triggerKey] = TriggerRow(rid, newEntry, renamedDefinition) + } + + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + private fun withForeignKeys(schema: TableSchema, foreignKeys: List): TableSchema = TableSchema( + schema.name, schema.columns, foreignKeys, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + + /** + * `ALTER TABLE t ADD CONSTRAINT name FOREIGN KEY (...) REFERENCES ...` - mirrors + * [kestrel.catalog.Catalog.addForeignKey]: [kestrel.store.sql.DiskExecutor.executeAddConstraint] + * has already validated every existing row against [foreignKey] before calling this, so installing + * it is pure schema metadata - [DiskTable.replaceSchemaMetadata] plus a re-persisted directory + * entry, no row rewrite (row shape is unchanged) and no index to build. + */ + fun addForeignKey(tableName: String, foreignKey: ForeignKeyConstraint) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys + foreignKey, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** [addForeignKey]'s counterpart for `ADD CONSTRAINT name UNIQUE (...)` - see its own doc for the validate-before-install contract. */ + fun addUniqueConstraint(tableName: String, unique: UniqueConstraint) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks, schema.uniqueConstraints + unique, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** [addForeignKey]'s counterpart for `ADD CONSTRAINT name CHECK (...)` - see its own doc for the validate-before-install contract. */ + fun addCheckConstraint(tableName: String, check: CheckConstraint) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks + check, schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + /** + * `ALTER TABLE t DROP CONSTRAINT name [RESTRICT | CASCADE]` - mirrors + * [kestrel.catalog.Catalog.dropConstraint]: finds a named CHECK, UNIQUE, or FOREIGN KEY + * constraint on [tableName] (case-insensitive; a PRIMARY KEY has no name to match against) and + * removes it. Only a UNIQUE constraint has cross-table fallout - see [dropUniqueConstraint]. + */ + fun dropConstraint(tableName: String, name: String, cascade: Boolean) { + val key = tableName.lowercase() + val row = rowOrThrow(tableName) + val schema = row.table.schema + val foreignKey = schema.foreignKeys.find { it.name?.equals(name, ignoreCase = true) == true } + val check = schema.checks.find { it.name?.equals(name, ignoreCase = true) == true } + val unique = schema.uniqueConstraints.find { it.name?.equals(name, ignoreCase = true) == true } + when { + foreignKey != null -> { + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys - foreignKey, schema.checks, + schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + check != null -> { + val newSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks - check, + schema.uniqueConstraints, schema.compositePrimaryKey + ) + val txn = engine.beginTransaction() + try { + row.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + unique != null -> dropUniqueConstraint(tableName, unique, cascade) + else -> throw DiskCatalogException("no such constraint: $name in table $tableName") + } + } + + /** + * Mirrors [kestrel.catalog.Catalog.dropUniqueConstraint]: a FOREIGN KEY's `referencedColumns` + * must exactly match one of the parent's enforced-unique key groups to have been accepted in the + * first place, so dropping a UNIQUE constraint can silently invalidate any FOREIGN KEY (on this + * table itself, via a self-reference, or on any other table) that leans on exactly this column + * set for its uniqueness guarantee, UNLESS another remaining group still covers it. RESTRICT + * (default) blocks the drop when a real dependent exists; CASCADE strips that FOREIGN KEY right + * along with it, mirroring [dropTable]'s own FOREIGN KEY handling. + */ + private fun dropUniqueConstraint(tableName: String, unique: UniqueConstraint, cascade: Boolean) { + val key = tableName.lowercase() + val schema = rowOrThrow(tableName).table.schema + val remainingUniques = schema.uniqueConstraints - unique + val targetSet = unique.columns.map { it.lowercase() }.toSet() + val stillEnforced = TableSchema( + schema.name, schema.columns, schema.foreignKeys, schema.checks, remainingUniques, schema.compositePrimaryKey + ).enforcedUniqueKeyGroups().any { group -> group.map { it.lowercase() }.toSet() == targetSet } + + fun dependsOnTarget(fk: ForeignKeyConstraint) = + fk.referencedTable.equals(tableName, ignoreCase = true) && fk.referencedColumns.map { it.lowercase() }.toSet() == targetSet + + val dependents = if (stillEnforced) emptyList() else rowsByName.values.filter { other -> other.table.schema.foreignKeys.any(::dependsOnTarget) } + if (!cascade && dependents.isNotEmpty()) { + val names = dependents.map { it.table.schema.name }.distinct() + throw DiskCatalogException( + "cannot drop constraint ${unique.name} because table(s) ${names.joinToString(", ")} have a " + + "FOREIGN KEY referencing it; use CASCADE" + ) + } + + val txn = engine.beginTransaction() + try { + for (other in dependents) { + val otherKey = other.table.schema.name.lowercase() + val strippedForeignKeys = other.table.schema.foreignKeys.filterNot(::dependsOnTarget) + other.table.replaceSchemaMetadata(withForeignKeys(other.table.schema, strippedForeignKeys)) + persistEntry(otherKey, txn) + } + // Re-fetched: a self-referencing FOREIGN KEY on tableName itself may have just been + // rewritten above (dependents is not restricted to OTHER tables). + val current = rowOrThrow(tableName) + val newSchema = TableSchema( + current.table.schema.name, current.table.schema.columns, current.table.schema.foreignKeys, + current.table.schema.checks, current.table.schema.uniqueConstraints - unique, current.table.schema.compositePrimaryKey + ) + current.table.replaceSchemaMetadata(newSchema) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + private fun renameInList(names: List, oldName: String, newName: String): List = + names.map { if (it.equals(oldName, ignoreCase = true)) newName else it } + + /** Convenience for a single-column secondary index; delegates to the general, composite-capable overload below. */ + fun createIndex(tableName: String, indexName: String, columnName: String, unique: Boolean = false): SecondaryIndexHandle = + createIndex(tableName, indexName, listOf(columnName), unique) + + /** + * Creates a secondary index over [columnNames] (more than one entry for a composite, + * multi-column index) - see [SecondaryIndexHandle]/[DiskTable] for how a composite key is + * encoded and how uniqueness is enforced. + */ + fun createIndex(tableName: String, indexName: String, columnNames: List, unique: Boolean = false): SecondaryIndexHandle { + require(columnNames.isNotEmpty()) { "index must cover at least one column" } + val key = tableName.lowercase() + val row = rowsByName[key] ?: throw DiskCatalogException("no such table: $tableName") + if (row.table.secondaryIndexes.any { it.name.equals(indexName, ignoreCase = true) }) { + throw DiskCatalogException("index '$indexName' already exists") + } + val columnIndices = columnNames.map { columnName -> + row.table.schema.columnIndex(columnName) + ?: throw DiskCatalogException("no such column: $columnName in table $tableName") + } + + val index = engine.createIndex() + val handle = SecondaryIndexHandle(indexName, columnNames, columnIndices, index, unique) + val backfill = engine.beginTransaction() + try { + row.table.backfillSecondaryIndex(backfill, handle) + } catch (failure: Exception) { + engine.abort(backfill) + throw failure + } + engine.commit(backfill) + + row.table.attachSecondaryIndex(handle) + val persistTxn = engine.beginTransaction() + persistEntry(key, persistTxn) + engine.commit(persistTxn) + return handle + } + + /** + * Drops a secondary index by name, searched across every table since `DROP INDEX` names only + * the index, not its owning table - mirrors [kestrel.catalog.Catalog.dropIndex]'s flat, + * table-less namespace. Unlike [dropColumn]'s CASCADE-guarded removal, this never needs a + * dependency check: nothing in this schema (FOREIGN KEY/CHECK/UNIQUE constraint) ever + * references an index by name, only by the columns it happens to cover, so + * [kestrel.catalog.Catalog.dropIndex] never needed one either. Frees the dropped B+Tree's own + * pages back to the shared free list via [DiskTable.dropSecondaryIndexByName], the same + * reclamation [dropTable] does for a whole table's storage. + */ + fun dropIndex(indexName: String) { + val (key, row) = rowsByName.entries.firstOrNull { (_, row) -> + row.table.secondaryIndexes.any { it.name.equals(indexName, ignoreCase = true) } + }?.let { it.key to it.value } ?: throw DiskCatalogException("no such index: $indexName") + val txn = engine.beginTransaction() + try { + row.table.dropSecondaryIndexByName(indexName, txn) + persistEntry(key, txn) + engine.commit(txn) + } catch (failure: Exception) { + engine.abort(txn) + throw failure + } + } + + private fun persistEntry(tableKey: String, txnId: Long) { + val row = rowsByName.getValue(tableKey) + val updated = row.table.snapshotDirectoryEntry() + row.rid = directory.update(txnId, row.rid, taggedTable(updated)) + row.entry = updated + } + + private fun taggedTable(entry: TableDirectoryEntry): ByteArray = + byteArrayOf(ENTRY_TAG_TABLE.toByte()) + DirectoryEntryCodec.encode(entry) + + private fun taggedView(entry: ViewDirectoryEntry): ByteArray = + byteArrayOf(ENTRY_TAG_VIEW.toByte()) + ViewDirectoryEntryCodec.encode(entry) + + private fun taggedTrigger(entry: TriggerDirectoryEntry): ByteArray = + byteArrayOf(ENTRY_TAG_TRIGGER.toByte()) + TriggerDirectoryEntryCodec.encode(entry) + + val viewNames: List get() = viewsByName.values.map { it.entry.viewName } + + fun viewOrNull(name: String): QueryStatement? = viewsByName[name.lowercase()]?.query + + /** + * Persists a view's defining [query] as rendered SQL text (see [ViewDirectoryEntry]) - mirrors + * [kestrel.catalog.Catalog.createView]'s name-collision rules (a view and a table can never + * share a name, `orReplace` only waives the view-already-exists check). + */ + fun createView(name: String, query: QueryStatement, orReplace: Boolean) { + val key = name.lowercase() + if (rowsByName.containsKey(key)) throw DiskCatalogException("a table named '$name' already exists") + if (!orReplace && viewsByName.containsKey(key)) throw DiskCatalogException("view '$name' already exists") + val entry = ViewDirectoryEntry(name, SqlWriter.write(CreateViewStatement(name, query, orReplace = false))) + val bytes = taggedView(entry) + val txn = engine.beginTransaction() + val existing = viewsByName[key] + val rid = if (existing != null) directory.update(txn, existing.rid, bytes) else directory.insert(txn, bytes) + engine.commit(txn) + viewsByName[key] = ViewRow(rid, entry, query) + } + + /** + * Drops a view, blocked (RESTRICT, the default) by any other view depending on it - directly or + * transitively through a chain of views - unless [cascade] is set, in which case the whole + * transitive closure of dependents is dropped alongside it. Mirrors + * [kestrel.catalog.Catalog.dropView]/`dropDependentViews`'s "diamond-safe" ordering: the closure + * is computed once, up front, before anything is removed. + */ + fun dropView(name: String, cascade: Boolean = false) { + val key = name.lowercase() + if (!viewsByName.containsKey(key)) { + if (rowsByName.containsKey(key)) throw DiskCatalogException("'$name' is a table, use DROP TABLE") + throw DiskCatalogException("no such view: $name") + } + val closure = transitiveViewDependents(key) + if (closure.isNotEmpty() && !cascade) { + val names = closure.mapNotNull { viewsByName[it]?.entry?.viewName } + throw DiskCatalogException( + "cannot drop '$name' because view(s) ${names.joinToString(", ")} depend on it; use CASCADE" + ) + } + val txn = engine.beginTransaction() + for (depKey in closure) directory.delete(txn, viewsByName.getValue(depKey).rid) + directory.delete(txn, viewsByName.getValue(key).rid) + engine.commit(txn) + for (depKey in closure) viewsByName.remove(depKey) + viewsByName.remove(key) + } + + /** + * Every view depending on [rootKey] (already lowercase), directly or through another view - + * [rootKey] itself may name a table or a view, [DependencyAnalysis] does not care which. + * Identical fixed-point walk to [kestrel.catalog.Catalog.transitiveViewDependents]. + */ + private fun transitiveViewDependents(rootKey: String): Set { + val found = linkedSetOf() + var frontier = setOf(rootKey) + while (true) { + val next = viewsByName.entries + .filter { (key, _) -> key != rootKey && key !in found } + .filter { (_, row) -> frontier.any { DependencyAnalysis.referencesRelation(row.query, it) } } + .map { it.key } + .toSet() + if (next.isEmpty()) return found + found.addAll(next) + frontier = next + } + } + + // ---- SEQUENCE ---- + + /** + * Rebuilds a live [Sequence] from its persisted definition plus whatever [SequenceDirectoryEntry. + * positionPageId] currently holds - the exact `(value, called)` pair [Sequence.snapshotPosition] + * last wrote there, so reload reproduces the live instance's internal state precisely rather than + * approximating it (see [Sequence.snapshotPosition]'s own doc for why [Sequence.peekNext] is the + * wrong thing to persist this way). + */ + private fun sequenceFromEntry(entry: SequenceDirectoryEntry): Sequence { + val definition = SequenceDefinition(entry.name, entry.incrementBy, entry.minValue, entry.maxValue, entry.startValue, entry.cycle) + val reader = ByteReader(engine.readRawPage(entry.positionPageId)) + val value = reader.readFixedLong() + val called = reader.readByte() != 0 + return Sequence(definition, initialValue = value, initialCalled = called) + } + + private fun writeSequencePosition(pageId: Long, position: Pair) { + val (value, called) = position + engine.writeRawPage(pageId, ByteWriter(9).writeFixedLong(value).writeByte(if (called) 1 else 0).toByteArray()) + } + + private fun taggedSequence(entry: SequenceDirectoryEntry): ByteArray = + byteArrayOf(ENTRY_TAG_SEQUENCE.toByte()) + SequenceDirectoryEntryCodec.encode(entry) + + val sequenceNames: List get() = sequencesByName.values.map { it.entry.name } + + fun sequence(name: String): Sequence = sequenceOrNull(name) ?: throw DiskCatalogException("no such sequence: $name") + + fun sequenceOrNull(name: String): Sequence? = sequencesByName[name.lowercase()]?.sequence + + /** + * Sequence names deliberately live in their own namespace here, same as + * [kestrel.catalog.Catalog.createSequence] - no collision check against [rowsByName]/[viewsByName]. + */ + fun createSequence(definition: SequenceDefinition): Sequence { + val key = definition.name.lowercase() + if (sequencesByName.containsKey(key)) throw DiskCatalogException("sequence '${definition.name}' already exists") + val positionPageId = engine.allocateRawPage() + writeSequencePosition(positionPageId, definition.startValue to false) + val entry = SequenceDirectoryEntry( + definition.name, definition.incrementBy, definition.minValue, definition.maxValue, + definition.startValue, definition.cycle, positionPageId + ) + val txn = engine.beginTransaction() + val rid = directory.insert(txn, taggedSequence(entry)) + engine.commit(txn) + val sequence = Sequence(definition) + sequencesByName[key] = SequenceRow(rid, entry, sequence) + return sequence + } + + /** + * Frees the dropped sequence's [SequenceDirectoryEntry.positionPageId] back to the shared page + * free list via [StorageEngine.freeRawPage], outside the directory-row transaction - like every + * other read/write of that page (see [writeSequencePosition]), freeing it is deliberately not + * part of the transactional/WAL-logged path, so an abort of surrounding SQL never resurrects a + * page already handed back to the free list. + */ + fun dropSequence(name: String) { + val key = name.lowercase() + val row = sequencesByName[key] ?: throw DiskCatalogException("no such sequence: $name") + val txn = engine.beginTransaction() + directory.delete(txn, row.rid) + engine.commit(txn) + engine.freeRawPage(row.entry.positionPageId) + sequencesByName.remove(key) + } + + /** + * Mirrors [kestrel.catalog.Catalog.alterSequence]: [Sequence.altered] computes the position ALTER + * SEQUENCE leaves the counter at (carried over unchanged, or reset by `RESTART`, see [Sequence]'s + * own doc) purely in memory first. Only once the transactional definition update in [directory] + * has actually committed is that new position written to the raw counter page - so a failure + * building or persisting the new [SequenceDirectoryEntry] (e.g. the sequence vanished from under + * a concurrent caller) can never leave the on-disk counter reflecting an ALTER whose definition + * change did not itself survive. + */ + fun alterSequence(name: String, newDefinition: SequenceDefinition, restartAt: Long?) { + val key = name.lowercase() + val row = sequencesByName[key] ?: throw DiskCatalogException("no such sequence: $name") + val altered = row.sequence.altered(newDefinition, restartAt) + val newEntry = SequenceDirectoryEntry( + newDefinition.name, newDefinition.incrementBy, newDefinition.minValue, newDefinition.maxValue, + newDefinition.startValue, newDefinition.cycle, row.entry.positionPageId + ) + val txn = engine.beginTransaction() + val rid = directory.update(txn, row.rid, taggedSequence(newEntry)) + engine.commit(txn) + writeSequencePosition(row.entry.positionPageId, altered.snapshotPosition()) + row.rid = rid + row.entry = newEntry + row.sequence = altered + } + + /** + * Advances [name] and durably persists the new position immediately via + * [StorageEngine.writeRawPage][kestrel.store.engine.StorageEngine.writeRawPage] - deliberately + * outside of any [beginTransaction]/[commit]/[abort], so the advance survives even a ROLLBACK of + * whichever statement's transaction called it (see [SequenceDirectoryEntry]'s own doc). Guarded + * by a lock on this catalog instance rather than [kestrel.store.txn.LockManager] (which the raw + * counter page never goes through at all) - the one piece of this catalog actually exercised by + * more than one thread at a time in this codebase (see the concurrent-session BEGIN/COMMIT test), + * so two overlapping NEXTVAL calls on the same sequence must not race on [Sequence.next]'s plain, + * unsynchronized `var`s. + */ + fun nextValue(name: String): Long = synchronized(this) { + val row = sequencesByName[name.lowercase()] ?: throw DiskCatalogException("no such sequence: $name") + val value = row.sequence.next() + writeSequencePosition(row.entry.positionPageId, row.sequence.snapshotPosition()) + value + } + + // ---- TRIGGER ---- + + val triggerNames: List get() = triggersByName.values.map { it.entry.triggerName } + + /** Every trigger definition, in creation order - mirrors [kestrel.catalog.Catalog.triggerDefinitions]. */ + fun triggerDefinitions(): List = triggersByName.values.map { it.definition } + + /** Whether any trigger (any timing/event) is defined ON [tableName] - mirrors [kestrel.catalog.Catalog.hasTriggers]. */ + fun hasTriggers(tableName: String): Boolean = triggersByName.values.any { it.definition.tableName.equals(tableName, ignoreCase = true) } + + /** Every trigger registered for ([tableName], [timing], [event]), in creation order - mirrors [kestrel.catalog.Catalog.triggersOn]. */ + fun triggersOn(tableName: String, timing: TriggerTiming, event: TriggerEvent): List = + triggersByName.values.filter { + it.definition.tableName.equals(tableName, ignoreCase = true) && it.definition.timing == timing && it.definition.event == event + }.map { it.definition } + + /** + * Persists [definition] as rendered `CREATE TRIGGER ...;` text (see [TriggerDirectoryEntry]) - + * mirrors [kestrel.catalog.Catalog.createTrigger]'s name-collision rule. Every other eager check + * (the trigger's own table and every body statement's target table exist, the WHEN clause shape) + * is [DiskExecutor]'s responsibility, same split as [createView] versus [DiskExecutor. + * executeCreateView]. + */ + fun createTrigger(definition: TriggerDefinition) { + val key = definition.name.lowercase() + if (triggersByName.containsKey(key)) throw DiskCatalogException("trigger '${definition.name}' already exists") + val entry = TriggerDirectoryEntry(definition.name, SqlWriter.triggerText(definition)) + val txn = engine.beginTransaction() + val rid = directory.insert(txn, taggedTrigger(entry)) + engine.commit(txn) + triggersByName[key] = TriggerRow(rid, entry, definition) + } + + fun dropTrigger(name: String) { + val key = name.lowercase() + val row = triggersByName[key] ?: throw DiskCatalogException("no such trigger: $name") + val txn = engine.beginTransaction() + directory.delete(txn, row.rid) + engine.commit(txn) + triggersByName.remove(key) + } + + override fun close() = engine.close() +} diff --git a/src/main/kotlin/kestrel/store/sql/DiskExecutor.kt b/src/main/kotlin/kestrel/store/sql/DiskExecutor.kt new file mode 100644 index 0000000..34f006b --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DiskExecutor.kt @@ -0,0 +1,4431 @@ +package kestrel.store.sql + +import java.math.BigDecimal +import kestrel.KestrelException +import kestrel.catalog.CheckConstraint +import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.Sequence +import kestrel.catalog.SequenceDefinition +import kestrel.catalog.TableSchema +import kestrel.catalog.TriggerDefinition +import kestrel.catalog.UniqueConstraint +import kestrel.engine.AggregateFunctions +import kestrel.engine.AlterSequenceResult +import kestrel.engine.AlterTableResult +import kestrel.engine.BeginResult +import kestrel.engine.ColumnResolver +import kestrel.engine.CommitResult +import kestrel.engine.collationFoldedKey +import kestrel.engine.collationFoldedUniquenessKey +import kestrel.engine.compareOrderKeyValues +import kestrel.engine.CreateIndexResult +import kestrel.engine.CreateSequenceResult +import kestrel.engine.CreateTableResult +import kestrel.engine.CreateTriggerResult +import kestrel.engine.CreateViewResult +import kestrel.engine.DeallocateResult +import kestrel.engine.DeleteResult +import kestrel.engine.DropIndexResult +import kestrel.engine.DropSequenceResult +import kestrel.engine.DropTableResult +import kestrel.engine.DropTriggerResult +import kestrel.engine.DropViewResult +import kestrel.engine.Evaluator +import kestrel.engine.ExplainResult +import kestrel.engine.GroupAggregation +import kestrel.engine.GroupingSetExpansion +import kestrel.engine.InsertResult +import kestrel.engine.MergeResult +import kestrel.engine.ParameterSubstitution +import kestrel.engine.PrepareResult +import kestrel.engine.ReleaseSavepointResult +import kestrel.engine.RollbackResult +import kestrel.engine.RollbackToSavepointResult +import kestrel.engine.SavepointResult +import kestrel.engine.SelectResult +import kestrel.engine.SequenceAccess +import kestrel.engine.StatementResult +import kestrel.engine.SubqueryRunner +import kestrel.engine.TriggerRowContext +import kestrel.engine.TruncateTableResult +import kestrel.engine.UpdateResult +import kestrel.sql.ast.AddColumn +import kestrel.sql.ast.AddConstraint +import kestrel.sql.ast.AddedCheck +import kestrel.sql.ast.AddedConstraint +import kestrel.sql.ast.AddedForeignKey +import kestrel.sql.ast.AddedUnique +import kestrel.sql.ast.AlterSequenceStatement +import kestrel.sql.ast.AlterTableStatement +import kestrel.sql.ast.BeginStatement +import kestrel.sql.ast.BetweenExpression +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.ChangeColumnType +import kestrel.sql.ast.CollateExpression +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CommitStatement +import kestrel.sql.ast.CommonTableExpression +import kestrel.sql.ast.ConflictDoNothing +import kestrel.sql.ast.ConflictDoUpdate +import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateSequenceStatement +import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.DeallocateStatement +import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropColumn +import kestrel.sql.ast.DropConstraint +import kestrel.sql.ast.DropIndexStatement +import kestrel.sql.ast.DropMode +import kestrel.sql.ast.DropSequenceStatement +import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.DropTriggerStatement +import kestrel.sql.ast.DropViewStatement +import kestrel.sql.ast.ExecuteStatement +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.ExplainStatement +import kestrel.sql.ast.Expression +import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBound +import kestrel.sql.ast.FrameBoundType +import kestrel.sql.ast.FrameExclude +import kestrel.sql.ast.FrameUnit +import kestrel.sql.ast.FromItem +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression +import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.JoinClause +import kestrel.sql.ast.JoinType +import kestrel.sql.ast.LikeExpression +import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.OnConflictClause +import kestrel.sql.ast.OrderKey +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.PreparableStatement +import kestrel.sql.ast.PrepareStatement +import kestrel.sql.ast.Projection +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.ReleaseSavepointStatement +import kestrel.sql.ast.RenameColumn +import kestrel.sql.ast.RenameTable +import kestrel.sql.ast.RollbackStatement +import kestrel.sql.ast.RollbackToSavepointStatement +import kestrel.sql.ast.SavepointStatement +import kestrel.sql.ast.ScalarSubqueryExpression +import kestrel.sql.ast.SelectItem +import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.SetOperator +import kestrel.sql.ast.StarProjection +import kestrel.sql.ast.Statement +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.sql.ast.TruncateTableStatement +import kestrel.sql.ast.UnaryExpression +import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WindowFrame +import kestrel.sql.ast.WindowFunctionKind +import kestrel.sql.ast.WithQuery +import kestrel.store.heap.RecordId +import kestrel.storage.Collation +import kestrel.storage.ColumnType +import kestrel.storage.DecimalColumns +import kestrel.storage.Row +import kestrel.storage.Table +import kestrel.storage.Value +import kestrel.storage.displayWith +import kestrel.storage.typeName + +class DiskExecutorException(message: String) : KestrelException(message) + +/** + * Executes a deliberately narrow subset of the SQL AST directly against + * [DiskCatalog]/[DiskTable]: CREATE/DROP TABLE, INSERT, SELECT (WHERE/ORDER + * BY/LIMIT/GROUP BY/HAVING/aggregates, window functions (ROW_NUMBER/RANK/ + * DENSE_RANK/aggregate/LAG/LEAD/FIRST_VALUE/LAST_VALUE/NTILE OVER + * PARTITION BY/ORDER BY, including an explicit ROWS/RANGE/GROUPS frame with + * EXCLUDE), INNER/LEFT/RIGHT/FULL/CROSS JOIN across any number of table, derived-table + * (`FROM (SELECT ...) AS alias`) or LATERAL derived-table sources, + * scalar/EXISTS/IN subqueries anywhere an expression is allowed - including + * correlated to an outer row - UNION/UNION ALL/INTERSECT/INTERSECT ALL/ + * EXCEPT/EXCEPT ALL and WITH/WITH RECURSIVE (CTE, fixed-point with an + * iteration limit)), UPDATE and DELETE, each top-level statement running inside its own disk + * transaction (commit on success, abort on any exception so a failure never + * leaves a partially applied statement); a subquery or nested query (set + * operation branch, CTE definition) runs inside the same transaction as the + * statement that contains it rather than opening one of its own. + * + * This is additive, not a replacement for [kestrel.engine.Executor]: it + * reuses the same AST ([kestrel.sql.ast]), the same [Evaluator] and its + * [SubqueryRunner] hook, the same [GroupAggregation]/[AggregateFunctions] + * (both operate purely on `List` rows, so they need no disk-specific + * counterpart), the same [StatementResult] types and the same + * [Column]/[TableSchema] the in-memory engine already uses (both + * [DiskCatalog] and [DiskTable] were built against those exact types for + * this reason), so no translation layer sits between the parser and this + * executor - a query that resolves against the disk backend evaluates + * expressions with exactly the same semantics as the in-memory one. Window + * function evaluation itself (frame resolution, peer groups, RANGE distance + * search) is deliberately duplicated from [kestrel.engine.SelectRunner] + * rather than shared, for the same free-standing-isolation reason as every + * other validation this executor duplicates (see [validateUniqueDefinitions]). + * `CREATE [OR REPLACE] VIEW`/`DROP VIEW [CASCADE]` are supported (materialized fresh on every + * reference - as a plain FROM/JOIN source, inside a CTE body, or on either side of a set operation, + * since all three already resolve a name through the same [resolveFromSource] - view-over-view, + * recursive-definition and RESTRICT/CASCADE dependent-view detection, mirroring + * [kestrel.engine.Executor]/[kestrel.catalog.Catalog]'s own view handling; see + * [DiskCatalog.createView]/[DiskCatalog.dropView]). A derived table (`FROM (SELECT ...) AS alias`, + * see [resolveFromItem]) runs eagerly, once, since it cannot correlate to anything; a LATERAL derived + * table (`JOIN LATERAL (SELECT ...) AS alias ON ...`) is instead re-run once per row already + * accumulated from earlier FROM/JOIN sources, correlated to that row exactly like a WHERE-clause + * correlated subquery (see [executeLateralJoin]) - mirroring [kestrel.engine.SelectRunner]'s own + * handling of both. + * Table-level composite PRIMARY KEY, UNIQUE (inline/table-level `UNIQUE` + * constraints, any number of columns), `CREATE [UNIQUE] INDEX` + * (single-column or composite), column-level/table-level CHECK constraints (validated column + * references/no subquery-aggregate-window at CREATE TABLE time via [validateCheckDefinitions], + * enforced on every INSERT/UPDATE via [enforceChecks], persisted as rendered expression text - see + * [CheckConstraintEntry]), and column-level/table-level FOREIGN KEY constraints - including + * composite keys, self-reference, and `ON DELETE`/`ON UPDATE` `CASCADE`/`SET NULL`/`RESTRICT` (default + * `NO ACTION` behaves like `RESTRICT`), enforced via [DiskForeignKeyEnforcer] and blocking `DROP + * TABLE` (RESTRICT by default, `CASCADE` strips the constraint from every referencing table without + * dropping it - see [DiskCatalog.dropTable]) - are supported, mirroring + * [kestrel.engine.Executor]'s own handling of them. `ALTER TABLE ... ADD/DROP/RENAME COLUMN/RENAME + * TO` (see [executeAlterTable]/[DiskCatalog.addColumn]/[DiskCatalog.dropColumn]/ + * [DiskCatalog.renameColumn]/[DiskCatalog.renameTable]) is supported with the same RESTRICT/CASCADE + * and FOREIGN KEY/UNIQUE/CHECK/index propagation rules as [kestrel.engine.Executor]/ + * [kestrel.catalog.Catalog]. `ALTER TABLE ... ADD CONSTRAINT name CHECK/UNIQUE/FOREIGN KEY` and + * `DROP CONSTRAINT name` (see [executeAddConstraint]/[DiskCatalog.dropConstraint]) validate every + * existing row before installing a new constraint (via a probe transaction, mirroring + * [executeAddColumn]'s own `probeTxn`) and give a UNIQUE constraint the same RESTRICT/CASCADE + * treatment against a dependent FOREIGN KEY that [DiskCatalog.dropTable] already gives a whole table, + * and so is `DROP TABLE ... CASCADE` itself (see [executeDropTable]), + * which additionally drops every transitively dependent view - no DDL mode remains unsupported. + * `BEGIN`/`COMMIT`/`ROLLBACK` (see [executeBegin]/[executeCommit]/ + * [executeRollback]) group any number of INSERT/SELECT/UPDATE/DELETE statements into one disk + * transaction with one commit/abort boundary, backed directly by the storage engine's own WAL/2PL + * transaction rather than a copy of [kestrel.engine.Executor]'s whole-catalog snapshot/restore - + * DDL is deliberately rejected while a transaction is open (see [isDdl] in [execute]) because there + * is no equivalent way to undo a [DiskCatalog] schema change by aborting a storage-engine + * transaction. `CREATE`/`DROP`/`ALTER SEQUENCE` and `NEXTVAL`/`CURRVAL` (see [executeCreateSequence]/ + * [executeDropSequence]/[executeAlterSequence]/[nextValue]/[currentValue]) are supported with the + * same Postgres-mirroring semantics as [kestrel.engine.Executor] - NEXTVAL alone survives a ROLLBACK + * that undoes everything else the rolled-back statement did (see [DiskCatalog.nextValue]), CURRVAL is + * scoped to this [DiskExecutor] instance the same way it is scoped to one [kestrel.engine.Executor], + * and both are restricted to exactly the same three positions (INSERT VALUES/DEFAULT, UPDATE SET/ + * DEFAULT) for the same volatile-function reason - CHECK constraints and `WHERE` reject them outright. + * `SAVEPOINT`/`RELEASE [SAVEPOINT]`/`ROLLBACK TO SAVEPOINT` (see [executeSavepoint]/ + * [executeReleaseSavepoint]/[executeRollbackToSavepoint]) are supported inside an explicit transaction, + * with the same stack semantics (reusing a name shadows the older savepoint of that name; releasing or + * rolling back to one forgets every savepoint established after it, but rolling back keeps the target + * itself usable again) as [kestrel.engine.Executor]. Unlike that whole-catalog-snapshot + * implementation, this one partially rolls back the storage engine's own WAL transaction - see + * [kestrel.store.wal.LogRecord.CompensationUpdate] and [kestrel.store.wal.RecoveryManager.rollbackTo] + * for the ARIES-style compensation log records that make undoing only a suffix of a transaction safe + * to repeat and safe to crash through, and [kestrel.store.engine.StorageEngine.savepoint]/ + * [kestrel.store.engine.StorageEngine.rollbackToSavepoint] for how a B+Tree's root pointer (plain + * in-memory state no log record tracks) and the buffer pool's cache of pages the undo just overwrote + * on disk are kept in sync with it. + * `CREATE`/`DROP TRIGGER` `BEFORE`/`AFTER INSERT`/`UPDATE`/`DELETE FOR EACH ROW` (see + * [executeCreateTrigger]/[executeDropTrigger]/[fireRowTriggers]/[runTriggerBody]) mirror + * [kestrel.engine.Executor]'s own trigger semantics 1:1 (optional `WHEN`, `NEW`/`OLD` resolved via + * the shared [kestrel.engine.TriggerRowContext], cascading fires across tables up to + * [MAX_TRIGGER_DEPTH]) with one deliberate difference in how atomicity is achieved: instead of a + * whole-catalog snapshot/restore, a whole statement plus every trigger it cascades into is undone by + * [withTriggerAtomicity] taking one internal storage-engine savepoint (see [DiskCatalog.savepoint]) + * at the outermost call and rolling back to it on any failure - reusing the exact same primitive + * `SAVEPOINT` is built on, just invisibly. `INSERT ... ON CONFLICT` on a table with a trigger defined + * fires that table's own INSERT triggers for a row that lands fresh and its own UPDATE triggers for a + * row folded into `DO UPDATE` (never both for the same row, and never anything for `DO NOTHING`) - see + * [executeInsertWithConflictHandling] - mirroring the in-memory engine's own resolution of the same gap. + * `TRUNCATE TABLE t1 [, t2, ...] [RESTART IDENTITY | CONTINUE IDENTITY] [CASCADE | RESTRICT]` (see + * [executeTruncate]/[DiskCatalog.truncateTables]) empties every listed table - and, under CASCADE, + * every table transitively reaching it via a FOREIGN KEY - without firing a single row-level trigger + * or consulting any FOREIGN KEY's `onDelete` action, sharing the calling statement's transaction + * exactly like a WHERE-less DELETE; `RESTART IDENTITY` additionally restarts every sequence backing a + * `DEFAULT NEXTVAL(...)` column of a truncated table, the one part of this statement that is NOT + * covered by that same transaction - see [executeTruncate]'s own doc for why, mirroring + * [DiskCatalog.nextValue]'s already-established non-transactional sequence position. + * `MERGE INTO target USING source ON onCondition WHEN [NOT] MATCHED [AND cond] THEN ...` (see + * [executeMerge]/[executeMergeInner]) mirrors [kestrel.engine.Executor]'s own MERGE 1:1 - `source` is + * a table or derived-table subquery resolved once, up front, into a fixed snapshot via + * [resolveMergeSource]/[resolveFromItem], `ON` is matched against a snapshot of the target table taken + * before any mutation (not the live, mutating [DiskTable]), and a target row already affected earlier + * in the same statement throws rather than silently applying a second action - the one difference from + * the in-memory engine is that "already affected"/"this pair's target row" is tracked by + * [kestrel.store.heap.RecordId] rather than [Row] object identity, since a disk row is decoded fresh + * on every read and so is never `===` to an earlier read of "the same" row. + * `PREPARE name AS `/`EXECUTE name(args...)`/`DEALLOCATE [PREPARE] + * name|ALL` (see [executePrepare]/[resolveExecute]/[executeDeallocate]) are ported 1:1 from + * [kestrel.engine.Executor]'s own registry - session-scoped, kept outside [DiskCatalog] for the same + * reason ([explicitTxn] `COMMIT`/`ROLLBACK` never touches it), sharing the same [ParameterSubstitution] + * walkers so `$1`, `$2`, ... resolve identically on both engines. + */ +class DiskExecutor(private val catalog: DiskCatalog) : SequenceAccess { + private val accessPlanner = DiskIndexAccessPlanner() + private val activeCtes = mutableListOf() + private val expandingViews = mutableSetOf() + + /** Guards against a trigger whose body (directly or transitively) fires itself forever - mirrors [kestrel.engine.Executor.triggerDepth]. */ + private var triggerDepth = 0 + + /** + * Postgres-session-scoped CURRVAL cache, ported 1:1 from [kestrel.engine.Executor. + * lastSequenceValues] - see that field's own doc for why it is keyed by [Sequence] identity + * rather than by name. + */ + private val lastSequenceValues = HashMap() + + override fun nextValue(sequenceName: String): Long { + val sequence = catalog.sequence(sequenceName) + return catalog.nextValue(sequenceName).also { lastSequenceValues[sequence] = it } + } + + override fun currentValue(sequenceName: String): Long { + val sequence = catalog.sequence(sequenceName) + return lastSequenceValues[sequence] + ?: throw DiskExecutorException("currval of sequence '$sequenceName' is not yet defined in this session - call NEXTVAL first") + } + + /** + * Session-scoped, ported 1:1 from [kestrel.engine.Executor.preparedStatements] - deliberately + * outside [DiskCatalog], since a prepared statement carries no data of its own to undo: it is + * unaffected by `COMMIT`/`ROLLBACK` on [explicitTxn] the same way it is unaffected by + * catalog-snapshot rollback on the in-memory engine. + */ + private val preparedStatements = linkedMapOf() + + val preparedStatementNames: List get() = preparedStatements.keys.toList() + + /** The disk transaction opened by a `BEGIN` still awaiting `COMMIT`/`ROLLBACK`, or null outside one. */ + private var explicitTxn: Long? = null + + /** Whether an explicit `BEGIN` is currently open - mirrors [kestrel.engine.Executor.inTransaction]. */ + val inTransaction: Boolean get() = explicitTxn != null + + /** + * Named savepoints established inside the currently open [explicitTxn], oldest first, each paired + * with the storage engine LSN [DiskCatalog.savepoint] captured for it - mirrors + * [kestrel.engine.Executor]'s own `savepoints` stack (catalog snapshots there, WAL positions here) + * including reuse-shadows-the-older-one-of-the-same-name semantics via [indexOfLastSavepoint]. + */ + private val savepoints = mutableListOf>() + + /** + * A temp table's rows live in a plain [Table.rows] list outside [DiskCatalog]'s own WAL/2PL + * transaction (see [DiskCatalog]'s KDoc on [DiskCatalog.temporaryTableValues]), so `ROLLBACK`/ + * `ROLLBACK TO SAVEPOINT` cannot undo a temp table's INSERT/UPDATE/DELETE the way they undo a + * real [DiskTable]'s writes just by aborting/rewinding [explicitTxn] - this stack is what makes + * them do it anyway. Index 0 is the snapshot taken at `BEGIN`; index `i + 1` is the snapshot + * taken when `savepoints[i]` was established, so `ROLLBACK` restores index 0 and `ROLLBACK TO + * SAVEPOINT` restores `temporaryTableSnapshots[index + 1]` - kept in lockstep with [savepoints] + * by every method below. Only *row contents* ever need restoring, never which temp tables exist: + * `CREATE`/`DROP TEMPORARY TABLE` are DDL, already rejected inside an explicit transaction (see + * [isDdl]), so the set of temp tables is fixed for the whole lifetime of one [explicitTxn]. + */ + private val temporaryTableSnapshots = mutableListOf>>() + + private fun snapshotTemporaryTables(): Map> = + catalog.temporaryTableValues.associateWith { it.rows.toList() } + + private fun restoreTemporaryTables(snapshot: Map>) { + for ((table, rows) in snapshot) { + table.rows.clear() + table.rows.addAll(rows) + } + } + + fun execute(statement: Statement): StatementResult { + if (explicitTxn != null && isDdl(statement)) { + throw DiskExecutorException( + "${statement::class.simpleName} cannot run inside an explicit transaction on the " + + "disk-backed executor - COMMIT or ROLLBACK first. Unlike the in-memory engine's " + + "whole-catalog snapshot/restore, an explicit transaction here is the storage " + + "engine's own WAL/2PL transaction (see BEGIN/COMMIT/ROLLBACK below), which has no " + + "way to undo a schema change recorded by DiskCatalog outside of it." + ) + } + return when (statement) { + is CreateTableStatement -> executeCreateTable(statement) + is DropTableStatement -> executeDropTable(statement) + is TruncateTableStatement -> executeTruncate(statement) + is AlterTableStatement -> executeAlterTable(statement) + is CreateIndexStatement -> executeCreateIndex(statement) + is DropIndexStatement -> executeDropIndex(statement) + is CreateViewStatement -> executeCreateView(statement) + is DropViewStatement -> executeDropView(statement) + is CreateSequenceStatement -> executeCreateSequence(statement) + is DropSequenceStatement -> executeDropSequence(statement) + is AlterSequenceStatement -> executeAlterSequence(statement) + is CreateTriggerStatement -> executeCreateTrigger(statement) + is DropTriggerStatement -> executeDropTrigger(statement) + is InsertStatement -> executeInsert(statement) + is MergeStatement -> executeMerge(statement) + is QueryStatement -> executeQuery(statement) + is ExplainStatement -> executeExplain(statement) + is UpdateStatement -> executeUpdate(statement) + is DeleteStatement -> executeDelete(statement) + is BeginStatement -> executeBegin() + is CommitStatement -> executeCommit() + is RollbackStatement -> executeRollback() + is SavepointStatement -> executeSavepoint(statement) + is ReleaseSavepointStatement -> executeReleaseSavepoint(statement) + is RollbackToSavepointStatement -> executeRollbackToSavepoint(statement) + is PrepareStatement -> executePrepare(statement) + is ExecuteStatement -> execute(resolveExecute(statement)) + is DeallocateStatement -> executeDeallocate(statement) + else -> throw DiskExecutorException( + "${statement::class.simpleName} is not supported by the disk-backed executor yet" + ) + } + } + + private fun isDdl(statement: Statement): Boolean = when (statement) { + is CreateTableStatement, is DropTableStatement, is AlterTableStatement, + is CreateIndexStatement, is DropIndexStatement, is CreateViewStatement, is DropViewStatement, + is CreateSequenceStatement, is DropSequenceStatement, is AlterSequenceStatement, + is CreateTriggerStatement, is DropTriggerStatement -> true + else -> false + } + + private fun executePrepare(statement: PrepareStatement): StatementResult { + val key = statement.name.lowercase() + if (preparedStatements.containsKey(key)) { + throw DiskExecutorException("prepared statement '${statement.name}' already exists") + } + preparedStatements[key] = statement.target + return PrepareResult(statement.name) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.resolveExecute] - looks [ExecuteStatement.name] up, + * evaluates every argument as a CONSTANT (a bare [Evaluator] with no resolver/subquery/sequence + * access, exactly like a DEFAULT expression), and substitutes the results for every `$1`, `$2`, + * ... via the same [ParameterSubstitution] both engines share. The result is always a concrete + * [PreparableStatement] - never [ExecuteStatement] itself. + */ + fun resolveExecute(statement: ExecuteStatement): Statement { + val target = preparedStatements[statement.name.lowercase()] + ?: throw DiskExecutorException("no such prepared statement: ${statement.name}") + val required = ParameterSubstitution.parameterCount(target) + if (statement.arguments.size != required) { + throw DiskExecutorException( + "prepared statement '${statement.name}' expects $required parameter(s) but got ${statement.arguments.size}" + ) + } + val values = statement.arguments.map { Evaluator().evaluate(it) } + return ParameterSubstitution.substitute(target, values) + } + + private fun executeDeallocate(statement: DeallocateStatement): StatementResult { + if (statement.all) { + preparedStatements.clear() + return DeallocateResult(null, all = true) + } + val name = statement.name!! + if (preparedStatements.remove(name.lowercase()) == null) { + throw DiskExecutorException("no such prepared statement: $name") + } + return DeallocateResult(name, all = false) + } + + private fun executeBegin(): StatementResult { + if (explicitTxn != null) throw DiskExecutorException("a transaction is already in progress") + explicitTxn = catalog.beginTransaction() + temporaryTableSnapshots.clear() + temporaryTableSnapshots.add(snapshotTemporaryTables()) + return BeginResult + } + + private fun executeCommit(): StatementResult { + val txn = explicitTxn ?: throw DiskExecutorException("no transaction is in progress") + catalog.commit(txn) + explicitTxn = null + savepoints.clear() + temporaryTableSnapshots.clear() + return CommitResult + } + + private fun executeRollback(): StatementResult { + val txn = explicitTxn ?: throw DiskExecutorException("no transaction is in progress") + catalog.abort(txn) + explicitTxn = null + savepoints.clear() + restoreTemporaryTables(temporaryTableSnapshots[0]) + temporaryTableSnapshots.clear() + return RollbackResult + } + + /** + * Establishing a savepoint costs nothing beyond capturing the storage engine's current WAL + * position and every open index's root pointer (see [DiskCatalog.savepoint]) plus a temp table + * row snapshot (see [temporaryTableSnapshots]) - unlike [kestrel.engine.Executor], there is no + * whole-CATALOG snapshot to take, since DDL (the only thing that mutates [catalog] itself rather + * than a table's data) is already rejected inside an explicit transaction on this executor. + */ + private fun executeSavepoint(statement: SavepointStatement): StatementResult { + val txn = explicitTxn ?: throw DiskExecutorException("SAVEPOINT can only be used inside a transaction") + savepoints.add(statement.name to catalog.savepoint(txn)) + temporaryTableSnapshots.add(snapshotTemporaryTables()) + return SavepointResult(statement.name) + } + + /** Forgets the named savepoint and every savepoint established after it, without undoing anything. */ + private fun executeReleaseSavepoint(statement: ReleaseSavepointStatement): StatementResult { + val txn = explicitTxn ?: throw DiskExecutorException("RELEASE SAVEPOINT can only be used inside a transaction") + val index = indexOfLastSavepoint(statement.name) + while (savepoints.size > index) { + val (_, lsn) = savepoints.removeAt(savepoints.size - 1) + catalog.discardSavepoint(txn, lsn) + temporaryTableSnapshots.removeAt(temporaryTableSnapshots.size - 1) + } + return ReleaseSavepointResult(statement.name) + } + + /** + * Undoes every write made since the named savepoint, but - matching [kestrel.engine.Executor] and + * real SQL savepoint semantics - keeps that savepoint itself usable for a later rollback, only + * discarding the ones established after it. + */ + private fun executeRollbackToSavepoint(statement: RollbackToSavepointStatement): StatementResult { + val txn = explicitTxn ?: throw DiskExecutorException("ROLLBACK TO SAVEPOINT can only be used inside a transaction") + val index = indexOfLastSavepoint(statement.name) + catalog.rollbackToSavepoint(txn, savepoints[index].second) + restoreTemporaryTables(temporaryTableSnapshots[index + 1]) + while (savepoints.size > index + 1) { + val (_, lsn) = savepoints.removeAt(savepoints.size - 1) + catalog.discardSavepoint(txn, lsn) + temporaryTableSnapshots.removeAt(temporaryTableSnapshots.size - 1) + } + return RollbackToSavepointResult(statement.name) + } + + private fun indexOfLastSavepoint(name: String): Int { + val index = savepoints.indexOfLast { it.first.equals(name, ignoreCase = true) } + if (index < 0) throw DiskExecutorException("no such savepoint: $name") + return index + } + + /** + * Runs [block] against the transaction opened by a `BEGIN` still in progress so several + * statements share one commit/abort boundary, or against a fresh transaction scoped to just this + * statement (committed on success, aborted on failure via [runInTransaction]) when there isn't + * one - the same per-statement auto-commit [kestrel.engine.Executor] gets for free from running + * directly against an in-memory [kestrel.catalog.Catalog] with no transaction concept at all. + * Locks taken by [block] are held by the shared transaction until that later COMMIT/ROLLBACK per + * strict two-phase locking (see [kestrel.store.txn.LockManager]), so a SELECT inside an open + * transaction can block a concurrent writer the way a real multi-statement transaction should. + */ + private fun withStatementTransaction(block: (Long) -> T): T { + val active = explicitTxn + if (active != null) return block(active) + val txn = catalog.beginTransaction() + return runInTransaction(txn) { block(txn) } + } + + /** + * Savepoint-based analog of [kestrel.engine.Executor.withStatementAtomicity]: without this, a + * BEFORE trigger's own INSERT/UPDATE/DELETE would survive even when the statement that fired it + * goes on to fail its own constraint check (or an AFTER trigger fails after the row already + * landed) - a real database treats a whole statement plus everything its triggers do as one + * atomic unit. Only the OUTERMOST statement in a trigger cascade takes the savepoint + * ([triggerContext] is null exactly there, mirroring the in-memory engine's own `triggerContext == + * null` check): [catalog.savepoint]/[catalog.rollbackToSavepoint] are keyed purely by WAL LSN, + * entirely independent of - and safely nestable inside - the executor's own named `SAVEPOINT` + * stack ([savepoints]), so this composes with an explicit transaction the user is already inside + * without disturbing it. Skipped entirely when [tableName] has no trigger at all, so the + * overwhelmingly common trigger-free INSERT/UPDATE/DELETE pays no savepoint cost. + */ + private fun withTriggerAtomicity(tableName: String, txn: Long, triggerContext: TriggerRowContext?, block: () -> T): T { + if (triggerContext != null || !catalog.hasTriggers(tableName)) return block() + val savepointLsn = catalog.savepoint(txn) + val result = try { + block() + } catch (failure: Exception) { + catalog.rollbackToSavepoint(txn, savepointLsn) + catalog.discardSavepoint(txn, savepointLsn) + throw failure + } + catalog.discardSavepoint(txn, savepointLsn) + return result + } + + /** + * Resolves `NEW.col`/`OLD.col` against [trigger] first (when both the qualifier and that side of + * the row are present); everything else - including a bare column with no qualifier at all - + * falls through to [base] exactly as it would with no trigger in play. Ported 1:1 from + * [kestrel.engine.Executor.composeResolver], just with [DiskExecutorException] for the fallthrough. + */ + private fun composeResolver(base: ColumnResolver?, trigger: TriggerRowContext?): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value = + if (trigger != null && trigger.matches(expression)) trigger.resolve(expression) + else base?.resolve(expression) ?: throw DiskExecutorException("column '${expression.name}' is not allowed here") + + override fun declaredCollation(expression: ColumnExpression): Collation? = + if (trigger != null && trigger.matches(expression)) trigger.declaredCollation(expression) + else base?.declaredCollation(expression) + } + + /** Ported 1:1 from [kestrel.engine.Executor.rowResolver] - a [ColumnResolver] [composeResolver] can layer a [TriggerRowContext] on top of. */ + private fun rowResolver(schema: TableSchema, row: Row): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val index = schema.columnIndex(expression.name) ?: throw DiskExecutorException("no such column: ${expression.name}") + return row.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = schema.column(expression.name)?.collation + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.evaluateReturning]: evaluates an INSERT/UPDATE/ + * DELETE statement's `RETURNING` clause against the rows it actually affected - the freshly + * inserted/updated row for INSERT/UPDATE, the pre-deletion row for DELETE. [checkReturningColumns] + * validates every projected expression up front so a bad column name is reported even when zero + * rows matched, not just lazily the first time [rowResolver] happens to see one. + */ + private fun evaluateReturning(schema: TableSchema, rows: List, projection: Projection, subqueries: SubqueryRunner): SelectResult { + val columns = when (projection) { + is StarProjection -> schema.columns.map { it.name } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> + checkReturningColumns(item.expression, schema) + columnLabel(item, index) + } + } + val resultRows = rows.map { row -> + when (projection) { + is StarProjection -> row.values.toList() + is ExpressionProjection -> { + val evaluator = Evaluator(rowResolver(schema, row), subqueries) + projection.items.map { evaluator.evaluate(it.expression) } + } + } + } + return SelectResult(columns, resultRows) + } + + /** + * Fires every trigger registered for ([schema], [timing], [event]) in creation order, once per + * (oldRow, newRow) pair in [rows] - mirrors [kestrel.engine.Executor.fireRowTriggers] exactly, + * including that a `WHEN` clause evaluating to false/NULL skips only that trigger for that row. + */ + private fun fireRowTriggers(schema: TableSchema, timing: TriggerTiming, event: TriggerEvent, rows: List>, txn: Long) { + val triggers = catalog.triggersOn(schema.name, timing, event) + if (triggers.isEmpty()) return + val subqueries = subqueryRunnerFor(txn) + for ((oldRow, newRow) in rows) { + val context = TriggerRowContext(schema, newRow, oldRow) + for (trigger in triggers) { + val whenClause = trigger.whenClause + if (whenClause != null) { + val evaluator = Evaluator(composeResolver(null, context), subqueries) + if (!isTrue(evaluator.evaluate(whenClause))) continue + } + runTriggerBody(trigger, context, txn) + } + } + } + + /** + * Runs a fired trigger's body statements in order, directly against [txn] - NOT through + * [executeInsert]/[executeUpdate]/[executeDelete] (which would re-select a transaction via + * [withStatementTransaction] and re-take a trigger-atomicity savepoint via + * [withTriggerAtomicity]) - so the whole cascade shares exactly one disk transaction and exactly + * one savepoint, taken once by the outermost caller. Mirrors [kestrel.engine.Executor. + * runTriggerBody], including [MAX_TRIGGER_DEPTH]. + */ + private fun runTriggerBody(trigger: TriggerDefinition, context: TriggerRowContext, txn: Long) { + if (triggerDepth >= MAX_TRIGGER_DEPTH) { + throw DiskExecutorException( + "trigger recursion limit ($MAX_TRIGGER_DEPTH) exceeded while firing '${trigger.name}' - " + + "check for a trigger whose body ends up re-triggering itself" + ) + } + triggerDepth++ + try { + for (bodyStatement in trigger.body) { + when (bodyStatement) { + is InsertStatement -> { + val table = catalog.table(bodyStatement.tableName) + executeInsertInTxn(bodyStatement, table, table.schema, txn, context) + } + is UpdateStatement -> { + val table = catalog.table(bodyStatement.tableName) + executeUpdateInTxn(bodyStatement, buildAssignments(table.schema, bodyStatement), table, table.schema, txn, context) + } + is DeleteStatement -> { + val table = catalog.table(bodyStatement.tableName) + executeDeleteInTxn(bodyStatement, table, table.schema, txn, context) + } + else -> error("unreachable: trigger bodies only ever contain INSERT/UPDATE/DELETE") + } + } + } finally { + triggerDepth-- + } + } + + private fun buildAssignments(schema: TableSchema, statement: UpdateStatement): List> = + statement.assignments.map { assignment -> + val index = schema.columnIndex(assignment.columnName) + ?: throw DiskExecutorException("no such column: ${assignment.columnName} in table ${schema.name}") + rejectGeneratedAssignment(schema.columns[index], explicitValueGiven = assignment.value != null) + index to assignment.value + } + + private fun executeCreateIndex(statement: CreateIndexStatement): StatementResult { + catalog.createIndex(statement.tableName, statement.indexName, statement.columnNames, statement.unique) + return CreateIndexResult(statement.indexName) + } + + private fun executeDropIndex(statement: DropIndexStatement): StatementResult { + catalog.dropIndex(statement.indexName) + return DropIndexResult(statement.indexName) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.executeCreateSequence] - see that method's own doc + * for the ascending/descending default resolution and the one thing [SequenceDefinition]'s own + * `init` cannot check itself (START WITH falling inside [MINVALUE, MAXVALUE]). + */ + private fun executeCreateSequence(statement: CreateSequenceStatement): StatementResult { + val incrementBy = statement.incrementBy ?: 1L + if (incrementBy == 0L) throw DiskExecutorException("sequence '${statement.sequenceName}' INCREMENT BY cannot be zero") + val ascending = incrementBy > 0 + val minValue = statement.minValue ?: if (ascending) 1L else Long.MIN_VALUE + val maxValue = statement.maxValue ?: if (ascending) Long.MAX_VALUE else -1L + if (minValue > maxValue) { + throw DiskExecutorException("sequence '${statement.sequenceName}' MINVALUE ($minValue) must not exceed MAXVALUE ($maxValue)") + } + val startValue = statement.startValue ?: if (ascending) minValue else maxValue + if (startValue < minValue || startValue > maxValue) { + throw DiskExecutorException( + "START WITH ($startValue) must be between MINVALUE ($minValue) and MAXVALUE ($maxValue)" + ) + } + catalog.createSequence(SequenceDefinition(statement.sequenceName, incrementBy, minValue, maxValue, startValue, statement.cycle)) + return CreateSequenceResult(statement.sequenceName) + } + + private fun executeDropSequence(statement: DropSequenceStatement): StatementResult { + catalog.dropSequence(statement.sequenceName) + return DropSequenceResult(statement.sequenceName) + } + + /** Ported 1:1 from [kestrel.engine.Executor.executeAlterSequence] - see that method's own doc. */ + private fun executeAlterSequence(statement: AlterSequenceStatement): StatementResult { + val existing = catalog.sequence(statement.sequenceName).definition + val incrementBy = statement.incrementBy ?: existing.incrementBy + if (incrementBy == 0L) throw DiskExecutorException("sequence '${statement.sequenceName}' INCREMENT BY cannot be zero") + val ascending = incrementBy > 0 + val minValue = when { + statement.clearMinValue -> if (ascending) 1L else Long.MIN_VALUE + statement.minValue != null -> statement.minValue + else -> existing.minValue + } + val maxValue = when { + statement.clearMaxValue -> if (ascending) Long.MAX_VALUE else -1L + statement.maxValue != null -> statement.maxValue + else -> existing.maxValue + } + if (minValue > maxValue) { + throw DiskExecutorException("sequence '${statement.sequenceName}' MINVALUE ($minValue) must not exceed MAXVALUE ($maxValue)") + } + val cycle = statement.cycle ?: existing.cycle + val newDefinition = SequenceDefinition(statement.sequenceName, incrementBy, minValue, maxValue, existing.startValue, cycle) + val restartAt = if (statement.restart) (statement.restartWith ?: existing.startValue) else null + if (restartAt != null && (restartAt < minValue || restartAt > maxValue)) { + throw DiskExecutorException("RESTART value ($restartAt) must be between MINVALUE ($minValue) and MAXVALUE ($maxValue)") + } + catalog.alterSequence(statement.sequenceName, newDefinition, restartAt) + return AlterSequenceResult(statement.sequenceName) + } + + /** + * Runs [CreateViewStatement.query] once, discarding the result, purely to fail fast on a bad + * column/table reference before anything is persisted - mirrors [kestrel.engine.Executor. + * executeCreateView] calling `selectRunner.explain` for the same reason, just via a real + * execution rather than a plan-only walk (this executor has no `explain`). + */ + private fun executeCreateView(statement: CreateViewStatement): StatementResult { + val txn = catalog.beginTransaction() + runInTransaction(txn) { runQuery(statement.query, txn, outer = null) } + catalog.createView(statement.viewName, statement.query, statement.orReplace) + return CreateViewResult(statement.viewName) + } + + private fun executeDropView(statement: DropViewStatement): StatementResult { + catalog.dropView(statement.viewName, cascade = statement.mode == DropMode.Cascade) + return DropViewResult(statement.viewName) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.executeCreateTrigger]: body statements are + * deliberately NOT deep-validated here (no eager check that every column referenced in a body + * statement's WHERE/SET/VALUES actually exists) - a trigger body can target a different table per + * statement, and this executor already defers exactly this kind of check to the moment the + * trigger actually fires, same as SQLite. What IS checked eagerly: the trigger's own table and + * every body statement's target table both exist right now, and the WHEN clause - evaluated + * directly by this executor rather than through a nested statement run, so nothing else will ever + * validate it - contains no subquery/aggregate and only ever references NEW/OLD columns that both + * exist on this table and are actually available for this trigger's event. + */ + private fun executeCreateTrigger(statement: CreateTriggerStatement): StatementResult { + val schema = catalog.table(statement.tableName).schema + statement.whenClause?.let { whenClause -> + requireNoSubquery(whenClause) + validateTriggerRowReferences(whenClause, statement.event, schema) + } + for (bodyStatement in statement.body) validateTriggerBodyStatementTarget(bodyStatement) + catalog.createTrigger( + TriggerDefinition(statement.triggerName, statement.timing, statement.event, statement.tableName, statement.whenClause, statement.body) + ) + return CreateTriggerResult(statement.triggerName) + } + + private fun validateTriggerBodyStatementTarget(statement: Statement) { + when (statement) { + is InsertStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + is UpdateStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + is DeleteStatement -> { catalog.table(statement.tableName); rejectReturning(statement.returning) } + else -> throw DiskExecutorException("trigger body only supports INSERT, UPDATE, or DELETE statements") + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.rejectReturning]: a trigger body statement's own [runTriggerBody] discards whatever [StatementResult] it returns, so a RETURNING clause there would compute output nobody could ever see. */ + private fun rejectReturning(returning: Projection?) { + if (returning != null) { + throw DiskExecutorException("RETURNING is not allowed in a trigger body statement - its result would never be visible to anything") + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.validateTriggerRowReferences]. */ + private fun validateTriggerRowReferences(expression: Expression, event: TriggerEvent, schema: TableSchema) { + when (expression) { + is ColumnExpression -> { + val qualifier = expression.qualifier + ?: throw DiskExecutorException("column '${expression.name}' in trigger WHEN clause must be qualified with NEW or OLD") + val isNew = qualifier.equals("new", ignoreCase = true) + val isOld = qualifier.equals("old", ignoreCase = true) + if (!isNew && !isOld) throw DiskExecutorException("no such table or alias: $qualifier") + val availableForEvent = when (event) { + TriggerEvent.Insert -> isNew + TriggerEvent.Delete -> isOld + TriggerEvent.Update -> true + } + if (!availableForEvent) { + throw DiskExecutorException("${if (isNew) "NEW" else "OLD"} is not available in a trigger ON ${event.name.uppercase()}") + } + if (schema.columnIndex(expression.name) == null) { + throw DiskExecutorException("no such column: ${expression.name}") + } + } + is BinaryExpression -> { + validateTriggerRowReferences(expression.left, event, schema) + validateTriggerRowReferences(expression.right, event, schema) + } + is UnaryExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is CallExpression -> expression.arguments.forEach { validateTriggerRowReferences(it, event, schema) } + is IsNullExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is InExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + expression.values.forEach { validateTriggerRowReferences(it, event, schema) } + } + is BetweenExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + validateTriggerRowReferences(expression.low, event, schema) + validateTriggerRowReferences(expression.high, event, schema) + } + is LikeExpression -> { + validateTriggerRowReferences(expression.operand, event, schema) + validateTriggerRowReferences(expression.pattern, event, schema) + } + is CaseExpression -> { + expression.operand?.let { validateTriggerRowReferences(it, event, schema) } + expression.branches.forEach { + validateTriggerRowReferences(it.condition, event, schema) + validateTriggerRowReferences(it.result, event, schema) + } + expression.elseExpression?.let { validateTriggerRowReferences(it, event, schema) } + } + is CastExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is CollateExpression -> validateTriggerRowReferences(expression.operand, event, schema) + is LiteralExpression -> Unit + is FunctionExpression, is WindowExpression, is ScalarSubqueryExpression, + is ExistsExpression, is InSubqueryExpression, + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> + error("unreachable: banned by requireNoSubquery") + } + } + + private fun executeDropTrigger(statement: DropTriggerStatement): StatementResult { + catalog.dropTrigger(statement.triggerName) + return DropTriggerResult(statement.triggerName) + } + + private fun executeCreateTable(statement: CreateTableStatement): StatementResult { + if (statement.temporary) return executeCreateTemporaryTable(statement) + if (statement.columns.count { it.primaryKey } > 1 || + (statement.compositePrimaryKey.isNotEmpty() && statement.columns.any { it.primaryKey }) + ) { + throw DiskExecutorException("table ${statement.tableName} declares more than one primary key") + } + validateUniqueDefinitions(statement) + validateCheckDefinitions(statement) + val columns = applyCompositePrimaryKey(statement) + validateForeignKeyDefinitions(statement, columns) + for (column in columns) { + when (val default = column.default) { + is NextValueExpression -> validateNextValueDefault(default, column) + is CurrentValueExpression -> throw DiskExecutorException("DEFAULT expression cannot contain CURRVAL") + else -> Unit + } + } + validateGeneratedDefinitions(columns) + catalog.createTable( + statement.tableName, columns, statement.compositePrimaryKey, statement.uniqueConstraints, + statement.checks, statement.foreignKeys + ) + return CreateTableResult(statement.tableName) + } + + /** + * `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` - the disk-backed executor's counterpart of + * [executeCreateTable], but building a [kestrel.storage.Table] via [DiskCatalog.createTemporaryTable] + * instead of a [DiskTable]: a real, live table for the rest of this process's lifetime (INSERT/ + * SELECT/UPDATE/DELETE, WHERE/JOIN/ORDER BY/subqueries, NOT NULL/PRIMARY KEY/UNIQUE/CHECK/DEFAULT/ + * GENERATED all enforced exactly like a permanent table - see [executeTemporaryInsert]/ + * [executeTemporaryUpdate]/[executeTemporaryDelete] and [resolveFromSource]), simply never + * touching the pager/WAL and never surviving a reopen. FOREIGN KEY is rejected outright rather + * than silently ignored: [DiskForeignKeyEnforcer] and its cascade bookkeeping are wired to + * [DiskTable]/[DiskCatalog]'s directory, which a temp table deliberately has no entry in, so + * honoring the constraint would need a parallel enforcement path this session scoped out - a + * named, documented scope-cut rather than a constraint that looks declared but is never checked. + */ + private fun executeCreateTemporaryTable(statement: CreateTableStatement): StatementResult { + if (statement.foreignKeys.isNotEmpty()) { + throw DiskExecutorException("FOREIGN KEY on a temporary table is not supported by the disk-backed executor yet") + } + if (statement.columns.count { it.primaryKey } > 1 || + (statement.compositePrimaryKey.isNotEmpty() && statement.columns.any { it.primaryKey }) + ) { + throw DiskExecutorException("table ${statement.tableName} declares more than one primary key") + } + validateUniqueDefinitions(statement) + validateCheckDefinitions(statement) + val columns = applyCompositePrimaryKey(statement) + for (column in columns) { + when (val default = column.default) { + is NextValueExpression -> validateNextValueDefault(default, column) + is CurrentValueExpression -> throw DiskExecutorException("DEFAULT expression cannot contain CURRVAL") + else -> Unit + } + } + validateGeneratedDefinitions(columns) + val schema = TableSchema( + statement.tableName, columns, + checks = statement.checks, + uniqueConstraints = statement.uniqueConstraints, + compositePrimaryKey = statement.compositePrimaryKey, + temporary = true + ) + catalog.createTemporaryTable(schema) + return CreateTableResult(statement.tableName) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.validateNextValueDefault]: checked statically rather + * than by evaluating the DEFAULT for real, since a bare `CREATE TABLE` must not itself consume a + * sequence value before a single row exists. + */ + private fun validateNextValueDefault(default: NextValueExpression, column: Column) { + catalog.sequence(default.sequenceName) + if (column.type != ColumnType.Integer) { + throw DiskExecutorException("NEXTVAL can only be used as a DEFAULT for an INTEGER column, but ${column.name} is ${column.type.displayName}") + } + } + + /** + * Deliberately duplicated from [kestrel.engine.Executor.validateForeignKeyDefinitions] (same + * reasoning as [coerceForColumn]): [columns] is already normalized by [applyCompositePrimaryKey], + * matching the in-memory engine's own call order. A self-reference (`referencedTable` equal to + * the table being created) is checked against a [TableSchema] built from [columns] rather than + * the catalog, because the table doesn't exist there yet while it's being created. + */ + private fun validateForeignKeyDefinitions(statement: CreateTableStatement, columns: List) { + for (fk in statement.foreignKeys) { + if (fk.columns.size != fk.referencedColumns.size) { + throw DiskExecutorException( + "FOREIGN KEY (${fk.columns.joinToString(", ")}) column count does not match " + + "REFERENCES ${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")})" + ) + } + for (columnName in fk.columns) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw DiskExecutorException("FOREIGN KEY refers to unknown column: $columnName") + } + } + val selfReference = fk.referencedTable.equals(statement.tableName, ignoreCase = true) + val hasEnforcedUniqueness = if (selfReference) { + for (columnName in fk.referencedColumns) { + if (columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw DiskExecutorException("FOREIGN KEY references unknown column: ${fk.referencedTable}.$columnName") + } + } + val selfSchema = TableSchema( + statement.tableName, columns, + uniqueConstraints = statement.uniqueConstraints, + compositePrimaryKey = statement.compositePrimaryKey + ) + selfSchema.hasEnforcedUniqueness(fk.referencedColumns) + } else { + val referencedTable = catalog.tableOrNull(fk.referencedTable) + ?: throw DiskExecutorException("FOREIGN KEY references unknown table: ${fk.referencedTable}") + for (columnName in fk.referencedColumns) { + if (referencedTable.schema.column(columnName) == null) { + throw DiskExecutorException("FOREIGN KEY references unknown column: ${fk.referencedTable}.$columnName") + } + } + referencedTable.schema.hasEnforcedUniqueness(fk.referencedColumns) + } + if (!hasEnforcedUniqueness) { + throw DiskExecutorException( + "FOREIGN KEY must reference a PRIMARY KEY or UNIQUE column(s), but " + + "${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")}) is not one" + ) + } + } + } + + /** + * Deliberately duplicated from [kestrel.engine.Executor.validateCheckDefinitions] (same + * reasoning as [coerceForColumn]): every CHECK expression must only reference a real column of + * the table being created and must contain no subquery, aggregate, or window function - there is + * no row yet to run one against, and this executor's [Evaluator] has no aggregation pass of its + * own outside [GroupAggregation]. + */ + private fun validateCheckDefinitions(statement: CreateTableStatement) { + val schema = TableSchema(statement.tableName, statement.columns) + for (check in statement.checks) { + checkColumns(check.expression, schema) + requireNoSubquery(check.expression) + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.checkColumns] - see [validateCheckDefinitions]. */ + private fun checkColumns(expression: Expression, schema: TableSchema) { + when (expression) { + is ColumnExpression -> if (schema.columnIndex(expression.name) == null) { + throw DiskExecutorException("no such column: ${expression.name}") + } + is BinaryExpression -> { + checkColumns(expression.left, schema) + checkColumns(expression.right, schema) + } + is UnaryExpression -> checkColumns(expression.operand, schema) + is FunctionExpression -> throw DiskExecutorException("aggregate functions are not allowed here") + is WindowExpression -> throw DiskExecutorException("window functions are not allowed here") + is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { checkColumns(it, schema) } + is IsNullExpression -> checkColumns(expression.operand, schema) + is InExpression -> { + checkColumns(expression.operand, schema) + expression.values.forEach { checkColumns(it, schema) } + } + is BetweenExpression -> { + checkColumns(expression.operand, schema) + checkColumns(expression.low, schema) + checkColumns(expression.high, schema) + } + is LikeExpression -> { + checkColumns(expression.operand, schema) + checkColumns(expression.pattern, schema) + } + is CaseExpression -> { + expression.operand?.let { checkColumns(it, schema) } + expression.branches.forEach { + checkColumns(it.condition, schema) + checkColumns(it.result, schema) + } + expression.elseExpression?.let { checkColumns(it, schema) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> checkColumns(expression.operand, schema) + is CastExpression -> checkColumns(expression.operand, schema) + is CollateExpression -> checkColumns(expression.operand, schema) + is NextValueExpression, is CurrentValueExpression -> + throw DiskExecutorException("NEXTVAL/CURRVAL are not allowed in a CHECK constraint or GENERATED column expression") + is MergeActionExpression -> throw DiskExecutorException("MERGE_ACTION() is not allowed in a CHECK constraint or GENERATED column expression") + is ParameterExpression -> throw DiskExecutorException("a parameter reference is not allowed in a CHECK constraint or GENERATED column expression") + } + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.checkColumns] (the general-purpose variant used for + * WHERE/SET/RETURNING) - NOT [checkColumns] above, which is CHECK-constraint specific and + * rejects NEXTVAL/CURRVAL outright. Used only by [evaluateReturning]: validates every column + * reference in a RETURNING projection item exists on [schema] and that no aggregate/window + * function sneaks in. + */ + private fun checkReturningColumns(expression: Expression, schema: TableSchema) { + when (expression) { + is ColumnExpression -> if (schema.columnIndex(expression.name) == null) { + throw DiskExecutorException("no such column: ${expression.name}") + } + is BinaryExpression -> { + checkReturningColumns(expression.left, schema) + checkReturningColumns(expression.right, schema) + } + is UnaryExpression -> checkReturningColumns(expression.operand, schema) + is FunctionExpression -> throw DiskExecutorException("aggregate functions are not allowed here") + is WindowExpression -> throw DiskExecutorException("window functions are not allowed here") + is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { checkReturningColumns(it, schema) } + is IsNullExpression -> checkReturningColumns(expression.operand, schema) + is InExpression -> { + checkReturningColumns(expression.operand, schema) + expression.values.forEach { checkReturningColumns(it, schema) } + } + is BetweenExpression -> { + checkReturningColumns(expression.operand, schema) + checkReturningColumns(expression.low, schema) + checkReturningColumns(expression.high, schema) + } + is LikeExpression -> { + checkReturningColumns(expression.operand, schema) + checkReturningColumns(expression.pattern, schema) + } + is CaseExpression -> { + expression.operand?.let { checkReturningColumns(it, schema) } + expression.branches.forEach { + checkReturningColumns(it.condition, schema) + checkReturningColumns(it.result, schema) + } + expression.elseExpression?.let { checkReturningColumns(it, schema) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> checkReturningColumns(expression.operand, schema) + is CastExpression -> checkReturningColumns(expression.operand, schema) + is CollateExpression -> checkReturningColumns(expression.operand, schema) + // Structurally harmless (no column reference to check) - whether MERGE_ACTION() is + // actually legal at this particular call site (MERGE's own RETURNING: yes; plain INSERT/ + // UPDATE/DELETE's RETURNING: no) is enforced by whether the Evaluator this feeds into was + // actually given a mergeAction, exactly like NEXTVAL/CURRVAL is enforced by SequenceAccess - + // see evaluateMergeReturning, the one call site that binds one. + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> Unit + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.requireNoSubquery] - see [validateCheckDefinitions]. */ + private fun requireNoSubquery(expression: Expression) { + when (expression) { + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression -> + throw DiskExecutorException("CHECK constraints and GENERATED column expressions cannot contain subqueries") + is FunctionExpression -> throw DiskExecutorException("aggregate functions are not allowed in CHECK constraints") + is BinaryExpression -> { + requireNoSubquery(expression.left) + requireNoSubquery(expression.right) + } + is UnaryExpression -> requireNoSubquery(expression.operand) + is CallExpression -> expression.arguments.forEach { requireNoSubquery(it) } + is IsNullExpression -> requireNoSubquery(expression.operand) + is InExpression -> { + requireNoSubquery(expression.operand) + expression.values.forEach { requireNoSubquery(it) } + } + is BetweenExpression -> { + requireNoSubquery(expression.operand) + requireNoSubquery(expression.low) + requireNoSubquery(expression.high) + } + is LikeExpression -> { + requireNoSubquery(expression.operand) + requireNoSubquery(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { requireNoSubquery(it) } + expression.branches.forEach { + requireNoSubquery(it.condition) + requireNoSubquery(it.result) + } + expression.elseExpression?.let { requireNoSubquery(it) } + } + is CastExpression -> requireNoSubquery(expression.operand) + is CollateExpression -> requireNoSubquery(expression.operand) + is WindowExpression -> throw DiskExecutorException("window functions are not allowed in CHECK constraints") + is NextValueExpression, is CurrentValueExpression -> + throw DiskExecutorException("NEXTVAL/CURRVAL are not allowed in a CHECK constraint or GENERATED column expression") + is MergeActionExpression -> throw DiskExecutorException("MERGE_ACTION() is not allowed in a CHECK constraint or GENERATED column expression") + is ParameterExpression -> throw DiskExecutorException("a parameter reference is not allowed in a CHECK constraint or GENERATED column expression") + is ColumnExpression, is LiteralExpression -> Unit + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.validateGeneratedDefinitions] - see that method's own doc. */ + private fun validateGeneratedDefinitions(columns: List) { + val schema = TableSchema("", columns) + val generatedNames = columns.filter { it.generated != null }.map { it.name.lowercase() }.toSet() + for (column in columns) { + val generated = column.generated ?: continue + if (column.default != null) { + throw DiskExecutorException("column ${column.name} cannot declare both DEFAULT and GENERATED ALWAYS AS") + } + checkColumns(generated, schema) + requireNoSubquery(generated) + requireNoGeneratedColumnReference(generated, generatedNames, column.name) + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.requireNoGeneratedColumnReference]. */ + private fun requireNoGeneratedColumnReference(expression: Expression, generatedNames: Set, ownerColumn: String) { + when (expression) { + is ColumnExpression -> if (expression.name.lowercase() in generatedNames) { + throw DiskExecutorException( + "GENERATED column $ownerColumn cannot reference another GENERATED column: ${expression.name}" + ) + } + is LiteralExpression -> Unit + is BinaryExpression -> { + requireNoGeneratedColumnReference(expression.left, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.right, generatedNames, ownerColumn) + } + is UnaryExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is CallExpression -> expression.arguments.forEach { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + is IsNullExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is InExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + expression.values.forEach { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + } + is BetweenExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.low, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.high, generatedNames, ownerColumn) + } + is LikeExpression -> { + requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(expression.pattern, generatedNames, ownerColumn) + } + is CaseExpression -> { + expression.operand?.let { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + expression.branches.forEach { + requireNoGeneratedColumnReference(it.condition, generatedNames, ownerColumn) + requireNoGeneratedColumnReference(it.result, generatedNames, ownerColumn) + } + expression.elseExpression?.let { requireNoGeneratedColumnReference(it, generatedNames, ownerColumn) } + } + is CastExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is CollateExpression -> requireNoGeneratedColumnReference(expression.operand, generatedNames, ownerColumn) + is FunctionExpression, is WindowExpression, is ScalarSubqueryExpression, is ExistsExpression, + is InSubqueryExpression, is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, + is ParameterExpression -> Unit + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.applyGeneratedColumns]. */ + private fun applyGeneratedColumns(schema: TableSchema, values: MutableList, subqueries: SubqueryRunner) { + if (schema.columns.none { it.generated != null }) return + val evaluator = Evaluator(schema, Row(values), subqueries) + for ((index, column) in schema.columns.withIndex()) { + val generated = column.generated ?: continue + values[index] = coerceForColumn(evaluator.evaluate(generated), column) + } + } + + /** Ported 1:1 from [kestrel.engine.Executor.rejectGeneratedAssignment]. */ + private fun rejectGeneratedAssignment(column: Column, explicitValueGiven: Boolean) { + if (explicitValueGiven && column.generated != null) { + throw DiskExecutorException("cannot assign an explicit value to GENERATED column ${column.name}") + } + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.enforceChecks]: a CHECK expression that evaluates to + * SQL `FALSE` rejects the row; `TRUE` or `NULL` (three-valued "unknown", e.g. a comparison against + * a NULL column) both accept it, per standard SQL. + */ + private fun enforceChecks(schema: TableSchema, row: Row, subqueries: SubqueryRunner) { + for (check in schema.checks) { + val result = Evaluator(schema, row, subqueries).evaluate(check.expression) + if (result is Value.BooleanValue && !result.value) { + val label = check.name?.let { " $it" } ?: "" + throw DiskExecutorException("CHECK constraint$label failed: ${schema.name}") + } + } + } + + /** + * NOT NULL enforcement for a temp table row - [DiskTable.insert]/[DiskTable.update] get this for + * free from their own private `requireArity`, but a temp table's plain [kestrel.storage.Table] + * has no such gate, so [executeTemporaryInsert]/[executeTemporaryUpdate] call this directly. + * Ported 1:1 from [kestrel.engine.Executor.validateNotNull]. + */ + private fun validateTemporaryNotNull(schema: TableSchema, values: List) { + for ((index, column) in schema.columns.withIndex()) { + if (column.notNull && values[index] is Value.NullValue) { + throw DiskExecutorException("NOT NULL constraint failed: ${schema.name}.${column.name}") + } + } + } + + /** + * PRIMARY KEY uniqueness for a temp table, checked by full scan over [rows] (the whole candidate + * set post-mutation, not just the newly touched rows - same reasoning as + * [kestrel.engine.Executor.enforcePrimaryKeyUniqueness]) since a temp table has no B+Tree to + * enforce it structurally the way [DiskTable.rejectDuplicatePrimaryKey] does. Ported 1:1 from + * that same in-memory method, [DiskExecutorException] aside. + */ + private fun enforceTemporaryPrimaryKeyUniqueness(schema: TableSchema, rows: List) { + val indices = schema.primaryKeyIndices + if (indices.isEmpty()) return + val seen = HashSet>() + for (row in rows) { + val key = collationFoldedUniquenessKey(schema, indices, indices.map { row.values[it] }) + if (!seen.add(key)) { + val columnList = indices.joinToString(", ") { schema.columns[it].name } + throw DiskExecutorException("UNIQUE constraint failed: ${schema.name}($columnList)") + } + } + } + + /** UNIQUE-constraint counterpart of [enforceTemporaryPrimaryKeyUniqueness] - ported 1:1 from [kestrel.engine.Executor.enforceUniqueConstraints]. */ + private fun enforceTemporaryUniqueConstraints(schema: TableSchema, rows: List) { + for (unique in schema.uniqueConstraints) { + val columnIndices = unique.columns.map { schema.columnIndex(it)!! } + val seen = HashSet>() + for (row in rows) { + val rawKey = columnIndices.map { row.values[it] } + if (rawKey.any { it is Value.NullValue }) continue + if (!seen.add(collationFoldedUniquenessKey(schema, columnIndices, rawKey))) { + val label = unique.name?.let { " $it" } ?: "" + val columnList = unique.columns.joinToString(", ") + throw DiskExecutorException("UNIQUE constraint$label failed: ${schema.name}($columnList)") + } + } + } + } + + /** + * Deliberately duplicated from [kestrel.engine.Executor.validateUniqueDefinitions] (same + * reasoning as [coerceForColumn]): only checks that every referenced column exists, mirroring + * the in-memory engine exactly - it does not reject a constraint naming the same column twice + * (`UNIQUE (a, a)`), because the in-memory engine does not either. + */ + private fun validateUniqueDefinitions(statement: CreateTableStatement) { + for (unique in statement.uniqueConstraints) { + for (columnName in unique.columns) { + if (statement.columns.none { it.name.equals(columnName, ignoreCase = true) }) { + throw DiskExecutorException("UNIQUE constraint refers to unknown column: $columnName") + } + } + } + } + + /** + * A table-level `PRIMARY KEY (a, b, ...)` implies NOT NULL on every listed column, exactly + * like the inline `col TYPE PRIMARY KEY` spelling already does. Deliberately duplicated from + * [kestrel.engine.Executor.applyCompositePrimaryKey] (same reasoning as [coerceForColumn]): + * this executor reuses the in-memory engine's AST/types but must stay free-standing so a + * change to the in-memory engine's error handling can never silently affect the disk-backed + * path or vice versa. + */ + private fun applyCompositePrimaryKey(statement: CreateTableStatement): List { + if (statement.compositePrimaryKey.isEmpty()) return statement.columns + val declared = statement.compositePrimaryKey.map { it.lowercase() }.toSet() + if (declared.size != statement.compositePrimaryKey.size) { + throw DiskExecutorException("table '${statement.tableName}' declares a duplicate column in its PRIMARY KEY") + } + for (name in statement.compositePrimaryKey) { + if (statement.columns.none { it.name.equals(name, ignoreCase = true) }) { + throw DiskExecutorException("PRIMARY KEY refers to unknown column: $name") + } + } + return statement.columns.map { column -> + if (column.name.lowercase() in declared) column.copy(notNull = true) else column + } + } + + private fun executeDropTable(statement: DropTableStatement): StatementResult { + if (catalog.temporaryTableOrNull(statement.tableName) != null) { + catalog.dropTemporaryTable(statement.tableName) + } else { + catalog.dropTable(statement.tableName, cascade = statement.mode == DropMode.Cascade) + } + return DropTableResult(statement.tableName) + } + + /** + * Deliberately NOT in [isDdl] - like DELETE, TRUNCATE's row-clearing is fully covered by the + * storage engine's own WAL/2PL transaction, so it may run inside an explicit `BEGIN` and roll + * back cleanly with everything else that transaction did. `RESTART IDENTITY` is the one + * exception: [DiskCatalog.alterSequence] always opens and commits its own transaction (mirroring + * [DiskCatalog.nextValue]'s already-non-transactional sequence position - see that method's own + * doc), so a sequence restarted here stays restarted even if the enclosing statement transaction + * later rolls back. This is a deliberate, bounded asymmetry with [kestrel.engine.Executor]'s + * in-memory engine, where RESTART IDENTITY *is* undone by ROLLBACK because ALTER SEQUENCE there + * mutates a map entry covered by the whole-catalog snapshot/restore (see + * [kestrel.catalog.Catalog.truncateTables]). + */ + private fun executeTruncate(statement: TruncateTableStatement): StatementResult { + val truncated = withStatementTransaction { txn -> + catalog.truncateTables(txn, statement.tableNames, statement.mode == DropMode.Cascade) + } + if (statement.restartIdentity) { + for (name in truncated) restartOwnedSequences(catalog.table(name).schema) + } + return TruncateTableResult(truncated) + } + + /** Ported 1:1 from [kestrel.catalog.Catalog.restartOwnedSequences] - see that method's own doc for why a NEXTVAL DEFAULT is this engine's only signal for "this column is identity-backed". */ + private fun restartOwnedSequences(schema: TableSchema) { + for (column in schema.columns) { + val default = column.default + if (default is NextValueExpression) { + val sequence = catalog.sequenceOrNull(default.sequenceName) ?: continue + catalog.alterSequence(default.sequenceName, sequence.definition, restartAt = sequence.definition.startValue) + } + } + } + + private fun executeAlterTable(statement: AlterTableStatement): StatementResult { + when (val action = statement.action) { + is AddColumn -> executeAddColumn(statement.tableName, action.column) + is DropColumn -> catalog.dropColumn(statement.tableName, action.columnName, cascade = action.mode == DropMode.Cascade) + is RenameColumn -> catalog.renameColumn(statement.tableName, action.oldName, action.newName) + is RenameTable -> catalog.renameTable(statement.tableName, action.newName) + is AddConstraint -> executeAddConstraint(statement.tableName, action.constraint) + is DropConstraint -> catalog.dropConstraint(statement.tableName, action.name, cascade = action.mode == DropMode.Cascade) + is ChangeColumnType -> executeChangeColumnType(statement.tableName, action) + } + val resultName = (statement.action as? RenameTable)?.newName ?: statement.tableName + return AlterTableResult(resultName) + } + + /** Mirrors [kestrel.engine.Executor.executeAddConstraint] - see that method's own doc. */ + private fun executeAddConstraint(tableName: String, added: AddedConstraint) { + when (added) { + is AddedCheck -> executeAddCheckConstraint(tableName, added.check) + is AddedUnique -> executeAddUniqueConstraint(tableName, added.unique) + is AddedForeignKey -> executeAddForeignKeyConstraint(tableName, added.foreignKey) + } + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name CHECK (...)` - mirrors + * [kestrel.engine.Executor.executeAddCheckConstraint]: validated exactly like a CREATE TABLE-time + * CHECK ([validateCheckDefinitions]: known columns only, no subquery), PLUS every existing row is + * evaluated against it up front (under its own probe transaction, mirroring [executeAddColumn]'s + * `probeTxn`) before the constraint is installed via [DiskCatalog.addCheckConstraint]. + */ + private fun executeAddCheckConstraint(tableName: String, check: CheckConstraint) { + val table = catalog.table(tableName) + val schema = table.schema + check.name?.let { requireUniqueConstraintName(schema, it) } + checkColumns(check.expression, schema) + requireNoSubquery(check.expression) + val probeTxn = catalog.beginTransaction() + runInTransaction(probeTxn) { + val subqueries = subqueryRunnerFor(probeTxn) + for ((_, row) in table.scan(probeTxn)) { + val result = Evaluator(schema, row, subqueries).evaluate(check.expression) + if (result is Value.BooleanValue && !result.value) { + val label = check.name?.let { " $it" } ?: "" + throw DiskExecutorException("CHECK constraint$label failed: an existing row in $tableName violates it") + } + } + } + catalog.addCheckConstraint(tableName, check) + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name UNIQUE (...)` - mirrors + * [kestrel.engine.Executor.executeAddUniqueConstraint]: every existing row is checked for a + * duplicate combination of [unique]'s columns (folded through [collationFoldedUniquenessKey], NULL + * exempt per standard SQL) under its own probe transaction before the constraint is installed. + */ + private fun executeAddUniqueConstraint(tableName: String, unique: UniqueConstraint) { + val table = catalog.table(tableName) + val schema = table.schema + unique.name?.let { requireUniqueConstraintName(schema, it) } + for (columnName in unique.columns) { + if (schema.columnIndex(columnName) == null) { + throw DiskExecutorException("UNIQUE constraint refers to unknown column: $columnName") + } + } + val columnIndices = unique.columns.map { schema.columnIndex(it)!! } + val probeTxn = catalog.beginTransaction() + runInTransaction(probeTxn) { + val seen = HashSet>() + for ((_, row) in table.scan(probeTxn)) { + val rawKey = columnIndices.map { row.values[it] } + if (rawKey.any { it is Value.NullValue }) continue + if (!seen.add(collationFoldedUniquenessKey(schema, columnIndices, rawKey))) { + val label = unique.name?.let { " $it" } ?: "" + throw DiskExecutorException( + "UNIQUE constraint$label failed: an existing row combination in " + + "$tableName(${unique.columns.joinToString(", ")}) is already duplicated" + ) + } + } + } + catalog.addUniqueConstraint(tableName, unique) + } + + /** + * `ALTER TABLE t ADD CONSTRAINT name FOREIGN KEY (...) REFERENCES ...` - mirrors + * [kestrel.engine.Executor.executeAddForeignKeyConstraint]: validated the same way + * [validateForeignKeyDefinitions] validates one at CREATE TABLE time, PLUS every existing row of + * [tableName] is checked against it via [DiskForeignKeyEnforcer.validateRow] (under its own probe + * transaction) against a throwaway [TableSchema] carrying [foreignKey] alongside every FOREIGN KEY + * [tableName] already has, so a self-referencing FOREIGN KEY can find its own sibling rows. + */ + private fun executeAddForeignKeyConstraint(tableName: String, foreignKey: ForeignKeyConstraint) { + val table = catalog.table(tableName) + val schema = table.schema + foreignKey.name?.let { requireUniqueConstraintName(schema, it) } + if (foreignKey.columns.size != foreignKey.referencedColumns.size) { + throw DiskExecutorException( + "FOREIGN KEY (${foreignKey.columns.joinToString(", ")}) column count does not match " + + "REFERENCES ${foreignKey.referencedTable}(${foreignKey.referencedColumns.joinToString(", ")})" + ) + } + for (columnName in foreignKey.columns) { + if (schema.columnIndex(columnName) == null) { + throw DiskExecutorException("FOREIGN KEY refers to unknown column: $columnName") + } + } + val selfReference = foreignKey.referencedTable.equals(tableName, ignoreCase = true) + val referencedSchema = if (selfReference) { + schema + } else { + catalog.tableOrNull(foreignKey.referencedTable)?.schema + ?: throw DiskExecutorException("FOREIGN KEY references unknown table: ${foreignKey.referencedTable}") + } + for (columnName in foreignKey.referencedColumns) { + if (referencedSchema.column(columnName) == null) { + throw DiskExecutorException("FOREIGN KEY references unknown column: ${foreignKey.referencedTable}.$columnName") + } + } + if (!referencedSchema.hasEnforcedUniqueness(foreignKey.referencedColumns)) { + throw DiskExecutorException( + "FOREIGN KEY must reference a PRIMARY KEY or UNIQUE column(s), but " + + "${foreignKey.referencedTable}(${foreignKey.referencedColumns.joinToString(", ")}) is not one" + ) + } + val tempSchema = TableSchema( + schema.name, schema.columns, schema.foreignKeys + foreignKey, schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + val probeTxn = catalog.beginTransaction() + runInTransaction(probeTxn) { + for ((_, row) in table.scan(probeTxn)) { + DiskForeignKeyEnforcer.validateRow(catalog, probeTxn, tempSchema, row.values) + } + } + catalog.addForeignKey(tableName, foreignKey) + } + + /** + * Mirrors [kestrel.engine.Executor.executeChangeColumnType] - same RESTRICT-only dependency + * checks (PRIMARY KEY/FOREIGN KEY on either side/UNIQUE constraint/secondary index/GENERATED, + * all rejected outright rather than threading a type change through them) and the same DEFAULT + * compatibility pre-check. The one structural difference: [DiskCatalog.changeColumnType]'s + * `valueForRow` callback runs DURING the real per-row rewrite (via [DiskTable.rewriteRows]), not + * before it - see that method's own doc for why a NOT NULL/CHECK failure discovered mid-rewrite + * is still safe (transaction abort unwinds it), the same contract a NOT NULL `GENERATED ALWAYS + * AS` backfill already relies on. + */ + private fun executeChangeColumnType(tableName: String, action: ChangeColumnType) { + val table = catalog.table(tableName) + val schema = table.schema + val columnIndex = schema.columnIndex(action.columnName) + ?: throw DiskExecutorException("no such column: ${action.columnName} in table $tableName") + val column = schema.columns[columnIndex] + if (column.generated != null) { + throw DiskExecutorException( + "cannot change the type of GENERATED column ${action.columnName} in $tableName; drop and re-add it instead" + ) + } + if (schema.primaryKeyColumns.any { it.equals(action.columnName, ignoreCase = true) }) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName} because it is part of the PRIMARY KEY of $tableName" + ) + } + if (schema.foreignKeys.any { fk -> fk.columns.any { it.equals(action.columnName, ignoreCase = true) } }) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName} because it is used by one of ${tableName}'s own " + + "FOREIGN KEY constraints; drop the constraint first" + ) + } + val referencingTables = catalog.tableNames.filter { other -> + !other.equals(tableName, ignoreCase = true) && + catalog.tableOrNull(other)?.schema?.foreignKeys?.any { fk -> + fk.referencedTable.equals(tableName, ignoreCase = true) && + fk.referencedColumns.any { it.equals(action.columnName, ignoreCase = true) } + } == true + } + if (referencingTables.isNotEmpty()) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName} because table(s) ${referencingTables.joinToString(", ")} " + + "have a FOREIGN KEY referencing it; drop the constraint first" + ) + } + if (schema.uniqueConstraints.any { unique -> unique.columns.any { it.equals(action.columnName, ignoreCase = true) } }) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName} because it is used by a UNIQUE constraint of $tableName; " + + "drop the constraint first" + ) + } + val dependentIndexes = table.secondaryIndexes.filter { it.columnNames.any { c -> c.equals(action.columnName, ignoreCase = true) } } + if (dependentIndexes.isNotEmpty()) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName} because index(es) " + + "${dependentIndexes.joinToString(", ") { it.name }} are defined on it; drop the index first" + ) + } + + val newColumn = column.copy(type = action.newType, precision = action.precision, scale = action.scale) + newColumn.default?.let { default -> + try { + coerceForColumn(Evaluator().evaluate(default), newColumn) + } catch (e: KestrelException) { + throw DiskExecutorException( + "cannot change the type of column ${action.columnName}: its DEFAULT expression is not compatible " + + "with ${action.newType.displayWith(action.precision, action.scale)} (${e.message})" + ) + } + } + val newSchema = TableSchema( + schema.name, schema.columns.toMutableList().also { it[columnIndex] = newColumn }, schema.foreignKeys, + schema.checks, schema.uniqueConstraints, schema.compositePrimaryKey + ) + + val using = action.using ?: ColumnExpression(action.columnName) + catalog.changeColumnType(tableName, action.columnName, newColumn) { row -> + val value = coerceForColumn(Evaluator(schema, row).evaluate(using), newColumn) + if (newColumn.notNull && value is Value.NullValue) { + throw DiskExecutorException("NOT NULL constraint failed: $tableName.${action.columnName}") + } + val newRow = Row(row.values.toMutableList().also { it[columnIndex] = value }) + for (check in newSchema.checks) { + val result = Evaluator(newSchema, newRow).evaluate(check.expression) + if (result is Value.BooleanValue && !result.value) { + val label = check.name?.let { " $it" } ?: "" + throw DiskExecutorException("CHECK constraint$label failed: $tableName") + } + } + value + } + } + + /** Mirrors [kestrel.engine.Executor.requireUniqueConstraintName] - see that method's own doc. */ + private fun requireUniqueConstraintName(schema: TableSchema, name: String) { + val taken = schema.foreignKeys.any { it.name?.equals(name, ignoreCase = true) == true } || + schema.checks.any { it.name?.equals(name, ignoreCase = true) == true } || + schema.uniqueConstraints.any { it.name?.equals(name, ignoreCase = true) == true } + if (taken) throw DiskExecutorException("constraint '$name' already exists on table ${schema.name}") + } + + /** + * A NOT NULL column added to a table that already has rows must carry a DEFAULT (there is no + * value to backfill existing rows with otherwise) - an empty table has no such requirement. + * Mirrors [kestrel.engine.Executor.executeAddColumn] (same reasoning as [coerceForColumn]): + * duplicated rather than shared, since this executor's [DiskTable] has no equivalent of the + * in-memory engine's `Table.rows` to check emptiness against without a transaction of its own. + * GENERATED ALWAYS AS sidesteps the NOT NULL/DEFAULT requirement entirely - see + * [addGeneratedColumnValue]. + */ + private fun executeAddColumn(tableName: String, column: Column) { + val table = catalog.table(tableName) + if (column.generated != null) { + validateAddGeneratedColumn(table.schema, column) + catalog.addColumn(tableName, column) { row -> addGeneratedColumnValue(table.schema, column, row) } + return + } + column.default?.let { validateConstantDefault(it) } + val probeTxn = catalog.beginTransaction() + val hasRows = runInTransaction(probeTxn) { table.scan(probeTxn).isNotEmpty() } + if (column.notNull && column.default == null && hasRows) { + throw DiskExecutorException( + "cannot add NOT NULL column ${column.name} to table $tableName with existing rows without a DEFAULT value" + ) + } + val defaultValue = column.default?.let { coerceForColumn(Evaluator().evaluate(it), column) } ?: Value.NullValue + catalog.addColumn(tableName, column) { defaultValue } + } + + /** Mirrors [kestrel.engine.Executor.validateAddGeneratedColumn] - see that method's own doc. */ + private fun validateAddGeneratedColumn(schema: TableSchema, column: Column) { + val generated = column.generated!! + checkColumns(generated, schema) + requireNoSubquery(generated) + val generatedNames = schema.columns.filter { it.generated != null }.map { it.name.lowercase() }.toSet() + requireNoGeneratedColumnReference(generated, generatedNames, column.name) + } + + /** Mirrors [kestrel.engine.Executor.addGeneratedColumnValue] - see that method's own doc. */ + private fun addGeneratedColumnValue(schema: TableSchema, column: Column, row: Row): Value { + val value = coerceForColumn(Evaluator(schema, row).evaluate(column.generated!!), column) + if (column.notNull && value is Value.NullValue) { + throw DiskExecutorException("NOT NULL constraint failed: ${schema.name}.${column.name}") + } + return value + } + + /** + * A DEFAULT must be a constant expression: no column reference (there is no row to read one + * from when backfilling `ALTER TABLE ... ADD COLUMN`), no subquery, and no aggregate. Ported + * 1:1 from [kestrel.engine.Executor.validateConstantDefault] - see [checkColumns] for why this + * executor duplicates rather than shares this kind of validation. + */ + private fun validateConstantDefault(expression: Expression) { + when (expression) { + is ColumnExpression -> throw DiskExecutorException("DEFAULT expression cannot reference a column") + is ScalarSubqueryExpression, is ExistsExpression, is InSubqueryExpression -> + throw DiskExecutorException("DEFAULT expression cannot contain a subquery") + is FunctionExpression -> throw DiskExecutorException("DEFAULT expression cannot contain an aggregate function") + is BinaryExpression -> { + validateConstantDefault(expression.left) + validateConstantDefault(expression.right) + } + is UnaryExpression -> validateConstantDefault(expression.operand) + is CallExpression -> expression.arguments.forEach { validateConstantDefault(it) } + is IsNullExpression -> validateConstantDefault(expression.operand) + is InExpression -> { + validateConstantDefault(expression.operand) + expression.values.forEach { validateConstantDefault(it) } + } + is BetweenExpression -> { + validateConstantDefault(expression.operand) + validateConstantDefault(expression.low) + validateConstantDefault(expression.high) + } + is LikeExpression -> { + validateConstantDefault(expression.operand) + validateConstantDefault(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { validateConstantDefault(it) } + expression.branches.forEach { + validateConstantDefault(it.condition) + validateConstantDefault(it.result) + } + expression.elseExpression?.let { validateConstantDefault(it) } + } + is CastExpression -> validateConstantDefault(expression.operand) + is CollateExpression -> validateConstantDefault(expression.operand) + is WindowExpression -> throw DiskExecutorException("DEFAULT expression cannot contain a window function") + is NextValueExpression -> + throw DiskExecutorException( + "NEXTVAL cannot be used as an ADD COLUMN DEFAULT - it would give every existing row " + + "the exact same value instead of a growing sequence, silently defeating the one thing NEXTVAL is for" + ) + is CurrentValueExpression -> throw DiskExecutorException("DEFAULT expression cannot contain CURRVAL") + is MergeActionExpression -> throw DiskExecutorException("DEFAULT expression cannot contain MERGE_ACTION()") + is ParameterExpression -> throw DiskExecutorException("DEFAULT expression cannot contain a parameter reference") + is LiteralExpression -> Unit + } + } + + /** + * Builds every row's final values first (in [newRows], none written yet) and only then validates + * and inserts them one at a time - not inline in a single pass - so that a self-referencing FK + * can see sibling rows from the same multi-row INSERT regardless of their order in the statement, + * exactly like [kestrel.engine.Executor.executeInsert]'s own `uncommittedSiblings = newRows`. + */ + private fun executeInsert(statement: InsertStatement, triggerContext: TriggerRowContext? = null): StatementResult { + catalog.temporaryTableOrNull(statement.tableName)?.let { return executeTemporaryInsert(statement, it) } + val table = catalog.table(statement.tableName) + val schema = table.schema + return withStatementTransaction { txn -> + withTriggerAtomicity(schema.name, txn, triggerContext) { + executeInsertInTxn(statement, table, schema, txn, triggerContext) + } + } + } + + /** + * INSERT into a temp table's plain [Table.rows] list - ported from the same validate-the-whole- + * batch-before-mutating-anything shape as [executeInsertInTxn]/[kestrel.engine.Executor. + * executeInsertInner], minus [DiskForeignKeyEnforcer]/trigger firing (both rejected outright at + * `CREATE TEMP TABLE` time - see [executeCreateTemporaryTable]) and minus ON CONFLICT (a separate, + * documented scope-cut: [executeInsertWithConflictHandling]'s `findConflictingRow` scans a live + * [DiskTable] via [DiskTable.scan], which a temp table has none of). Still opens a real disk + * transaction via [withStatementTransaction] purely so a DEFAULT/CHECK/GENERATED expression's + * scalar subquery (via [subqueryRunnerFor]) or a NEXTVAL default can read/advance the disk + * catalog consistently - nothing about the temp table's own row list ever touches it. + */ + private fun executeTemporaryInsert(statement: InsertStatement, table: Table): StatementResult { + if (statement.onConflict != null) { + throw DiskExecutorException("INSERT ... ON CONFLICT on a temporary table is not supported by the disk-backed executor yet") + } + val schema = table.schema + return withStatementTransaction { txn -> + val targetIndices = resolveInsertTargets(schema, statement.columns) + val subqueries = subqueryRunnerFor(txn) + val evaluator = Evaluator(composeResolver(null, null), subqueries, this) + val newRows = statement.rows.map { rawRow -> + if (rawRow.size != targetIndices.size) { + throw DiskExecutorException( + "table ${schema.name} expects ${targetIndices.size} values but ${rawRow.size} were supplied" + ) + } + val values = MutableList(schema.columns.size) { Value.NullValue } + val touched = BooleanArray(schema.columns.size) + for ((position, expression) in rawRow.withIndex()) { + val columnIndex = targetIndices[position] + if (expression != null) { + rejectGeneratedAssignment(schema.columns[columnIndex], explicitValueGiven = true) + values[columnIndex] = coerceForColumn(evaluator.evaluate(expression), schema.columns[columnIndex]) + touched[columnIndex] = true + } + } + for ((index, column) in schema.columns.withIndex()) { + val default = column.default + if (!touched[index] && default != null) { + values[index] = coerceForColumn(evaluator.evaluate(default), column) + } + } + applyGeneratedColumns(schema, values, subqueries) + validateTemporaryNotNull(schema, values) + Row(values) + } + for (row in newRows) enforceChecks(schema, row, subqueries) + enforceTemporaryPrimaryKeyUniqueness(schema, table.rows + newRows) + enforceTemporaryUniqueConstraints(schema, table.rows + newRows) + table.rows.addAll(newRows) + val returning = statement.returning?.let { evaluateReturning(schema, newRows, it, subqueries) } + InsertResult(newRows.size, returning) + } + } + + private fun executeInsertInTxn( + statement: InsertStatement, table: DiskTable, schema: TableSchema, txn: Long, triggerContext: TriggerRowContext? + ): StatementResult { + val targetIndices = resolveInsertTargets(schema, statement.columns) + val onConflict = statement.onConflict + val subqueries = subqueryRunnerFor(txn) + val evaluator = Evaluator(composeResolver(null, triggerContext), subqueries, this) + val newRows = statement.rows.map { rawRow -> + if (rawRow.size != targetIndices.size) { + throw DiskExecutorException( + "table ${schema.name} expects ${targetIndices.size} values but ${rawRow.size} were supplied" + ) + } + val values = MutableList(schema.columns.size) { Value.NullValue } + val touched = BooleanArray(schema.columns.size) + for ((position, expression) in rawRow.withIndex()) { + val columnIndex = targetIndices[position] + if (expression != null) { + rejectGeneratedAssignment(schema.columns[columnIndex], explicitValueGiven = true) + values[columnIndex] = coerceForColumn(evaluator.evaluate(expression), schema.columns[columnIndex]) + touched[columnIndex] = true + } + } + for ((index, column) in schema.columns.withIndex()) { + val default = column.default + if (!touched[index] && default != null) { + values[index] = coerceForColumn(evaluator.evaluate(default), column) + } + } + applyGeneratedColumns(schema, values, subqueries) + Row(values) + } + if (onConflict == null) { + // Fires before any constraint is even checked, matching SQLite/Postgres: a BEFORE + // trigger can reject the whole statement (by having its body throw) before this + // executor's own FOREIGN KEY/CHECK/UNIQUE validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, newRows.map { null to it }, txn) + for (row in newRows) { + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, row.values, uncommittedSiblings = newRows) + enforceChecks(schema, row, subqueries) + } + for (row in newRows) table.insert(txn, row) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, newRows.map { null to it }, txn) + val returning = statement.returning?.let { evaluateReturning(schema, newRows, it, subqueries) } + return InsertResult(newRows.size, returning) + } + return executeInsertWithConflictHandling(schema, table, txn, newRows, onConflict, subqueries, statement.returning, triggerContext) + } + + private class DiskUniqueKey(val columnIndices: List, val columnNames: List) + + /** Ported 1:1 from [kestrel.engine.Executor.uniqueKeysOf]: PRIMARY KEY, then every declared UNIQUE constraint, then every UNIQUE secondary index - the same three sources a conflict target may name. */ + private fun diskUniqueKeysOf(schema: TableSchema, table: DiskTable): List { + val keys = mutableListOf() + val pkIndices = schema.primaryKeyIndices + if (pkIndices.isNotEmpty()) keys.add(DiskUniqueKey(pkIndices, pkIndices.map { schema.columns[it].name })) + for (unique in schema.uniqueConstraints) { + val indices = unique.columns.map { schema.columnIndex(it)!! } + keys.add(DiskUniqueKey(indices, unique.columns)) + } + for (handle in table.secondaryIndexes) { + if (!handle.unique) continue + keys.add(DiskUniqueKey(handle.columnIndices, handle.columnNames)) + } + return keys + } + + /** Ported 1:1 from [kestrel.engine.Executor.resolveConflictTargetKeys]: empty [targetColumns] (bare `ON CONFLICT DO ...`) matches against any of [keys]. */ + private fun resolveConflictTargetKeys(schema: TableSchema, targetColumns: List, keys: List): List { + if (targetColumns.isEmpty()) return keys + val targetIndices = targetColumns.map { name -> + schema.columnIndex(name) ?: throw DiskExecutorException("no such column: $name") + } + val targetSet = targetIndices.toSet() + val matched = keys.firstOrNull { it.columnIndices.toSet() == targetSet } + ?: throw DiskExecutorException( + "no unique or primary key constraint matching ON CONFLICT specification: " + + targetColumns.joinToString(", ") + ) + return listOf(matched) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.findConflictingRow], but scanning the live + * [DiskTable] (via [DiskTable.scan]) rather than an in-memory `List` snapshot - the + * transactional heap already reflects any row this same statement inserted or updated + * earlier in [executeInsertWithConflictHandling]'s per-row loop, giving the same + * read-your-own-writes semantics the in-memory engine gets from folding sibling rows into + * its candidate list by hand. NULLs never collide with each other (standard SQL). + */ + private fun findConflictingRow(txn: Long, table: DiskTable, candidate: List, keys: List): Pair? { + for ((existingRid, existing) in table.scan(txn)) { + for (key in keys) { + val candidateKey = key.columnIndices.map { candidate[it] } + if (candidateKey.any { it is Value.NullValue }) continue + val existingKey = key.columnIndices.map { existing.values[it] } + if (candidateKey == existingKey) return existingRid to existing + } + } + return null + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.executeInsertWithConflictHandling]: each candidate + * row is processed as its own micro-insert against the table's current state (not a + * batch-validated snapshot), so a later row in the same multi-row INSERT correctly sees an + * earlier row's insert or update. [triggerContext] can be non-null here for two independent + * reasons: [schema] itself may have triggers of its own (fired per row below, exactly like + * [kestrel.engine.Executor] does), and/or this INSERT ... ON CONFLICT may itself be a trigger + * body statement targeting a *different* table, so `NEW`/`OLD` from the outer firing trigger + * must still resolve in its DO UPDATE assignment values - both compose without conflict since + * [fireRowTriggers] below builds its own fresh [TriggerRowContext] from [schema]'s own triggers, + * independent of [triggerContext]. Unlike the batch INSERT/UPDATE paths, no separate reindex step + * is needed around trigger firing here: [DiskTable]'s secondary indexes are B+Trees maintained + * incrementally by [table.insert]/[table.update] themselves, not rebuilt in a batch pass like + * [kestrel.catalog.Catalog.reindexTable], so a trigger body's own query against this table always + * sees a consistent index the instant this row's own mutation returns. + */ + private fun executeInsertWithConflictHandling( + schema: TableSchema, + table: DiskTable, + txn: Long, + candidateRows: List, + onConflict: OnConflictClause, + subqueries: SubqueryRunner, + returning: Projection? = null, + triggerContext: TriggerRowContext? = null + ): StatementResult { + val keys = diskUniqueKeysOf(schema, table) + val targetKeys = resolveConflictTargetKeys(schema, onConflict.targetColumns, keys) + var affected = 0 + // DO NOTHING contributes no row, and a DO UPDATE row whose WHERE guard fails `continue`s + // before reaching the append below - both absent from RETURNING, matching Postgres, same + // as the in-memory engine. + val returnedRows = mutableListOf() + for (row in candidateRows) { + val conflict = findConflictingRow(txn, table, row.values, targetKeys) + if (conflict == null) { + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, listOf(null to row), txn) + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, row.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, row, subqueries) + table.insert(txn, row) + affected++ + returnedRows.add(row) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, listOf(null to row), txn) + continue + } + val (conflictingRid, conflictingRow) = conflict + when (val action = onConflict.action) { + ConflictDoNothing -> Unit + is ConflictDoUpdate -> { + val resolver = composeResolver(excludedAwareResolver(schema, conflictingRow, row), triggerContext) + // NEXTVAL/CURRVAL are deliberately usable in the assignment values below but not + // in this WHERE-shaped guard, matching the in-memory engine's own two-Evaluator split. + val conditionEvaluator = Evaluator(resolver, subqueries) + if (action.where != null && !isTrue(conditionEvaluator.evaluate(action.where))) continue + val assignmentEvaluator = Evaluator(resolver, subqueries, this) + val values = conflictingRow.values.toMutableList() + for (assignment in action.assignments) { + val index = schema.columnIndex(assignment.columnName) + ?: throw DiskExecutorException("no such column: ${assignment.columnName}") + val column = schema.columns[index] + rejectGeneratedAssignment(column, explicitValueGiven = assignment.value != null) + values[index] = if (assignment.value != null) { + coerceForColumn(assignmentEvaluator.evaluate(assignment.value), column) + } else { + column.default?.let { coerceForColumn(assignmentEvaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values, subqueries) + val updated = Row(values) + // Fires before any constraint below is checked, same as executeUpdateInTxn's own + // BEFORE trigger - a BEFORE UPDATE trigger's body can reject this row's update + // before this executor's own validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, listOf(conflictingRow to updated), txn) + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, updated.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, updated, subqueries) + // Same as executeUpdate: any table with a FK pointing at one of this table's + // enforced-unique key groups needs a cascade/RESTRICT pass when that group's + // value actually changes, not just the PRIMARY KEY. + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val oldKey = columnIndices.map { conflictingRow.values[it] } + val newKey = columnIndices.map { updated.values[it] } + DiskForeignKeyEnforcer.cascadeUpdate(catalog, txn, schema.name, group, listOf(oldKey to newKey)) + } + table.update(txn, conflictingRid, updated) + affected++ + returnedRows.add(updated) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, listOf(conflictingRow to updated), txn) + } + } + } + return InsertResult(affected, returning?.let { evaluateReturning(schema, returnedRows, it, subqueries) }) + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.excludedAwareResolver]: bare column names resolve + * against [target] (the existing conflicting row, matching plain UPDATE semantics), while + * `EXCLUDED.column` resolves against [excluded] (the row proposed by this INSERT that failed + * to land) - the same pseudo-table Postgres exposes in DO UPDATE. + */ + private fun excludedAwareResolver(schema: TableSchema, target: Row, excluded: Row): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + val useExcluded = qualifier != null && qualifier.equals("excluded", ignoreCase = true) + if (qualifier != null && !useExcluded && !qualifier.equals(schema.name, ignoreCase = true)) { + throw DiskExecutorException("no such table or alias: $qualifier") + } + val index = schema.columnIndex(expression.name) + ?: throw DiskExecutorException("no such column: ${expression.name}") + return if (useExcluded) excluded.values[index] else target.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = schema.column(expression.name)?.collation + } + + private fun resolveInsertTargets(schema: TableSchema, columns: List?): List { + if (columns == null) return schema.columns.indices.toList() + return columns.map { columnName -> + schema.columnIndex(columnName) + ?: throw DiskExecutorException("no such column: $columnName in table ${schema.name}") + } + } + + private fun executeQuery(statement: QueryStatement): StatementResult = + withStatementTransaction { txn -> runQuery(statement, txn, outer = null) } + + /** + * Closes the "this executor has no `explain`" scope cut documented on [executeCreateView]: + * describes the plan [runSelect]/[runQuery] would use for [statement.select] without running + * it, mirroring [kestrel.engine.SelectRunner.explain]'s shape (Seq Scan/Index Scan header per + * driving source, Nested Loop per JOIN, GroupAggregate/Aggregate, Sort, Limit, CTE/UNION/WITH + * RECURSIVE nesting) with two differences forced by this executor's disk-backed row supply + * rather than the in-memory engine's already-resident rows: + * + * - A materialized CTE, a VIEW, or a non-LATERAL derived table in FROM still runs its defining + * query for real via [resolveFromItem]/[buildCteScope] (exactly what [runSelect] itself would + * do, and exactly what [kestrel.engine.SelectRunner.explain] does too through `resolveSource`) + * since only running it produces a real schema/row set to plan further nesting against. The + * query's own driving table(s) are never scanned this way, though: [resolveFromItem] only + * wraps a real [DiskTable] in a [DiskTableSource] (schema/index metadata, no I/O) - the + * [DiskTableSource.scan] a real [runSelect] would call to actually read rows is deliberately + * never invoked here, so `EXPLAIN SELECT * FROM huge_table` costs nothing proportional to + * `huge_table`'s size the way `SELECT * FROM huge_table` would. + * - [applyJoin] has no hash-join optimization at all (see its own doc), so "Nested Loop" is the + * only join header this ever prints - "Hash Join" (which the in-memory engine's EXPLAIN can + * print for an equality ON condition) never appears here. + * + * Deliberately narrower than [kestrel.engine.SelectRunner.explain] in one respect: it reuses + * every validation [runSelect] performs *structurally*, independent of row data - duplicate + * alias detection ([buildJoinSources]), window-function placement ([rejectWindowFunction]/ + * [validateWindows]), and GROUP BY/aggregate correctness ([validateAggregation], itself + * structural since [resolveIndex] only consults [JoinSource.schema], never row data) - but does + * NOT eagerly resolve WHERE/ON column references the way [kestrel.engine.SelectRunner.explain]'s + * `Plan.validate()` does. [runSelect] itself only resolves those against real row data inside + * `combined.filter { ... }`/[applyJoin], so on an empty table a bad WHERE/ON column reference + * never surfaces there either (a pre-existing, documented gap versus the in-memory engine, not + * a new one introduced here) - EXPLAIN staying exactly as permissive keeps its guarantees no + * stronger than what actually running the statement would give you. + */ + private fun executeExplain(statement: ExplainStatement): StatementResult = + withStatementTransaction { txn -> ExplainResult(explainQuery(statement.select, txn)) } + + private fun explainQuery(query: QueryStatement, txn: Long): List = when (query) { + is SelectStatement -> explainSelect(query, txn) + is SetOperationStatement -> explainSetOperation(query, txn) + is WithQuery -> explainWith(query, txn) + } + + /** Title-case labels for an EXPLAIN header - deliberately separate from [setOperatorKeyword]'s ALL-CAPS SQL-keyword text used in error messages. */ + private fun explainOperatorLabel(operator: SetOperator): String = when (operator) { + SetOperator.Union -> "Union" + SetOperator.UnionAll -> "Union All" + SetOperator.Intersect -> "Intersect" + SetOperator.IntersectAll -> "Intersect All" + SetOperator.Except -> "Except" + SetOperator.ExceptAll -> "Except All" + } + + private fun explainSetOperation(statement: SetOperationStatement, txn: Long): List { + val header = explainOperatorLabel(statement.operator) + val left = explainQuery(statement.left, txn).map { " -> $it" } + val right = explainQuery(statement.right, txn).map { " -> $it" } + var lines = listOf(header) + left + right + statement.orderBy?.let { lines = listOf("Sort") + lines.map { line -> " $line" } } + statement.offset?.let { lines = listOf("Offset ${statement.offset}") + lines.map { line -> " $line" } } + statement.limit?.let { lines = listOf("Limit ${statement.limit}") + lines.map { line -> " $line" } } + return lines + } + + /** + * Materializes every CTE for real via [buildCteScope] (needed so a query nested under this WITH + * can itself be explained against a real schema/row set), then re-walks each CTE's own query a + * second time purely to render its plan text - the same double pass [kestrel.engine.SelectRunner. + * explainWith] already accepts for the in-memory engine, including the same cost of re-running a + * *nested* recursive CTE's fixed point twice over if one WITH RECURSIVE is nested inside another. + */ + private fun explainWith(query: WithQuery, txn: Long): List { + var pushed = 0 + try { + for (cte in query.ctes) { + activeCtes.add(buildCteScope(query.recursive, cte, txn)) + pushed++ + } + val cteLines = query.ctes.flatMap { cte -> + listOf("CTE ${cte.name}") + explainQuery(cte.query, txn).map { " -> $it" } + } + return cteLines + explainQuery(query.body, txn) + } finally { + repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) } + } + } + + private fun explainSelect(statement: SelectStatement, txn: Long): List { + val fromItems = listOf(statement.from) + statement.joins.map { it.from } + val fromSources = fromItems.map { item -> resolveFromItem(item, txn) } + val sources = buildJoinSources(fromItems, fromSources.map { it.schema }) + statement.joins.forEach { join -> join.on?.let { rejectWindowFunction(it, "an ON condition") } } + var body = explainJoinLines(statement, sources, fromSources) + statement.where?.let { rejectWindowFunction(it, "WHERE") } + statement.groupBy.forEach { rejectWindowFunction(it, "GROUP BY") } + statement.having?.let { rejectWindowFunction(it, "HAVING") } + statement.distinctOn.forEach { rejectWindowFunction(it, "DISTINCT ON") } + val orderKeys = resolvedOrderKeys(statement) + val aggregated = statement.groupBy.isNotEmpty() || statement.groupingSpec != null || + containsAggregate(statement.projection) || + (statement.having != null && containsAggregate(statement.having)) + if (statement.distinctOn.isNotEmpty()) validateDistinctOn(statement, orderKeys, aggregated) + val windowExpressions = collectWindowExpressions(statement.projection, orderKeys) + if (windowExpressions.isNotEmpty()) validateWindows(windowExpressions, aggregated) + if (aggregated) { + if (statement.projection is StarProjection) { + throw DiskExecutorException("cannot use * with GROUP BY or aggregate functions") + } + validateAggregation(statement, orderKeys, sources) + val keys = statement.groupBy.map { describeExpression(it) } + val header = if (keys.isEmpty()) { + "Aggregate" + } else { + val setCount = GroupingSetExpansion.setsFor(statement.groupingSpec, statement.groupBy).size + val suffix = if (setCount > 1) " ($setCount grouping sets)" else "" + "GroupAggregate keys: ${keys.joinToString(", ")}$suffix" + } + body = listOf(header) + body.map { " $it" } + } + statement.orderBy?.let { body = listOf("Sort") + body.map { " $it" } } + if (statement.distinct) { + val label = if (statement.distinctOn.isNotEmpty()) "Unique (DISTINCT ON)" else "Unique" + body = listOf(label) + body.map { " $it" } + } + statement.offset?.let { body = listOf("Offset ${statement.offset}") + body.map { " $it" } } + statement.limit?.let { body = listOf("Limit ${statement.limit}") + body.map { " $it" } } + return body + } + + private fun explainJoinLines(statement: SelectStatement, sources: List, fromSources: List): List { + var lines = listOf(explainScanLine(statement, fromSources.first())) + statement.joins.forEachIndexed { position, join -> + val right = sources[position + 1] + val lateral = lateralSelectOf(join.from) != null + val label = joinLabel(join.type) + if (lateral) " Lateral" else "" + val header = "Nested Loop $label" + val rightScan = if (lateral) "Lateral Subquery Scan on ${right.schema.name}" else "Seq Scan on ${right.schema.name}" + lines = listOf(header) + lines.map { " -> $it" } + listOf(" -> $rightScan") + } + return lines + } + + /** The driving source's scan line: an indexed [DiskAccessPlan] when there is no JOIN and it is a real disk table, a full scan otherwise - mirrors [initialRows]'s own choice of when to push down. */ + private fun explainScanLine(statement: SelectStatement, fromSource: DiskFromSource): String { + if (statement.joins.isEmpty() && fromSource is DiskTableSource) { + val alias = (statement.from as? TableSource)?.alias + val plan = accessPlanner.plan(fromSource.table.schema, fromSource.table.secondaryIndexes, statement.where, alias) + return describeAccessPlan(plan, fromSource.table.schema, fromSource.table.secondaryIndexes) + } + return "Seq Scan on ${fromSource.schema.name}" + } + + private fun describeAccessPlan(plan: DiskAccessPlan, schema: TableSchema, secondaryIndexes: List): String { + val tableName = schema.name + return when (plan) { + DiskAccessPlan.FullScan -> "Seq Scan on $tableName" + is DiskAccessPlan.PrimaryKeyPoint -> { + val pkColumn = schema.columns[schema.primaryKeyIndex!!].name + "Primary Key Scan on $tableName ($pkColumn = ${literalText(plan.key)})" + } + is DiskAccessPlan.PrimaryKeyRange -> { + val pkColumn = schema.columns[schema.primaryKeyIndex!!].name + "Primary Key Range Scan on $tableName (${rangeDescription(pkColumn, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive)})" + } + is DiskAccessPlan.CompositePrimaryKeyPoint -> { + val parts = schema.primaryKeyColumns.zip(plan.keys) { name, value -> "$name = ${literalText(value)}" } + "Primary Key Scan on $tableName (${parts.joinToString(" AND ")})" + } + is DiskAccessPlan.CompositePrimaryKeyRange -> { + val columnName = schema.primaryKeyColumns[plan.prefix.size] + val prefixParts = schema.primaryKeyColumns.zip(plan.prefix) { name, value -> "$name = ${literalText(value)}" } + val rangePart = rangeDescription(columnName, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive) + "Primary Key Range Scan on $tableName (${(prefixParts + rangePart).joinToString(" AND ")})" + } + is DiskAccessPlan.Secondary -> { + val handle = secondaryIndexes.first { it.name == plan.indexName } + val parts = handle.columnNames.zip(plan.keys) { name, value -> "$name = ${literalText(value)}" } + "Index Scan using ${plan.indexName} on $tableName (${parts.joinToString(" AND ")})" + } + is DiskAccessPlan.SecondaryRange -> { + val handle = secondaryIndexes.first { it.name == plan.indexName } + val columnName = handle.columnNames[plan.prefix.size] + val prefixParts = handle.columnNames.zip(plan.prefix) { name, value -> "$name = ${literalText(value)}" } + val rangePart = rangeDescription(columnName, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive) + "Index Range Scan using ${plan.indexName} on $tableName (${(prefixParts + rangePart).joinToString(" AND ")})" + } + } + } + + private fun rangeDescription(columnName: String, lower: Value?, lowerInclusive: Boolean, upper: Value?, upperInclusive: Boolean): String { + val parts = mutableListOf() + lower?.let { parts.add("$columnName ${if (lowerInclusive) ">=" else ">"} ${literalText(it)}") } + upper?.let { parts.add("$columnName ${if (upperInclusive) "<=" else "<"} ${literalText(it)}") } + return parts.joinToString(" AND ") + } + + private fun joinLabel(type: JoinType): String = when (type) { + JoinType.Left -> "Left" + JoinType.Right -> "Right" + JoinType.Full -> "Full" + JoinType.Inner -> "Inner" + JoinType.Cross -> "Cross" + } + + private fun describe(expression: ColumnExpression): String = + expression.qualifier?.let { "$it.${expression.name}" } ?: expression.name + + /** Ported 1:1 from [kestrel.engine.SelectRunner.describeExpression] - renders any expression as SQL text for an EXPLAIN header (currently only GROUP BY keys reach this). */ + private fun describeExpression(expression: Expression): String = when (expression) { + is ColumnExpression -> describe(expression) + is LiteralExpression -> literalText(expression.value) + is FunctionExpression -> "${expression.name}(${describeFunctionArguments(expression)})${describeFilterClause(expression)}" + is BinaryExpression -> "${describeExpression(expression.left)} ${expression.operator.symbol} ${describeExpression(expression.right)}" + is UnaryExpression -> "${expression.operator.symbol} ${describeExpression(expression.operand)}" + is CallExpression -> "${expression.name}(${expression.arguments.joinToString(", ") { describeExpression(it) }})" + is IsNullExpression -> "${describeExpression(expression.operand)} IS ${if (expression.negated) "NOT " else ""}NULL" + is InExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (${expression.values.joinToString(", ") { describeExpression(it) }})" + is BetweenExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}BETWEEN ${describeExpression(expression.low)} AND ${describeExpression(expression.high)}" + is LikeExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}LIKE ${describeExpression(expression.pattern)}" + is CaseExpression -> buildString { + append("CASE ") + expression.operand?.let { append(describeExpression(it)); append(' ') } + expression.branches.forEach { append("WHEN ${describeExpression(it.condition)} THEN ${describeExpression(it.result)} ") } + expression.elseExpression?.let { append("ELSE ${describeExpression(it)} ") } + append("END") + } + is ScalarSubqueryExpression -> "(subquery)" + is ExistsExpression -> "EXISTS (subquery)" + is InSubqueryExpression -> + "${describeExpression(expression.operand)} ${if (expression.negated) "NOT " else ""}IN (subquery)" + is CastExpression -> + "CAST(${describeExpression(expression.operand)} AS ${expression.targetType.displayWith(expression.precision, expression.scale)})" + is CollateExpression -> "${describeExpression(expression.operand)} COLLATE ${expression.collation.sqlName}" + is WindowExpression -> + "${expression.name}(${if (expression.star) "*" else expression.argument?.let { describeExpression(it) } ?: ""}) OVER (...)" + is NextValueExpression -> "NEXTVAL('${expression.sequenceName}')" + is CurrentValueExpression -> "CURRVAL('${expression.sequenceName}')" + is MergeActionExpression -> "MERGE_ACTION()" + is ParameterExpression -> "\$${expression.index}" + } + + private fun describeFunctionArguments(expression: FunctionExpression): String { + if (expression.star) return "*" + val distinctPrefix = if (expression.distinct) "DISTINCT " else "" + val value = expression.argument?.let { describeExpression(it) } ?: "" + val separator = expression.separator ?: return "$distinctPrefix$value" + val orderBy = if (expression.orderBy.isNotEmpty()) { + " ORDER BY " + expression.orderBy.joinToString(", ") { key -> describeExpression(key.expression) + orderKeySuffix(key) } + } else "" + return "$distinctPrefix$value, ${describeExpression(separator)}$orderBy" + } + + private fun orderKeySuffix(key: OrderKey): String { + val direction = if (key.descending) " DESC" else "" + val nulls = when (key.nullsFirst) { + true -> " NULLS FIRST" + false -> " NULLS LAST" + null -> "" + } + return direction + nulls + } + + private fun describeFilterClause(expression: FunctionExpression): String = + expression.filter?.let { " FILTER (WHERE ${describeExpression(it)})" } ?: "" + + private fun literalText(value: Value): String = when (value) { + is Value.IntegerValue -> value.value.toString() + is Value.RealValue -> value.value.toString() + is Value.TextValue -> "'${value.value}'" + is Value.BooleanValue -> value.value.toString() + is Value.DecimalValue -> value.value.toPlainString() + is Value.DateValue -> value.value.toString() + is Value.NullValue -> "NULL" + } + + /** Runs [select] within [txn] and hands the result straight back to whichever [Evaluator] asked for it. */ + private fun subqueryRunnerFor(txn: Long): SubqueryRunner = SubqueryRunner { select, outer -> runSelect(select, txn, outer) } + + /** + * Dispatches [query] to the runner matching its concrete shape, mirroring + * [kestrel.engine.SelectRunner.runQuery] 1:1: a plain SELECT runs directly, a + * UNION/INTERSECT/EXCEPT chain combines two independently-run sides, and a WITH/WITH RECURSIVE + * query first materializes its common table expressions (visible only for the duration of this + * call, via [activeCtes]) before running its body. + */ + private fun runQuery(query: QueryStatement, txn: Long, outer: ColumnResolver?): SelectResult = when (query) { + is SelectStatement -> runSelect(query, txn, outer) + is SetOperationStatement -> runSetOperation(query, txn, outer) + is WithQuery -> runWith(query, txn, outer) + } + + private fun setOperatorKeyword(operator: SetOperator): String = when (operator) { + SetOperator.Union -> "UNION" + SetOperator.UnionAll -> "UNION ALL" + SetOperator.Intersect -> "INTERSECT" + SetOperator.IntersectAll -> "INTERSECT ALL" + SetOperator.Except -> "EXCEPT" + SetOperator.ExceptAll -> "EXCEPT ALL" + } + + /** + * Deliberately mirrors [kestrel.engine.SelectRunner]'s own `runSetOperation`: both sides run + * independently (each may itself be a nested set operation, WITH query, or plain SELECT), then + * combined by multiset/set semantics (UNION/INTERSECT/EXCEPT dedupe via `List` structural + * equality, the `ALL` variants preserve duplicate counts through [multisetFilter]), with ORDER + * BY/LIMIT applying only to the combined result - never to either side individually, which is + * why [kestrel.sql.parser.Parser] never lets an inner SELECT-core in a compound query own its own + * ORDER BY/LIMIT. + */ + private fun runSetOperation(statement: SetOperationStatement, txn: Long, outer: ColumnResolver?): SelectResult { + val left = runQuery(statement.left, txn, outer) + val right = runQuery(statement.right, txn, outer) + if (left.columns.size != right.columns.size) { + throw DiskExecutorException( + "${setOperatorKeyword(statement.operator)} queries must have the same number of columns " + + "but got ${left.columns.size} and ${right.columns.size}" + ) + } + val declaredCollationOf = declaredSetOperationCollation(statement, left.columns) + val outputExpressions = left.columns.map { ColumnExpression(it) } + fun foldedKey(row: List) = collationFoldedKey(row, outputExpressions, declaredCollationOf) + var rows = when (statement.operator) { + SetOperator.UnionAll -> left.rows + right.rows + SetOperator.Union -> (left.rows + right.rows).distinctBy(::foldedKey) + SetOperator.Intersect -> { + val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey) + left.rows.filter { foldedKey(it) in rightKeys }.distinctBy(::foldedKey) + } + SetOperator.Except -> { + val rightKeys = right.rows.mapTo(HashSet(), ::foldedKey) + left.rows.filter { foldedKey(it) !in rightKeys }.distinctBy(::foldedKey) + } + SetOperator.IntersectAll -> multisetFilter(left.rows, right.rows, keepOnMatch = true, keyOf = ::foldedKey) + SetOperator.ExceptAll -> multisetFilter(left.rows, right.rows, keepOnMatch = false, keyOf = ::foldedKey) + } + val orderKeys = statement.orderBy?.keys.orEmpty() + if (orderKeys.isNotEmpty()) { + rows = rows.sortedWith( + multiKeyComparator( + orderKeys, + evaluate = { expression, row -> evaluateOutputRow(left.columns, expression, row, txn) }, + declaredCollationOf = declaredCollationOf + ) + ) + } + statement.offset?.let { if (it < 0) throw DiskExecutorException("OFFSET must not be negative") } + statement.limit?.let { if (it < 0) throw DiskExecutorException("LIMIT must not be negative") } + if (statement.offset != null || statement.limit != null) { + rows = rows.drop(statement.offset ?: 0) + statement.limit?.let { rows = rows.take(it) } + } + return SelectResult(left.columns, rows) + } + + /** + * Mirrors [kestrel.engine.SelectRunner]'s own `declaredSetOperationCollation` 1:1 (separate + * copy, historical reasons like the rest of this class): the declared `COLLATE` of a set + * operation's combined output column, traced back to a real table column in [statement]'s LEFT + * branch only, resolving only for a plain `SELECT ... FROM realTable [JOIN realTable ...]` with + * no CTE, view, or derived table anywhere in its FROM clause. Everything else - an expression, + * a nested set operation, a subquery source - reports no declared collation, same as before this + * function existed. + */ + private fun declaredSetOperationCollation(statement: SetOperationStatement, columns: List): (ColumnExpression) -> Collation? = + (statement.left as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null } + + /** + * Mirrors [kestrel.engine.SelectRunner]'s own `declaredCollationForSelectOutput` 1:1 (separate + * copy, historical reasons like the rest of this class): factored out of + * [declaredSetOperationCollation] so [declaredRecursiveCteCollation] can apply the identical + * "plain SELECT over real tables" resolution to a recursive CTE's own anchor (base) term. + */ + private fun declaredCollationForSelectOutput(select: SelectStatement, columns: List): (ColumnExpression) -> Collation? { + val items = listOf(select.from) + select.joins.map { it.from } + val sources = items.map { item -> + (item as? TableSource)?.let { source -> catalog.tableOrNull(source.name)?.schema?.let { (source.alias ?: source.name) to it } } + } + if (sources.any { it == null }) return { null } + val resolvedSources = sources.filterNotNull() + fun columnCollation(expression: ColumnExpression): Collation? { + val matches = resolvedSources + .filter { (alias, _) -> expression.qualifier == null || alias.equals(expression.qualifier, ignoreCase = true) } + .mapNotNull { (_, schema) -> schema.column(expression.name) } + return matches.singleOrNull()?.collation + } + return when (val projection = select.projection) { + is ExpressionProjection -> outputExpr@{ outputExpr -> + val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) } + val itemExpression = index.takeIf { it in projection.items.indices }?.let { projection.items[it].expression } + (itemExpression as? ColumnExpression)?.let(::columnCollation) + } + is StarProjection -> outputExpr@{ outputExpr -> + val index = columns.indexOfFirst { it.equals(outputExpr.name, ignoreCase = true) } + if (index < 0) return@outputExpr null + var remaining = index + for ((_, schema) in resolvedSources) { + if (remaining < schema.columns.size) return@outputExpr schema.columns[remaining].collation + remaining -= schema.columns.size + } + null + } + } + } + + /** Mirrors [kestrel.engine.SelectRunner]'s own `declaredRecursiveCteCollation` 1:1 (separate copy, historical reasons like the rest of this class). */ + private fun declaredRecursiveCteCollation(base: QueryStatement, columns: List): (ColumnExpression) -> Collation? = + (base as? SelectStatement)?.let { declaredCollationForSelectOutput(it, columns) } ?: { null } + + private fun multisetFilter( + left: List>, + right: List>, + keepOnMatch: Boolean, + keyOf: (List) -> List + ): List> { + val remainingInRight = right.groupingBy(keyOf).eachCount().toMutableMap() + return left.filter { row -> + val key = keyOf(row) + val remaining = remainingInRight[key] ?: 0 + if (remaining > 0) { + remainingInRight[key] = remaining - 1 + keepOnMatch + } else { + !keepOnMatch + } + } + } + + private fun multiKeyComparator( + keys: List, + evaluate: (Expression, List) -> Value, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } + ): Comparator> = + Comparator { first, second -> + var result = 0 + for (key in keys) { + val comparison = compareOrderKeyValues( + evaluate(key.expression, first), evaluate(key.expression, second), key, declaredCollationOf + ) + if (comparison != 0) { + result = comparison + break + } + } + result + } + + /** Resolves ORDER BY columns of a combined UNION/INTERSECT/EXCEPT result purely by output column name (never qualified). */ + private fun evaluateOutputRow(columns: List, expression: Expression, row: List, txn: Long): Value { + val resolver = ColumnResolver { column -> + if (column.qualifier != null) throw DiskExecutorException("no such column: ${column.qualifier}.${column.name}") + val index = columns.indexOfFirst { it.equals(column.name, ignoreCase = true) } + if (index == -1) throw DiskExecutorException("no such column: ${column.name}") + row[index] + } + return Evaluator(resolver, subqueryRunnerFor(txn)).evaluate(expression) + } + + /** + * Pushes every CTE of [query] onto [activeCtes] (in definition order, so a later CTE may + * reference an earlier one, mirroring [kestrel.engine.SelectRunner.runWith]), runs [query]'s + * body, then pops them back off - a CTE is scoped to the single query that defines it, unlike a + * VIEW, which lives in the catalog. + */ + private fun runWith(query: WithQuery, txn: Long, outer: ColumnResolver?): SelectResult { + var pushed = 0 + try { + for (cte in query.ctes) { + activeCtes.add(buildCteScope(query.recursive, cte, txn)) + pushed++ + } + return runQuery(query.body, txn, outer) + } finally { + repeat(pushed) { activeCtes.removeAt(activeCtes.size - 1) } + } + } + + /** + * Materializes one CTE's rows. A non-recursive CTE just runs its query once. A recursive one + * (`WITH RECURSIVE name AS (base UNION [ALL] recursiveTerm)`, split by [splitRecursiveCte]) + * repeatedly re-runs [RecursiveCtePlan.recursiveTerm] with [CteScope.rows] set to only the + * *previous iteration's new rows* (the standard fixed-point working-set semantics, not the + * whole accumulated set - mirrors [kestrel.engine.SelectRunner.buildCteScope] exactly, including + * the [MAX_RECURSION_ITERATIONS] safety cap against a query that never reaches a fixed point). + */ + private fun buildCteScope(recursive: Boolean, cte: CommonTableExpression, txn: Long): CteScope { + val recursivePlan = if (recursive) splitRecursiveCte(cte) else null + if (recursivePlan == null) { + val result = runQuery(cte.query, txn, outer = null) + return CteScope(cte.name, cteSchema(cte, result), result.rows) + } + val (base, operator, recursiveTerm) = recursivePlan + val baseResult = runQuery(base, txn, outer = null) + val schema = cteSchema(cte, baseResult) + val scope = CteScope(cte.name, schema, baseResult.rows) + val declaredCollationOf = declaredRecursiveCteCollation(base, baseResult.columns) + val outputExpressions = baseResult.columns.map { ColumnExpression(it) } + val seenKeys = if (operator == SetOperator.Union) + baseResult.rows.mapTo(HashSet()) { collationFoldedKey(it, outputExpressions, declaredCollationOf) } + else null + activeCtes.add(scope) + try { + val accumulated = mutableListOf>() + accumulated.addAll(baseResult.rows) + var working = baseResult.rows + var iterations = 0 + while (working.isNotEmpty()) { + iterations++ + if (iterations > MAX_RECURSION_ITERATIONS) { + throw DiskExecutorException( + "recursive common table expression '${cte.name}' did not terminate within $MAX_RECURSION_ITERATIONS iterations" + ) + } + scope.rows = working + var newRows = runQuery(recursiveTerm, txn, outer = null).rows + if (operator == SetOperator.Union) { + newRows = newRows.filter { candidate -> seenKeys!!.add(collationFoldedKey(candidate, outputExpressions, declaredCollationOf)) } + } + if (newRows.isEmpty()) break + accumulated.addAll(newRows) + working = newRows + } + scope.rows = accumulated + } finally { + activeCtes.removeAt(activeCtes.size - 1) + } + return scope + } + + private data class RecursiveCtePlan(val base: QueryStatement, val operator: SetOperator, val recursiveTerm: QueryStatement) + + /** + * Recognizes `base UNION [ALL] recursiveTerm` where exactly one side names [cte] in its own + * FROM/JOIN - structurally, on the AST, without running anything - mirroring + * [kestrel.engine.SelectRunner.splitRecursiveCte] 1:1. Both sides referencing it, or neither, + * are rejected/treated as a plain (non-recursive) compound respectively, exactly like the + * in-memory engine. + */ + private fun splitRecursiveCte(cte: CommonTableExpression): RecursiveCtePlan? { + val query = cte.query + if (query !is SetOperationStatement) return null + if (query.operator != SetOperator.Union && query.operator != SetOperator.UnionAll) return null + val leftRefs = referencesName(query.left, cte.name) + val rightRefs = referencesName(query.right, cte.name) + return when { + leftRefs && rightRefs -> + throw DiskExecutorException( + "recursive common table expression '${cte.name}' must not reference itself on both sides of ${query.operator}" + ) + rightRefs -> RecursiveCtePlan(query.left, query.operator, query.right) + leftRefs -> RecursiveCtePlan(query.right, query.operator, query.left) + else -> null + } + } + + private fun referencesName(query: QueryStatement, name: String): Boolean = when (query) { + is SelectStatement -> referencesName(query.from, name) || query.joins.any { referencesName(it.from, name) } + is SetOperationStatement -> referencesName(query.left, name) || referencesName(query.right, name) + is WithQuery -> query.ctes.none { it.name.equals(name, ignoreCase = true) } && referencesName(query.body, name) + } + + private fun referencesName(item: FromItem, name: String): Boolean = when (item) { + is TableSource -> item.name.equals(name, ignoreCase = true) + is SubquerySource -> referencesName(item.select, name) + } + + private fun cteSchema(cte: CommonTableExpression, result: SelectResult): TableSchema { + val labels = cte.columns ?: result.columns + if (labels.size != result.columns.size) { + throw DiskExecutorException( + "common table expression '${cte.name}' expects ${labels.size} columns but its query produces ${result.columns.size}" + ) + } + return derivedSchema(cte.name, result, labels) + } + + /** + * A synthetic schema for a CTE's materialized rows: same idea as + * [kestrel.engine.SelectRunner.derivedSchema] - each column's type is inferred from the first + * non-NULL value seen in it (defaulting to TEXT when every row is NULL there), since a CTE has + * no backing [TableSchema] of its own to borrow one from. + */ + private fun derivedSchema(name: String, result: SelectResult, labels: List = result.columns): TableSchema { + val columns = labels.mapIndexed { index, columnName -> + val type = result.rows.asSequence().map { it[index] }.firstOrNull { !it.isNull }?.type ?: ColumnType.Text + Column(columnName, type, notNull = false, primaryKey = false) + } + return TableSchema(name, columns) + } + + private class CteScope(val name: String, val schema: TableSchema, var rows: List>) + + /** One FROM/JOIN source's row supply: either a real disk table or a CTE's already-materialized rows. */ + private sealed interface DiskFromSource { + val schema: TableSchema + fun scan(txn: Long): List> + } + + private class DiskTableSource(val table: DiskTable) : DiskFromSource { + override val schema get() = table.schema + override fun scan(txn: Long): List> = table.scan(txn).map { it.second.values } + } + + private class CteRowSource(override val schema: TableSchema, val rows: List>) : DiskFromSource { + override fun scan(txn: Long): List> = rows + } + + /** + * Resolves one FROM/JOIN [name] to its row supply: an active CTE shadows a same-named real + * table or view (checked innermost-first via [MutableList.lastOrNull], exactly like + * [kestrel.engine.SelectRunner.resolveNamedSource]'s own CTE lookup), then a temp table, then a + * real disk table, and only then a view - materialized fresh by running its defining query + * inside the same [txn], with [expandingViews] guarding against a view (directly or + * transitively) referencing itself, mirroring [kestrel.engine.SelectRunner.resolveNamedSource] + * exactly. A temp table's current rows are captured into a [CteRowSource] snapshot exactly like + * a CTE's or view's - a plain SELECT never mutates its own FROM source mid-query, so this is + * exactly as fresh as scanning a live [DiskTable] would be, and it is what gives a temp table + * full JOIN/subquery/aggregate participation for free instead of needing its own parallel query + * pipeline. A name that is neither falls through to [DiskCatalog.table], solely for its + * "no such table" [DiskCatalogException] - already confirmed absent from + * [DiskCatalog.temporaryTableOrNull]/[DiskCatalog.tableOrNull]/[DiskCatalog.viewOrNull], so this + * always throws, preserving the exact exception type callers relied on before views existed. + */ + private fun resolveFromSource(name: String, txn: Long): DiskFromSource { + activeCtes.lastOrNull { it.name.equals(name, ignoreCase = true) }?.let { return CteRowSource(it.schema, it.rows) } + catalog.temporaryTableOrNull(name)?.let { return CteRowSource(it.schema, it.rows.map { row -> row.values }) } + catalog.tableOrNull(name)?.let { return DiskTableSource(it) } + val view = catalog.viewOrNull(name) ?: return DiskTableSource(catalog.table(name)) + val key = name.lowercase() + if (!expandingViews.add(key)) { + throw DiskExecutorException("recursive view definition: $name") + } + try { + val result = runQuery(view, txn, outer = null) + return CteRowSource(derivedSchema(name, result), result.rows) + } finally { + expandingViews.remove(key) + } + } + + /** + * Resolves one FROM/JOIN [item] to its row supply, mirroring + * [kestrel.engine.SelectRunner.resolveSource]/`buildSources`: a [TableSource] falls through to + * [resolveFromSource] by name (CTE/table/view, unchanged); a non-LATERAL [SubquerySource] (a + * derived table) runs its query eagerly, right here, inside the same [txn] - it cannot correlate + * to anything, so its rows are fixed for the whole statement, exactly like a materialized view. A + * LATERAL [SubquerySource] is different: its rows depend on whichever row of an earlier FROM/JOIN + * source it correlates with, which does not exist yet at this point - only its column *names* are + * needed here (via [lateralSchema], computed structurally from the AST without running anything) + * for offset bookkeeping; its real rows are produced later, per left row, in [executeLateralJoin]. + */ + private fun resolveFromItem(item: FromItem, txn: Long): DiskFromSource = when (item) { + is TableSource -> resolveFromSource(item.name, txn) + is SubquerySource -> + if (item.lateral) { + CteRowSource(lateralSchema(item), emptyList()) + } else { + val result = runQuery(item.select, txn, outer = null) + CteRowSource(derivedSchema(item.alias, result), result.rows) + } + } + + private fun lateralSchema(item: SubquerySource): TableSchema { + val labels = structuralColumnNames(item.select) + val columns = labels.map { Column(it, ColumnType.Text, notNull = false, primaryKey = false) } + return TableSchema(item.alias, columns) + } + + /** + * The output column names of [query], derived purely from its AST (no execution, no catalog row + * data) - needed to give a LATERAL derived table a stable schema before any row of the source it + * correlates with is known. Mirrors [kestrel.engine.SelectRunner.structuralColumnNames] 1:1. + */ + private fun structuralColumnNames(query: QueryStatement, localCtes: Map> = emptyMap()): List = + when (query) { + is SelectStatement -> when (val projection = query.projection) { + is StarProjection -> + structuralSourceColumnNames(query.from, localCtes) + + query.joins.flatMap { structuralSourceColumnNames(it.from, localCtes) } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> columnLabel(item, index) } + } + is SetOperationStatement -> structuralColumnNames(query.left, localCtes) + is WithQuery -> { + val extended = localCtes.toMutableMap() + for (cte in query.ctes) { + extended[cte.name.lowercase()] = cte.columns ?: structuralColumnNames(cte.query, extended) + } + structuralColumnNames(query.body, extended) + } + } + + private fun structuralSourceColumnNames(item: FromItem, localCtes: Map>): List = + when (item) { + is TableSource -> { + val key = item.name.lowercase() + localCtes[key] + ?: activeCtes.lastOrNull { it.name.equals(item.name, ignoreCase = true) }?.schema?.columnNames + ?: catalog.temporaryTableOrNull(item.name)?.schema?.columnNames + ?: catalog.tableOrNull(item.name)?.schema?.columnNames + ?: catalog.viewOrNull(item.name)?.let { structuralColumnNames(it, localCtes) } + ?: throw DiskExecutorException("no such table: ${item.name}") + } + is SubquerySource -> structuralColumnNames(item.select, localCtes) + } + + /** + * Runs [statement] against the disk backend inside the already-open [txn]. [outer] is + * non-null exactly when this call is itself a scalar/EXISTS/IN subquery nested inside another + * statement's expression, reached through [Evaluator]'s [SubqueryRunner] hook - it lets a + * column [statement]'s own FROM/JOIN sources cannot resolve fall back to the row of whichever + * statement contains it, mirroring [kestrel.engine.SelectRunner.run]'s own `outer` parameter. + * A subquery therefore runs inside the same disk transaction as its containing statement + * rather than opening (and committing/aborting) one of its own. + */ + private fun runSelect(statement: SelectStatement, txn: Long, outer: ColumnResolver?): SelectResult { + val fromItems = listOf(statement.from) + statement.joins.map { it.from } + val fromSources = fromItems.map { item -> resolveFromItem(item, txn) } + val sources = buildJoinSources(fromItems, fromSources.map { it.schema }) + val subqueries = subqueryRunnerFor(txn) + var combined = initialRows(statement, txn, statement.from, fromSources.first()) + statement.joins.forEachIndexed { position, join -> + join.on?.let { rejectWindowFunction(it, "an ON condition") } + val rightSource = sources[position + 1] + val lateralSelect = lateralSelectOf(join.from) + combined = if (lateralSelect != null) { + executeLateralJoin(join, combined, lateralSelect, sources.subList(0, position + 1), rightSource, outer, subqueries, txn) + } else { + val rightRows = fromSources[position + 1].scan(txn) + applyJoin(join, combined, rightRows, sources.subList(0, position + 2), outer, subqueries, rightSource) + } + } + statement.where?.let { where -> + rejectWindowFunction(where, "WHERE") + combined = combined.filter { row -> isTrue(evaluateCombinedRow(sources, outer, subqueries, where, row)) } + } + statement.groupBy.forEach { rejectWindowFunction(it, "GROUP BY") } + statement.having?.let { rejectWindowFunction(it, "HAVING") } + statement.distinctOn.forEach { rejectWindowFunction(it, "DISTINCT ON") } + val orderKeys = resolvedOrderKeys(statement) + val aggregated = statement.groupBy.isNotEmpty() || statement.groupingSpec != null || + containsAggregate(statement.projection) || + (statement.having != null && containsAggregate(statement.having)) + if (statement.distinctOn.isNotEmpty()) validateDistinctOn(statement, orderKeys, aggregated) + val windowExpressions = collectWindowExpressions(statement.projection, orderKeys) + if (windowExpressions.isNotEmpty()) validateWindows(windowExpressions, aggregated) + if (aggregated) { + if (statement.projection is StarProjection) { + throw DiskExecutorException("cannot use * with GROUP BY or aggregate functions") + } + validateAggregation(statement, orderKeys, sources) + return GroupAggregation( + { expression -> resolveIndex(sources, expression) }, + { expression, row -> evaluateCombinedRow(sources, outer, subqueries, expression, row) }, + ::columnLabel, + subqueries, + { expression -> declaredCollationOrNull(sources, expression) } + ).aggregate(statement, orderKeys, combined) + } + val baseColumnCount = sources.sumOf { it.schema.columns.size } + if (windowExpressions.isNotEmpty()) { + val evaluateRow: (Expression, List) -> Value = { expression, row -> evaluateCombinedRow(sources, outer, subqueries, expression, row) } + val perExpression = windowExpressions.map { window -> + computeWindowValues(window, combined, evaluateRow) { expression -> declaredCollationOrNull(sources, expression) } + } + combined = combined.mapIndexed { rowIndex, row -> row + windowExpressions.indices.map { perExpression[it][rowIndex] } } + } + if (orderKeys.isNotEmpty()) { + combined = sortCombinedRows(sources, outer, subqueries, combined, orderKeys, windowExpressions, baseColumnCount) + } + if (statement.distinctOn.isNotEmpty()) { + val kept = LinkedHashMap, List>() + for (row in combined) { + val key = statement.distinctOn.map { evaluateWindowAware(sources, outer, subqueries, windowExpressions, baseColumnCount, it, row) } + val bucketKey = collationFoldedKey(key, statement.distinctOn) { expression -> declaredCollationOrNull(sources, expression) } + kept.putIfAbsent(bucketKey, row) + } + combined = kept.values.toList() + } + statement.offset?.let { if (it < 0) throw DiskExecutorException("OFFSET must not be negative") } + statement.limit?.let { if (it < 0) throw DiskExecutorException("LIMIT must not be negative") } + if (!statement.distinct) { + if (statement.offset != null || statement.limit != null) { + combined = combined.drop(statement.offset ?: 0) + statement.limit?.let { combined = combined.take(it) } + } + } + val (columns, projected) = projectCombined(statement.projection, sources, outer, subqueries, combined, windowExpressions, baseColumnCount) + var resultRows = projected + if (statement.distinct) { + if (statement.distinctOn.isEmpty()) { + val itemExpressions = when (statement.projection) { + is ExpressionProjection -> statement.projection.items.map { it.expression } + is StarProjection -> columns.map { ColumnExpression(it) } + } + resultRows = resultRows.distinctBy { row -> collationFoldedKey(row, itemExpressions) { expression -> declaredCollationOrNull(sources, expression) } } + } + if (statement.offset != null || statement.limit != null) { + resultRows = resultRows.drop(statement.offset ?: 0) + statement.limit?.let { resultRows = resultRows.take(it) } + } + } + return SelectResult(columns, resultRows) + } + + /** + * `ORDER BY` may name a `SELECT`-list alias instead of repeating its expression (e.g. + * `SELECT SUM(amount) AS total ... ORDER BY total`) - resolved here, once, before either + * [GroupAggregation] or [sortCombinedRows] ever sees the order keys, mirroring + * [kestrel.engine.SelectRunner]'s own `resolveOrderAlias`. A qualified column + * (`ORDER BY t.total`) can never be an alias reference, so it is left untouched. + */ + private fun resolvedOrderKeys(statement: SelectStatement): List = + statement.orderBy?.keys.orEmpty().map { key -> OrderKey(resolveOrderAlias(statement.projection, key.expression), key.descending, key.nullsFirst) } + + private fun resolveOrderAlias(projection: Projection, expression: Expression): Expression { + if (expression is ColumnExpression && expression.qualifier == null && projection is ExpressionProjection) { + val aliased = projection.items.firstOrNull { it.alias?.equals(expression.name, ignoreCase = true) == true } + if (aliased != null) return aliased.expression + } + return expression + } + + /** One joined table's schema plus the offset of its first column within a combined `List` row. */ + private class JoinSource(val alias: String, val schema: TableSchema, val offset: Int) + + private fun buildJoinSources(fromItems: List, schemas: List): List { + val sources = mutableListOf() + var offset = 0 + for ((item, schema) in fromItems.zip(schemas)) { + val alias = sourceLabel(item) + if (sources.any { it.alias.equals(alias, ignoreCase = true) }) { + throw DiskExecutorException("duplicate table alias: $alias") + } + sources.add(JoinSource(alias, schema, offset)) + offset += schema.columns.size + } + return sources + } + + private fun sourceLabel(item: FromItem): String = when (item) { + is TableSource -> item.alias ?: item.name + is SubquerySource -> item.alias + } + + /** + * The candidate rows a query's FIRST source starts from: an indexed lookup narrowed by WHERE + * via [candidateRows] when there is no JOIN and the source is a real disk table (preserving the + * same pushdown a plain single-table SELECT already gets), or a full scan otherwise - mirroring + * [kestrel.engine.SelectRunner]'s own choice to fall back to a full scan for the driving table + * as soon as more than one source is involved, since WHERE may combine columns from sources + * that are not open yet at this point in the plan. A CTE-backed source has no index to push + * down into regardless of JOIN, so it always scans its already-materialized rows; either way + * [statement.where] is re-applied in full immediately after by the caller, so skipping pushdown + * here only costs performance, never correctness. + */ + private fun initialRows(statement: SelectStatement, txn: Long, item: FromItem, fromSource: DiskFromSource): List> = + if (statement.joins.isEmpty() && fromSource is DiskTableSource) { + candidateRows(fromSource.table, txn, statement.where, (item as TableSource).alias).map { it.second.values } + } else { + fromSource.scan(txn) + } + + /** + * Nested-loop join: for every row already combined from earlier sources, scans the whole of + * [rightRows] evaluating [JoinClause.on] against each pairing. A LEFT/FULL JOIN row left + * unmatched still contributes one row, padded with NULLs for [rightSource]'s columns; a RIGHT/ + * FULL JOIN additionally emits every [rightRows] entry that never matched any [left] row, padded + * with NULLs for everything [rightSource.offset] counts as coming before it. Deliberately no + * hash-join optimization for an equality condition (unlike [kestrel.engine.SelectRunner]'s + * `hashJoinKeys`) - a documented scope cut: this JOIN exists to be correct first, in a solo + * session without peer review, and every disk-backed table already pays a full heap scan for + * any predicate an index cannot serve, so nested loop is not a new order-of-magnitude cost here. + */ + private fun applyJoin( + join: JoinClause, + left: List>, + rightRows: List>, + sourcesSoFar: List, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + rightSource: JoinSource + ): List> { + val nullRight = List(rightSource.schema.columns.size) { Value.NullValue } + val nullLeft = List(rightSource.offset) { Value.NullValue } + val outerLeft = join.type == JoinType.Left || join.type == JoinType.Full + val outerRight = join.type == JoinType.Right || join.type == JoinType.Full + val condition = join.on + val result = mutableListOf>() + val matchedRight = BooleanArray(rightRows.size) + for (leftRow in left) { + var matched = false + rightRows.forEachIndexed { index, rightRow -> + val candidate = leftRow + rightRow + if (condition == null || isTrue(evaluateCombinedRow(sourcesSoFar, outer, subqueries, condition, candidate))) { + result.add(candidate) + matched = true + matchedRight[index] = true + } + } + if (!matched && outerLeft) result.add(leftRow + nullRight) + } + if (outerRight) { + rightRows.forEachIndexed { index, rightRow -> if (!matchedRight[index]) result.add(nullLeft + rightRow) } + } + return result + } + + private fun lateralSelectOf(item: FromItem): QueryStatement? = (item as? SubquerySource)?.takeIf { it.lateral }?.select + + /** + * Unlike every other join, a LATERAL derived table has no fixed row set: it is re-evaluated + * once per row already accumulated on the left, correlated to that row via [lateralOuterResolver] + * exactly like a correlated subquery in WHERE/SELECT (same [SubqueryRunner]/[Evaluator] plumbing, + * just invoked from FROM instead) - mirrors [kestrel.engine.SelectRunner.executeLateralJoin] 1:1. + * Hash-joining is meaningless here (there is no static right-hand row set to build buckets from), + * so this is always a nested loop, matching how real engines execute LATERAL. + */ + private fun executeLateralJoin( + join: JoinClause, + left: List>, + lateralSelect: QueryStatement, + sourcesSoFar: List, + rightSource: JoinSource, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + txn: Long + ): List> { + val nullRight = List(rightSource.schema.columns.size) { Value.NullValue } + val allSources = sourcesSoFar + rightSource + val condition = join.on + val result = mutableListOf>() + for (leftRow in left) { + val rightRows = runQuery(lateralSelect, txn, lateralOuterResolver(sourcesSoFar, leftRow)).rows + var matched = false + for (rightRow in rightRows) { + val candidate = leftRow + rightRow + if (condition == null || isTrue(evaluateCombinedRow(allSources, outer, subqueries, condition, candidate))) { + result.add(candidate) + matched = true + } + } + if (!matched && join.type == JoinType.Left) result.add(leftRow + nullRight) + } + return result + } + + /** + * Resolves a column against [sourcesSoFar] - everything to the left of a LATERAL subquery in the + * FROM clause - and nothing else: a name that does not resolve there means the expression names a + * source not joined yet (or the LATERAL item itself), which SQL forbids, reported with a clear + * LATERAL-specific message rather than the generic "no such column". + */ + private fun lateralOuterResolver(sourcesSoFar: List, leftRow: List): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value = + resolveIndexOrNull(sourcesSoFar, expression)?.let { leftRow[it] } + ?: throw DiskExecutorException( + "LATERAL subquery cannot reference ${expression.name}: not joined yet at this point in the FROM clause" + ) + + override fun declaredCollation(expression: ColumnExpression): Collation? = + resolveIndexOrNull(sourcesSoFar, expression)?.let { declaredCollationOrNull(sourcesSoFar, expression) } + } + + private fun resolveIndexOrNull(sources: List, expression: ColumnExpression): Int? { + val qualifier = expression.qualifier + val matches = mutableListOf() + for (source in sources) { + if (qualifier != null && !source.alias.equals(qualifier, ignoreCase = true)) continue + val local = source.schema.columnIndex(expression.name) ?: continue + matches.add(source.offset + local) + } + return when { + matches.isEmpty() -> null + matches.size > 1 -> throw DiskExecutorException("ambiguous column reference: ${expression.name}") + else -> matches.single() + } + } + + private fun resolveIndex(sources: List, expression: ColumnExpression): Int = + resolveIndexOrNull(sources, expression) + ?: throw DiskExecutorException("no such column: ${expression.name}") + + /** [expression]'s declared `COLLATE` from whichever [sources] entry it resolves against - mirrors [resolveIndexOrNull]'s own matching, `null` on no match or an ambiguous one. */ + private fun declaredCollationOrNull(sources: List, expression: ColumnExpression): Collation? { + val qualifier = expression.qualifier + val matches = mutableListOf() + for (source in sources) { + if (qualifier != null && !source.alias.equals(qualifier, ignoreCase = true)) continue + val column = source.schema.column(expression.name) ?: continue + matches.add(column.collation) + } + return matches.singleOrNull() + } + + /** + * Resolves [expression]'s columns against [sources] first, falling back to [outer] (the + * enclosing statement's row, when this call is itself part of a correlated subquery) exactly + * once a name cannot be found locally - mirroring [kestrel.engine.SelectRunner]'s own + * `rowResolver`. The resulting [Evaluator] carries [subqueries] so any [ScalarSubqueryExpression]/ + * [ExistsExpression]/[InSubqueryExpression] nested inside [expression] can itself run, correlated + * back to this very row through that same resolver. + */ + private fun evaluateCombinedRow( + sources: List, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + expression: Expression, + row: List + ): Value { + val resolver = object : ColumnResolver { + override fun resolve(columnExpression: ColumnExpression): Value { + val index = resolveIndexOrNull(sources, columnExpression) + return if (index != null) { + row[index] + } else { + (outer ?: throw DiskExecutorException("no such column: ${columnExpression.name}")).resolve(columnExpression) + } + } + + override fun declaredCollation(columnExpression: ColumnExpression): Collation? = + declaredCollationOrNull(sources, columnExpression) ?: outer?.declaredCollation(columnExpression) + } + return Evaluator(resolver, subqueries).evaluate(expression) + } + + private fun sortCombinedRows( + sources: List, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + rows: List>, + orderKeys: List, + windowExpressions: List, + baseColumnCount: Int + ): List> { + val keyed = rows.map { row -> + row to orderKeys.map { key -> evaluateWindowAware(sources, outer, subqueries, windowExpressions, baseColumnCount, key.expression, row) } + } + val comparator = Comparator> { left, right -> + for ((index, key) in orderKeys.withIndex()) { + val comparison = compareOrderKeyValues(left[index], right[index], key) { expression -> + declaredCollationOrNull(sources, expression) + } + if (comparison != 0) return@Comparator comparison + } + 0 + } + return keyed.sortedWith(compareBy(comparator) { it.second }).map { it.first } + } + + /** + * When [windowExpressions] is non-empty, [row] may carry extra trailing values appended to + * [combined][runSelect] (one per entry, right after the real columns) - rewrites any + * [WindowExpression] node in [expression] into a [LiteralExpression] reading straight from + * that trailing slot before [evaluateCombinedRow] ever sees it, mirroring + * [kestrel.engine.SelectRunner]'s own `evaluateRow`/`substituteWindowValues`. + */ + private fun evaluateWindowAware( + sources: List, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + windowExpressions: List, + baseColumnCount: Int, + expression: Expression, + row: List + ): Value { + val prepared = if (windowExpressions.isEmpty()) expression else substituteWindowValues(expression, row, windowExpressions, baseColumnCount) + return evaluateCombinedRow(sources, outer, subqueries, prepared, row) + } + + private fun substituteWindowValues( + expression: Expression, + row: List, + windowExpressions: List, + baseColumnCount: Int + ): Expression { + fun recurse(inner: Expression) = substituteWindowValues(inner, row, windowExpressions, baseColumnCount) + return when (expression) { + is WindowExpression -> LiteralExpression(row[baseColumnCount + windowExpressions.indexOf(expression)]) + is BinaryExpression -> expression.copy(left = recurse(expression.left), right = recurse(expression.right)) + is UnaryExpression -> expression.copy(operand = recurse(expression.operand)) + is CallExpression -> expression.copy(arguments = expression.arguments.map(::recurse)) + is IsNullExpression -> expression.copy(operand = recurse(expression.operand)) + is InExpression -> expression.copy(operand = recurse(expression.operand), values = expression.values.map(::recurse)) + is BetweenExpression -> expression.copy(operand = recurse(expression.operand), low = recurse(expression.low), high = recurse(expression.high)) + is LikeExpression -> expression.copy(operand = recurse(expression.operand), pattern = recurse(expression.pattern)) + is CaseExpression -> expression.copy( + operand = expression.operand?.let(::recurse), + branches = expression.branches.map { it.copy(condition = recurse(it.condition), result = recurse(it.result)) }, + elseExpression = expression.elseExpression?.let(::recurse) + ) + is CastExpression -> expression.copy(operand = recurse(expression.operand)) + is CollateExpression -> expression.copy(operand = recurse(expression.operand)) + else -> expression + } + } + + private fun projectCombined( + projection: Projection, + sources: List, + outer: ColumnResolver?, + subqueries: SubqueryRunner, + rows: List>, + windowExpressions: List, + baseColumnCount: Int + ): Pair, List>> = + when (projection) { + // rows may carry trailing window-function values appended in runSelect (only reachable + // here if ORDER BY names one directly, since SELECT * cannot spell an OVER clause + // itself) - trimmed back off so the row width matches the schema's own columns. + StarProjection -> sources.flatMap { it.schema.columnNames } to rows.map { it.subList(0, baseColumnCount) } + is ExpressionProjection -> { + val columns = projection.items.mapIndexed { index, item -> columnLabel(item, index) } + val values = rows.map { row -> + projection.items.map { item -> + evaluateWindowAware(sources, outer, subqueries, windowExpressions, baseColumnCount, item.expression, row) + } + } + columns to values + } + } + + private fun columnLabel(item: SelectItem, index: Int): String = + item.alias ?: when (val expression = item.expression) { + is ColumnExpression -> expression.name + is FunctionExpression -> expression.name.lowercase() + is CallExpression -> expression.name.lowercase() + else -> "column${index + 1}" + } + + /** + * Deliberately duplicated from [kestrel.engine.SelectRunner]'s own `containsAggregate`/ + * `requireGrouped` (same reasoning as [coerceForColumn]): this executor must stay free-standing + * from the in-memory engine's own validation, so a change to one can never silently affect the + * other. `else -> false` covers every [Expression] variant this executor gives no + * aggregation-aware meaning to (a literal, a window function - rejected whenever it is actually + * evaluated, by [Evaluator] itself - or a subquery, whose own aggregates belong to that nested + * [SelectStatement]'s independent [runSelect] call, never to this one). + */ + private fun containsAggregate(projection: Projection): Boolean = when (projection) { + StarProjection -> false + is ExpressionProjection -> projection.items.any { containsAggregate(it.expression) } + } + + private fun containsAggregate(expression: Expression): Boolean = when (expression) { + is FunctionExpression -> true + is BinaryExpression -> containsAggregate(expression.left) || containsAggregate(expression.right) + is UnaryExpression -> containsAggregate(expression.operand) + is CallExpression -> expression.arguments.any { containsAggregate(it) } + is IsNullExpression -> containsAggregate(expression.operand) + is InExpression -> containsAggregate(expression.operand) || expression.values.any { containsAggregate(it) } + is BetweenExpression -> + containsAggregate(expression.operand) || containsAggregate(expression.low) || containsAggregate(expression.high) + is LikeExpression -> containsAggregate(expression.operand) || containsAggregate(expression.pattern) + is CaseExpression -> + (expression.operand?.let { containsAggregate(it) } ?: false) || + expression.branches.any { containsAggregate(it.condition) || containsAggregate(it.result) } || + (expression.elseExpression?.let { containsAggregate(it) } ?: false) + is CastExpression -> containsAggregate(expression.operand) + is CollateExpression -> containsAggregate(expression.operand) + else -> false + } + + /** See [kestrel.engine.SelectRunner]'s `Plan.validateDistinctOn` for why both checks exist. */ + private fun validateDistinctOn(statement: SelectStatement, orderKeys: List, aggregated: Boolean) { + if (aggregated) { + throw DiskExecutorException("DISTINCT ON cannot be combined with GROUP BY or aggregate functions") + } + if (statement.orderBy != null) { + val leading = orderKeys.take(statement.distinctOn.size).map { it.expression } + if (leading != statement.distinctOn) { + throw DiskExecutorException("SELECT DISTINCT ON expressions must match initial ORDER BY expressions") + } + } + } + + private fun validateAggregation(statement: SelectStatement, orderKeys: List, sources: List) { + val groupColumns = statement.groupBy.filterIsInstance() + .map { resolveIndex(sources, it) }.toSet() + if (statement.projection is ExpressionProjection) { + statement.projection.items.forEach { requireGrouped(it.expression, statement.groupBy, groupColumns, sources) } + } + statement.having?.let { requireGrouped(it, statement.groupBy, groupColumns, sources) } + orderKeys.forEach { requireGrouped(it.expression, statement.groupBy, groupColumns, sources) } + } + + private fun requireGrouped(expression: Expression, groupBy: List, groupColumns: Set, sources: List) { + if (expression in groupBy) return + when (expression) { + is FunctionExpression -> Unit + is ColumnExpression -> if (resolveIndex(sources, expression) !in groupColumns) { + throw DiskExecutorException( + "column ${expression.name} must appear in GROUP BY or be used in an aggregate function" + ) + } + is BinaryExpression -> { + requireGrouped(expression.left, groupBy, groupColumns, sources) + requireGrouped(expression.right, groupBy, groupColumns, sources) + } + is UnaryExpression -> requireGrouped(expression.operand, groupBy, groupColumns, sources) + is CallExpression -> expression.arguments.forEach { requireGrouped(it, groupBy, groupColumns, sources) } + is IsNullExpression -> requireGrouped(expression.operand, groupBy, groupColumns, sources) + is InExpression -> { + requireGrouped(expression.operand, groupBy, groupColumns, sources) + expression.values.forEach { requireGrouped(it, groupBy, groupColumns, sources) } + } + is BetweenExpression -> { + requireGrouped(expression.operand, groupBy, groupColumns, sources) + requireGrouped(expression.low, groupBy, groupColumns, sources) + requireGrouped(expression.high, groupBy, groupColumns, sources) + } + is LikeExpression -> { + requireGrouped(expression.operand, groupBy, groupColumns, sources) + requireGrouped(expression.pattern, groupBy, groupColumns, sources) + } + is CaseExpression -> { + expression.operand?.let { requireGrouped(it, groupBy, groupColumns, sources) } + expression.branches.forEach { + requireGrouped(it.condition, groupBy, groupColumns, sources) + requireGrouped(it.result, groupBy, groupColumns, sources) + } + expression.elseExpression?.let { requireGrouped(it, groupBy, groupColumns, sources) } + } + is CastExpression -> requireGrouped(expression.operand, groupBy, groupColumns, sources) + is CollateExpression -> requireGrouped(expression.operand, groupBy, groupColumns, sources) + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> requireGrouped(expression.operand, groupBy, groupColumns, sources) + else -> Unit + } + } + + /** Window functions are only legal in the SELECT list and ORDER BY - never WHERE/GROUP BY/HAVING/ON. */ + private fun containsWindowFunction(expression: Expression): Boolean = when (expression) { + is WindowExpression -> true + is BinaryExpression -> containsWindowFunction(expression.left) || containsWindowFunction(expression.right) + is UnaryExpression -> containsWindowFunction(expression.operand) + is CallExpression -> expression.arguments.any { containsWindowFunction(it) } + is IsNullExpression -> containsWindowFunction(expression.operand) + is InExpression -> containsWindowFunction(expression.operand) || expression.values.any { containsWindowFunction(it) } + is BetweenExpression -> + containsWindowFunction(expression.operand) || containsWindowFunction(expression.low) || containsWindowFunction(expression.high) + is LikeExpression -> containsWindowFunction(expression.operand) || containsWindowFunction(expression.pattern) + is CaseExpression -> + (expression.operand?.let { containsWindowFunction(it) } ?: false) || + expression.branches.any { containsWindowFunction(it.condition) || containsWindowFunction(it.result) } || + (expression.elseExpression?.let { containsWindowFunction(it) } ?: false) + is CastExpression -> containsWindowFunction(expression.operand) + is CollateExpression -> containsWindowFunction(expression.operand) + else -> false + } + + private fun rejectWindowFunction(expression: Expression, clause: String) { + if (containsWindowFunction(expression)) throw DiskExecutorException("window functions are not allowed in $clause") + } + + /** Every distinct window-function call reachable from [expression] (nested inside arithmetic, CASE, ...). */ + private fun collectWindowExpressions(expression: Expression): List = when (expression) { + is WindowExpression -> listOf(expression) + is BinaryExpression -> collectWindowExpressions(expression.left) + collectWindowExpressions(expression.right) + is UnaryExpression -> collectWindowExpressions(expression.operand) + is CallExpression -> expression.arguments.flatMap { collectWindowExpressions(it) } + is IsNullExpression -> collectWindowExpressions(expression.operand) + is InExpression -> collectWindowExpressions(expression.operand) + expression.values.flatMap { collectWindowExpressions(it) } + is BetweenExpression -> + collectWindowExpressions(expression.operand) + collectWindowExpressions(expression.low) + collectWindowExpressions(expression.high) + is LikeExpression -> collectWindowExpressions(expression.operand) + collectWindowExpressions(expression.pattern) + is CaseExpression -> + (expression.operand?.let { collectWindowExpressions(it) } ?: emptyList()) + + expression.branches.flatMap { collectWindowExpressions(it.condition) + collectWindowExpressions(it.result) } + + (expression.elseExpression?.let { collectWindowExpressions(it) } ?: emptyList()) + is CastExpression -> collectWindowExpressions(expression.operand) + is CollateExpression -> collectWindowExpressions(expression.operand) + else -> emptyList() + } + + /** + * Every distinct window-function call in the SELECT list or (post alias-resolution) ORDER BY - + * the only two clauses where one is allowed, mirroring [kestrel.engine.SelectRunner]'s own + * `windowExpressions` property. + */ + private fun collectWindowExpressions(projection: Projection, orderKeys: List): List { + val projectionWindows = (projection as? ExpressionProjection)?.items?.flatMap { collectWindowExpressions(it.expression) } ?: emptyList() + val orderWindows = orderKeys.flatMap { collectWindowExpressions(it.expression) } + return (projectionWindows + orderWindows).distinct() + } + + /** + * Window functions are rejected outright alongside GROUP BY/aggregates ([aggregated]) - + * windowing over an already-grouped result is valid SQL but requires running + * [GroupAggregation] first and then a second pass over its output rows, which this executor + * does not implement (same scope cut as [kestrel.engine.SelectRunner]). RANK/DENSE_RANK need an + * explicit tie-break (unlike ROW_NUMBER, which falls back to partition arrival order when OVER + * has no ORDER BY) and nesting one window function inside another's argument/PARTITION BY/ + * ORDER BY/frame offset is invalid SQL in the first place. + */ + private fun validateWindows(windowExpressions: List, aggregated: Boolean) { + if (aggregated) { + throw DiskExecutorException("window functions cannot be combined with GROUP BY or aggregate functions in the same query") + } + for (window in windowExpressions) { + if ((window.kind == WindowFunctionKind.Rank || window.kind == WindowFunctionKind.DenseRank) && window.orderBy.isEmpty()) { + throw DiskExecutorException("${window.name}() OVER (...) requires an ORDER BY clause") + } + if (window.frame != null && window.kind !in FRAME_CAPABLE_KINDS) { + throw DiskExecutorException("${window.name}() cannot be used with a ROWS/RANGE/GROUPS frame clause") + } + val nested = listOfNotNull( + window.argument, window.offset, window.defaultValue, + window.frame?.start?.offset, window.frame?.end?.offset + ) + window.partitionBy + window.orderBy.map { it.expression } + if (nested.any { containsWindowFunction(it) }) { + throw DiskExecutorException("window functions cannot be nested") + } + } + } + + private fun computeWindowValues( + window: WindowExpression, + combined: List>, + evaluateRow: (Expression, List) -> Value, + declaredCollationOf: (ColumnExpression) -> Collation? = { null } + ): List { + val values = arrayOfNulls(combined.size) + val partitions = combined.indices.groupBy { index -> window.partitionBy.map { evaluateRow(it, combined[index]) } } + for (memberIndices in partitions.values) { + val ordered = if (window.orderBy.isEmpty()) { + memberIndices + } else { + memberIndices.sortedWith(Comparator { first, second -> + var result = 0 + for (key in window.orderBy) { + val comparison = compareOrderKeyValues( + evaluateRow(key.expression, combined[first]), + evaluateRow(key.expression, combined[second]), + key, + declaredCollationOf + ) + if (comparison != 0) { + result = comparison + break + } + } + result + }) + } + assignWindowValues(window, ordered, combined, values, evaluateRow) + } + return values.map { it ?: error("window value not computed for every row") } + } + + private fun assignWindowValues( + window: WindowExpression, + ordered: List, + combined: List>, + out: Array, + evaluateRow: (Expression, List) -> Value + ) { + when (window.kind) { + WindowFunctionKind.RowNumber -> + ordered.forEachIndexed { position, index -> out[index] = Value.IntegerValue((position + 1).toLong()) } + WindowFunctionKind.Rank, WindowFunctionKind.DenseRank -> { + var rank = 0 + var denseRank = 0 + var previousKey: List? = null + ordered.forEachIndexed { position, index -> + val key = window.orderBy.map { evaluateRow(it.expression, combined[index]) } + if (previousKey == null || key != previousKey) { + rank = position + 1 + denseRank++ + previousKey = key + } + out[index] = Value.IntegerValue((if (window.kind == WindowFunctionKind.Rank) rank else denseRank).toLong()) + } + } + WindowFunctionKind.Aggregate -> assignAggregateWindowValues(window, ordered, combined, out, evaluateRow) + WindowFunctionKind.Lag -> assignOffsetWindowValues(window, ordered, combined, out, direction = -1, evaluateRow) + WindowFunctionKind.Lead -> assignOffsetWindowValues(window, ordered, combined, out, direction = 1, evaluateRow) + WindowFunctionKind.FirstValue -> { + val frame = window.frame + if (frame == null) { + val firstValue = evaluateRow(window.argument!!, combined[ordered.first()]) + ordered.forEach { index -> out[index] = firstValue } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined, evaluateRow) + out[ordered[position]] = + if (members.isEmpty()) Value.NullValue else evaluateRow(window.argument!!, combined[ordered[members.first()]]) + } + } + } + WindowFunctionKind.LastValue -> { + val frame = window.frame + if (frame == null) { + for (range in peerGroupRanges(window.orderBy, ordered, combined, evaluateRow)) { + val value = evaluateRow(window.argument!!, combined[ordered[range.last]]) + for (position in range) out[ordered[position]] = value + } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined, evaluateRow) + out[ordered[position]] = + if (members.isEmpty()) Value.NullValue else evaluateRow(window.argument!!, combined[ordered[members.last()]]) + } + } + } + WindowFunctionKind.Ntile -> assignNtileWindowValues(window, ordered, combined, out, evaluateRow) + } + } + + /** + * With no ORDER BY in OVER, the whole partition is one frame (every row gets the same value). + * With an ORDER BY, the default SQL frame is `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT + * ROW`: a running aggregate where rows tied on the order key (a "peer group") share one value - + * mirroring [kestrel.engine.SelectRunner]'s own `assignAggregateWindowValues` exactly. + */ + private fun assignAggregateWindowValues( + window: WindowExpression, + ordered: List, + combined: List>, + out: Array, + evaluateRow: (Expression, List) -> Value + ) { + val frame = window.frame + if (frame == null) { + for (range in peerGroupRanges(window.orderBy, ordered, combined, evaluateRow)) { + val value = aggregateWindowValue(window, (0..range.last).map { combined[ordered[it]] }, evaluateRow) + for (position in range) out[ordered[position]] = value + } + } else { + ordered.indices.forEach { position -> + val members = frameMembers(window, position, ordered, combined, evaluateRow) + out[ordered[position]] = aggregateWindowValue(window, members.map { combined[ordered[it]] }, evaluateRow) + } + } + } + + /** + * The frame's base [frameRange], narrowed by its [FrameExclude] mode (default + * [FrameExclude.NoOthers], which leaves the range untouched) - mirrors + * [kestrel.engine.SelectRunner]'s own `frameMembers` exactly. + */ + private fun frameMembers( + window: WindowExpression, + position: Int, + ordered: List, + combined: List>, + evaluateRow: (Expression, List) -> Value + ): List { + val range = frameRange(window, position, ordered, combined, evaluateRow) + if (range.isEmpty()) return emptyList() + return when (window.frame!!.exclude) { + FrameExclude.NoOthers -> range.toList() + FrameExclude.CurrentRow -> range.filter { it != position } + FrameExclude.Group -> { + val peer = peerBoundary(window.orderBy, position, ordered, combined, isStart = true, evaluateRow).. + peerBoundary(window.orderBy, position, ordered, combined, isStart = false, evaluateRow) + range.filter { it !in peer } + } + FrameExclude.Ties -> { + val peer = peerBoundary(window.orderBy, position, ordered, combined, isStart = true, evaluateRow).. + peerBoundary(window.orderBy, position, ordered, combined, isStart = false, evaluateRow) + range.filter { it == position || it !in peer } + } + } + } + + /** + * Resolves an explicit `ROWS|RANGE BETWEEN ... AND ...` [WindowFrame] to a range of positions + * into [ordered] for the row at [position] - clamped to the partition's edges rather than + * erroring, exactly like the implicit default frame already does. Mirrors + * [kestrel.engine.SelectRunner]'s own `frameRange` exactly. + */ + private fun frameRange( + window: WindowExpression, + position: Int, + ordered: List, + combined: List>, + evaluateRow: (Expression, List) -> Value + ): IntRange { + val frame = window.frame!! + if (frame.unit == FrameUnit.Groups) { + return groupsFrameRange(frame, window.orderBy, position, ordered, combined, evaluateRow) + } + val size = ordered.size + val start = resolveFrameBound(frame, frame.start, window, position, ordered, combined, isStart = true, evaluateRow) + val end = resolveFrameBound(frame, frame.end, window, position, ordered, combined, isStart = false, evaluateRow) + val clampedStart = start.coerceAtLeast(0) + val clampedEnd = end.coerceAtMost(size - 1) + return if (clampedStart > clampedEnd) IntRange.EMPTY else clampedStart..clampedEnd + } + + private fun resolveFrameBound( + frame: WindowFrame, + bound: FrameBound, + window: WindowExpression, + position: Int, + ordered: List, + combined: List>, + isStart: Boolean, + evaluateRow: (Expression, List) -> Value + ): Int = when (bound.type) { + FrameBoundType.UnboundedPreceding -> 0 + FrameBoundType.UnboundedFollowing -> ordered.size - 1 + FrameBoundType.CurrentRow -> + if (frame.unit == FrameUnit.Rows) position else peerBoundary(window.orderBy, position, ordered, combined, isStart, evaluateRow) + FrameBoundType.Preceding -> + if (frame.unit == FrameUnit.Rows) { + position - resolveIntegerFrameOffset(bound, combined[ordered[position]], "ROWS", evaluateRow) + } else { + rangeBoundary(bound, window.orderBy, position, ordered, combined, isStart, subtract = true, evaluateRow) + } + FrameBoundType.Following -> + if (frame.unit == FrameUnit.Rows) { + position + resolveIntegerFrameOffset(bound, combined[ordered[position]], "ROWS", evaluateRow) + } else { + rangeBoundary(bound, window.orderBy, position, ordered, combined, isStart, subtract = false, evaluateRow) + } + } + + private fun resolveIntegerFrameOffset( + bound: FrameBound, + row: List, + unit: String, + evaluateRow: (Expression, List) -> Value + ): Int { + val value = evaluateRow(bound.offset!!, row) + val amount = (value as? Value.IntegerValue)?.value ?: throw DiskExecutorException("$unit frame offset must be an integer") + if (amount < 0) throw DiskExecutorException("$unit frame offset must not be negative") + return amount.toInt() + } + + /** + * `GROUPS BETWEEN AND ` counts whole peer groups rather than physical rows + * ([FrameUnit.Rows]) or distance in a value ([FrameUnit.Range]) - mirrors + * [kestrel.engine.SelectRunner]'s own `groupsFrameRange` exactly. + */ + private fun groupsFrameRange( + frame: WindowFrame, + orderBy: List, + position: Int, + ordered: List, + combined: List>, + evaluateRow: (Expression, List) -> Value + ): IntRange { + val groups = peerGroupRanges(orderBy, ordered, combined, evaluateRow) + val currentGroup = groups.indexOfFirst { position in it } + val row = combined[ordered[position]] + val startGroup = resolveGroupBound(frame.start, currentGroup, groups.size, row, evaluateRow).coerceAtLeast(0) + val endGroup = resolveGroupBound(frame.end, currentGroup, groups.size, row, evaluateRow).coerceAtMost(groups.size - 1) + return if (startGroup > endGroup) IntRange.EMPTY else groups[startGroup].first..groups[endGroup].last + } + + private fun resolveGroupBound( + bound: FrameBound, + currentGroup: Int, + groupCount: Int, + row: List, + evaluateRow: (Expression, List) -> Value + ): Int = when (bound.type) { + FrameBoundType.UnboundedPreceding -> 0 + FrameBoundType.UnboundedFollowing -> groupCount - 1 + FrameBoundType.CurrentRow -> currentGroup + FrameBoundType.Preceding -> currentGroup - resolveIntegerFrameOffset(bound, row, "GROUPS", evaluateRow) + FrameBoundType.Following -> currentGroup + resolveIntegerFrameOffset(bound, row, "GROUPS", evaluateRow) + } + + /** + * `CURRENT ROW` in a RANGE frame means the boundary of the current row's peer group (every row + * tied with it on the OVER's ORDER BY, or the whole partition with no ORDER BY) - mirrors + * [kestrel.engine.SelectRunner]'s own `peerBoundary` exactly. + */ + private fun peerBoundary( + orderBy: List, + position: Int, + ordered: List, + combined: List>, + isStart: Boolean, + evaluateRow: (Expression, List) -> Value + ): Int { + if (orderBy.isEmpty()) return if (isStart) 0 else ordered.size - 1 + fun keyAt(index: Int) = orderBy.map { evaluateRow(it.expression, combined[ordered[index]]) } + val key = keyAt(position) + var boundary = position + if (isStart) { + while (boundary > 0 && keyAt(boundary - 1) == key) boundary-- + } else { + while (boundary < ordered.size - 1 && keyAt(boundary + 1) == key) boundary++ + } + return boundary + } + + /** + * ` PRECEDING`/` FOLLOWING` in a RANGE frame measure distance in the value of a + * single numeric or DATE ORDER BY column rather than a row count - binary search works because + * [ordered] is already sorted on this column. Mirrors [kestrel.engine.SelectRunner]'s own + * `rangeBoundary` exactly. + */ + private fun rangeBoundary( + bound: FrameBound, + orderBy: List, + position: Int, + ordered: List, + combined: List>, + isStart: Boolean, + subtract: Boolean, + evaluateRow: (Expression, List) -> Value + ): Int { + if (orderBy.size != 1) { + throw DiskExecutorException("a RANGE frame with a PRECEDING/FOLLOWING bound requires exactly one ORDER BY column") + } + val key = orderBy.single() + val current = rangeLogicalValue(key, combined[ordered[position]], evaluateRow) + val offset = resolveRangeFrameOffset(bound, combined[ordered[position]], evaluateRow) + val threshold = if (subtract) current - offset else current + offset + return if (isStart) { + rangeLowerBound(ordered, combined, key, threshold, evaluateRow) + } else { + rangeUpperBound(ordered, combined, key, threshold, evaluateRow) + } + } + + private fun rangeLowerBound( + ordered: List, + combined: List>, + key: OrderKey, + threshold: BigDecimal, + evaluateRow: (Expression, List) -> Value + ): Int { + var low = 0 + var high = ordered.size + while (low < high) { + val mid = (low + high) / 2 + if (rangeLogicalValue(key, combined[ordered[mid]], evaluateRow) >= threshold) high = mid else low = mid + 1 + } + return low + } + + private fun rangeUpperBound( + ordered: List, + combined: List>, + key: OrderKey, + threshold: BigDecimal, + evaluateRow: (Expression, List) -> Value + ): Int { + var low = 0 + var high = ordered.size + while (low < high) { + val mid = (low + high) / 2 + if (rangeLogicalValue(key, combined[ordered[mid]], evaluateRow) <= threshold) low = mid + 1 else high = mid + } + return low - 1 + } + + private fun rangeLogicalValue(key: OrderKey, row: List, evaluateRow: (Expression, List) -> Value): BigDecimal { + val magnitude = rangeMagnitude(evaluateRow(key.expression, row)) + return if (key.descending) magnitude.negate() else magnitude + } + + private fun rangeMagnitude(value: Value): BigDecimal = when (value) { + is Value.IntegerValue -> BigDecimal(value.value) + is Value.DecimalValue -> value.value + is Value.RealValue -> BigDecimal.valueOf(value.value) + is Value.DateValue -> BigDecimal(value.value.toEpochDay()) + else -> throw DiskExecutorException("a RANGE frame requires the ORDER BY column to be numeric or DATE") + } + + private fun resolveRangeFrameOffset(bound: FrameBound, row: List, evaluateRow: (Expression, List) -> Value): BigDecimal { + val offset = when (val value = evaluateRow(bound.offset!!, row)) { + is Value.IntegerValue -> BigDecimal(value.value) + is Value.DecimalValue -> value.value + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> throw DiskExecutorException("RANGE frame offset must be numeric") + } + if (offset.signum() < 0) throw DiskExecutorException("RANGE frame offset must not be negative") + return offset + } + + /** + * Splits [ordered] into contiguous "peer group" ranges of rows tied on the order key - or one + * range spanning the whole partition when there is no ORDER BY. Mirrors + * [kestrel.engine.SelectRunner]'s own `peerGroupRanges` exactly. + */ + private fun peerGroupRanges( + orderBy: List, + ordered: List, + combined: List>, + evaluateRow: (Expression, List) -> Value + ): List { + if (orderBy.isEmpty()) return listOf(ordered.indices) + val keys = ordered.map { index -> orderBy.map { key -> evaluateRow(key.expression, combined[index]) } } + val ranges = mutableListOf() + var i = 0 + while (i < ordered.size) { + var j = i + while (j < ordered.size && keys[j] == keys[i]) j++ + ranges.add(i until j) + i = j + } + return ranges + } + + private fun aggregateWindowValue(window: WindowExpression, rows: List>, evaluateRow: (Expression, List) -> Value): Value = + AggregateFunctions.compute(window.name, window.argument, window.star, rows, evaluateRow) + + /** + * LAG (direction -1) / LEAD (direction +1): reads [WindowExpression.argument] from the row + * [WindowExpression.offset] positions before/after the current one in [ordered] (default 1). + * Past either edge of the partition, falls back to [WindowExpression.defaultValue] (evaluated + * against the *current* row) or NULL. Mirrors [kestrel.engine.SelectRunner]'s own + * `assignOffsetWindowValues` exactly. + */ + private fun assignOffsetWindowValues( + window: WindowExpression, + ordered: List, + combined: List>, + out: Array, + direction: Int, + evaluateRow: (Expression, List) -> Value + ) { + ordered.forEachIndexed { position, index -> + val offset = resolveOffset(window, combined[index], evaluateRow) + val targetPosition = position + direction * offset + out[index] = if (targetPosition in ordered.indices) { + evaluateRow(window.argument!!, combined[ordered[targetPosition]]) + } else { + window.defaultValue?.let { evaluateRow(it, combined[index]) } ?: Value.NullValue + } + } + } + + private fun resolveOffset(window: WindowExpression, row: List, evaluateRow: (Expression, List) -> Value): Int { + val expression = window.offset ?: return 1 + val value = evaluateRow(expression, row) + val amount = (value as? Value.IntegerValue)?.value ?: throw DiskExecutorException("${window.name}() offset must be an integer") + if (amount < 0) throw DiskExecutorException("${window.name}() offset must not be negative") + return amount.toInt() + } + + /** + * NTILE(n): splits the partition into `n` buckets as evenly as possible - when the partition + * size does not divide evenly, the first `size % n` buckets get one extra row. Mirrors + * [kestrel.engine.SelectRunner]'s own `assignNtileWindowValues` exactly. + */ + private fun assignNtileWindowValues( + window: WindowExpression, + ordered: List, + combined: List>, + out: Array, + evaluateRow: (Expression, List) -> Value + ) { + val bucketValue = evaluateRow(window.argument!!, combined[ordered.first()]) + val buckets = (bucketValue as? Value.IntegerValue)?.value ?: throw DiskExecutorException("${window.name}() bucket count must be an integer") + if (buckets <= 0) throw DiskExecutorException("${window.name}() bucket count must be positive") + val bucketCount = buckets.toInt() + val base = ordered.size / bucketCount + val remainder = ordered.size % bucketCount + var position = 0 + for (bucket in 1..bucketCount) { + val size = base + if (bucket <= remainder) 1 else 0 + repeat(size) { + out[ordered[position]] = Value.IntegerValue(bucket.toLong()) + position++ + } + } + } + + /** + * Candidate (rid, row) pairs for [where] over [table]: an indexed + * point/range/secondary lookup via [DiskIndexAccessPlanner] when the + * predicate allows it, a full [DiskTable.scan] otherwise. Callers must + * still evaluate the full [where] against whatever this returns - the + * chosen plan only narrows candidates, it never claims to satisfy every + * conjunct of the predicate (see [DiskAccessPlan]'s own documentation). + */ + private fun candidateRows(table: DiskTable, txn: Long, where: Expression?, alias: String?): List> = + when (val plan = accessPlanner.plan(table.schema, table.secondaryIndexes, where, alias)) { + DiskAccessPlan.FullScan -> table.scan(txn) + is DiskAccessPlan.PrimaryKeyPoint -> listOfNotNull(table.findByPrimaryKey(txn, plan.key)) + is DiskAccessPlan.PrimaryKeyRange -> table.rangeByPrimaryKey(txn, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive) + is DiskAccessPlan.CompositePrimaryKeyPoint -> listOfNotNull(table.findByPrimaryKey(txn, plan.keys)) + is DiskAccessPlan.CompositePrimaryKeyRange -> + table.rangeByPrimaryKey(txn, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive, plan.prefix) + is DiskAccessPlan.Secondary -> table.findBySecondary(txn, plan.indexName, plan.keys) + is DiskAccessPlan.SecondaryRange -> + table.rangeBySecondary(txn, plan.indexName, plan.prefix, plan.lower, plan.lowerInclusive, plan.upper, plan.upperInclusive) + } + + private fun executeUpdate(statement: UpdateStatement, triggerContext: TriggerRowContext? = null): StatementResult { + catalog.temporaryTableOrNull(statement.tableName)?.let { return executeTemporaryUpdate(statement, it) } + val table = catalog.table(statement.tableName) + val schema = table.schema + val assignments = buildAssignments(schema, statement) + return withStatementTransaction { txn -> + withTriggerAtomicity(schema.name, txn, triggerContext) { + executeUpdateInTxn(statement, assignments, table, schema, txn, triggerContext) + } + } + } + + /** + * UPDATE on a temp table's plain [Table.rows] list - ported from the same whole-batch-then-swap + * shape as [executeUpdateInTxn]/[kestrel.engine.Executor.executeUpdateInner]: every row is + * planned into [newRows] first (evaluating SET/DEFAULT/GENERATED and validating NOT NULL) before + * [table.rows] is ever mutated, so a CHECK/UNIQUE/PRIMARY KEY failure on any one row leaves the + * table completely untouched rather than half-updated - the in-memory list equivalent of + * [DiskTable.update]'s own per-row validate-before-write ordering, just batched like the + * in-memory engine does instead of row-by-row like [DiskTable]. + */ + private fun executeTemporaryUpdate(statement: UpdateStatement, table: Table): StatementResult { + val schema = table.schema + val assignments = buildAssignments(schema, statement) + return withStatementTransaction { txn -> + val subqueries = subqueryRunnerFor(txn) + val oldRows = table.rows.toList() + val touchedRows = mutableListOf>() + val newRows = oldRows.map { row -> + val matches = statement.where == null || + isTrue(Evaluator(composeResolver(rowResolver(schema, row), null), subqueries).evaluate(statement.where)) + if (!matches) return@map row + val values = row.values.toMutableList() + val evaluator = Evaluator(composeResolver(rowResolver(schema, row), null), subqueries, this) + for ((index, expression) in assignments) { + val column = schema.columns[index] + values[index] = if (expression != null) { + coerceForColumn(evaluator.evaluate(expression), column) + } else { + column.default?.let { coerceForColumn(evaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values, subqueries) + validateTemporaryNotNull(schema, values) + val newRow = Row(values) + touchedRows.add(row to newRow) + newRow + } + for (row in newRows) enforceChecks(schema, row, subqueries) + enforceTemporaryPrimaryKeyUniqueness(schema, newRows) + enforceTemporaryUniqueConstraints(schema, newRows) + table.rows.clear() + table.rows.addAll(newRows) + val returning = statement.returning?.let { evaluateReturning(schema, touchedRows.map { it.second }, it, subqueries) } + UpdateResult(touchedRows.size, returning) + } + } + + private fun executeUpdateInTxn( + statement: UpdateStatement, + assignments: List>, + table: DiskTable, + schema: TableSchema, + txn: Long, + triggerContext: TriggerRowContext? + ): StatementResult { + val subqueries = subqueryRunnerFor(txn) + val matches = candidateRows(table, txn, statement.where, alias = null).filter { (_, row) -> + statement.where == null || + isTrue(Evaluator(composeResolver(rowResolver(schema, row), triggerContext), subqueries).evaluate(statement.where)) + } + // Built as a whole batch first (not applied inline) for the same reason as executeInsert: + // a self-referencing FK must see every row's final value for this statement, not just + // rows already committed to the table. + val planned = matches.map { (rid, row) -> + val evaluator = Evaluator(composeResolver(rowResolver(schema, row), triggerContext), subqueries, this) + val values = row.values.toMutableList() + for ((index, expression) in assignments) { + val column = schema.columns[index] + values[index] = if (expression != null) { + coerceForColumn(evaluator.evaluate(expression), column) + } else { + column.default?.let { coerceForColumn(evaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values, subqueries) + Triple(rid, row, Row(values)) + } + val newRows = planned.map { it.third } + // Fires before any constraint below is checked, same as executeInsert's BEFORE trigger - a + // BEFORE UPDATE trigger's body can reject the whole statement before this executor's own + // validation ever runs. + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, planned.map { it.second to it.third }, txn) + for ((_, _, newRow) in planned) { + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, newRow.values, uncommittedSiblings = newRows) + enforceChecks(schema, newRow, subqueries) + } + // Every enforced-unique key group (the PRIMARY KEY, single-column or composite, plus each + // UNIQUE constraint) is a valid FOREIGN KEY target, so each group needs its own + // cascade/RESTRICT pass on UPDATE - not just the PRIMARY KEY. Run before the parent's own + // rows are rewritten below, mirroring kestrel.engine.Executor.executeUpdate's own order. + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val changedPairs = planned.map { (_, oldRow, newRow) -> + columnIndices.map { oldRow.values[it] } to columnIndices.map { newRow.values[it] } + } + DiskForeignKeyEnforcer.cascadeUpdate(catalog, txn, schema.name, group, changedPairs) + } + for ((rid, _, newRow) in planned) table.update(txn, rid, newRow) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, planned.map { it.second to it.third }, txn) + val returning = statement.returning?.let { evaluateReturning(schema, planned.map { triple -> triple.third }, it, subqueries) } + return UpdateResult(planned.size, returning) + } + + private fun executeDelete(statement: DeleteStatement, triggerContext: TriggerRowContext? = null): StatementResult { + catalog.temporaryTableOrNull(statement.tableName)?.let { return executeTemporaryDelete(statement, it) } + val table = catalog.table(statement.tableName) + val schema = table.schema + return withStatementTransaction { txn -> + withTriggerAtomicity(schema.name, txn, triggerContext) { + executeDeleteInTxn(statement, table, schema, txn, triggerContext) + } + } + } + + private fun executeDeleteInTxn( + statement: DeleteStatement, table: DiskTable, schema: TableSchema, txn: Long, triggerContext: TriggerRowContext? + ): StatementResult { + val subqueries = subqueryRunnerFor(txn) + val matches = candidateRows(table, txn, statement.where, alias = null).filter { (_, row) -> + statement.where == null || + isTrue(Evaluator(composeResolver(rowResolver(schema, row), triggerContext), subqueries).evaluate(statement.where)) + } + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Delete, matches.map { it.second to null }, txn) + DiskForeignKeyEnforcer.cascadeDelete(catalog, txn, schema.name, matches.map { it.second }) + for ((rid, _) in matches) table.delete(txn, rid) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Delete, matches.map { it.second to null }, txn) + val returning = statement.returning?.let { evaluateReturning(schema, matches.map { it.second }, it, subqueries) } + return DeleteResult(matches.size, returning) + } + + /** DELETE on a temp table's plain [Table.rows] list - ported from the same shape as [executeDeleteInTxn]/[kestrel.engine.Executor.executeDeleteInner], minus trigger firing/[DiskForeignKeyEnforcer] cascade (both rejected outright at `CREATE TEMP TABLE` time). */ + private fun executeTemporaryDelete(statement: DeleteStatement, table: Table): StatementResult { + val schema = table.schema + return withStatementTransaction { txn -> + val subqueries = subqueryRunnerFor(txn) + val deletedRows = if (statement.where == null) { + table.rows.toList() + } else { + table.rows.filter { row -> + isTrue(Evaluator(composeResolver(rowResolver(schema, row), null), subqueries).evaluate(statement.where)) + } + } + table.rows.removeAll(deletedRows) + val returning = statement.returning?.let { evaluateReturning(schema, deletedRows, it, subqueries) } + DeleteResult(deletedRows.size, returning) + } + } + + private fun executeMerge(statement: MergeStatement): StatementResult { + val table = catalog.table(statement.targetTable) + val schema = table.schema + return withStatementTransaction { txn -> + withTriggerAtomicity(schema.name, txn, null) { + executeMergeInner(statement, table, schema, txn) + } + } + } + + /** + * Ported 1:1 from [kestrel.engine.Executor.executeMergeInner] - see that method's own doc for + * why [MergeStatement.onCondition] is matched against [originalTargetRows] (a snapshot of the + * target table taken once, before any mutation) rather than the live table, and for the "MERGE + * command cannot affect a target row a second time" guarantee. The one difference: "already + * affected" and "this pair's target row" are tracked by [RecordId] here rather than [Row] + * object identity (`===`) - a disk row is decoded fresh from its page on every read, so no two + * reads of "the same" row are ever the same object, but each row in [originalTargetRows] keeps + * the one [RecordId] it had at snapshot time for the rest of this statement (this MERGE is the + * only writer touching [table] during its own execution, and every [RecordId] this MERGE itself + * mutates is used at most once, guarded by [affectedRids] below, so it is never looked up again + * afterward under a since-changed identity). + */ + private fun executeMergeInner(statement: MergeStatement, table: DiskTable, schema: TableSchema, txn: Long): StatementResult { + val targetLabel = statement.targetAlias ?: statement.targetTable + val (sourceSchema, sourceRows) = resolveMergeSource(statement.source, txn) + val sourceLabel = sourceLabel(statement.source) + if (targetLabel.equals(sourceLabel, ignoreCase = true)) { + throw DiskExecutorException("MERGE target and source cannot share the same name or alias: $targetLabel") + } + checkMergeColumns(statement, schema, targetLabel, sourceSchema, sourceLabel) + + val subqueries = subqueryRunnerFor(txn) + val originalTargetRows = table.scan(txn) + val affectedRids = mutableSetOf() + val returnedRows = mutableListOf>() + var inserted = 0 + var updated = 0 + var deleted = 0 + + for (sourceRow in sourceRows) { + val matchedTargets = originalTargetRows.filter { (_, targetRow) -> + val resolver = mergeResolver(schema, targetLabel, targetRow, sourceSchema, sourceLabel, sourceRow) + isTrue(Evaluator(resolver, subqueries).evaluate(statement.onCondition)) + } + if (matchedTargets.isNotEmpty()) { + for ((targetRid, targetRow) in matchedTargets) { + val resolver = mergeResolver(schema, targetLabel, targetRow, sourceSchema, sourceLabel, sourceRow) + val clause = statement.whenClauses.firstOrNull { candidate -> + (candidate is MergeMatchedUpdate || candidate is MergeMatchedDelete) && + (candidate.condition == null || isTrue(Evaluator(resolver, subqueries).evaluate(candidate.condition!!))) + } ?: continue + if (!affectedRids.add(targetRid)) { + throw DiskExecutorException("MERGE command cannot affect a target row a second time") + } + when (clause) { + is MergeMatchedUpdate -> { + val updatedRow = applyMergeUpdate(schema, table, txn, targetRid, targetRow, clause, resolver, subqueries) + returnedRows.add(updatedRow to "UPDATE") + updated++ + } + is MergeMatchedDelete -> { + val deletedRow = applyMergeDelete(schema, table, txn, targetRid, targetRow) + returnedRows.add(deletedRow to "DELETE") + deleted++ + } + else -> error("unreachable: filtered to MATCHED clauses above") + } + } + continue + } + val resolver = notMatchedResolver(sourceSchema, sourceLabel, sourceRow, targetLabel) + val clause = statement.whenClauses.firstOrNull { candidate -> + candidate is MergeNotMatchedInsert && + (candidate.condition == null || isTrue(Evaluator(resolver, subqueries).evaluate(candidate.condition!!))) + } as? MergeNotMatchedInsert ?: continue + val insertedRow = applyMergeInsert(schema, table, txn, clause, resolver, subqueries) + returnedRows.add(insertedRow to "INSERT") + inserted++ + } + val returning = statement.returning?.let { evaluateMergeReturning(schema, returnedRows, it, subqueries) } + return MergeResult(inserted, updated, deleted, returning) + } + + /** Ported 1:1 from [kestrel.engine.Executor.evaluateMergeReturning] - see that method's own doc. */ + private fun evaluateMergeReturning( + schema: TableSchema, + actionedRows: List>, + projection: Projection, + subqueries: SubqueryRunner + ): SelectResult { + val columns = when (projection) { + is StarProjection -> schema.columns.map { it.name } + is ExpressionProjection -> projection.items.mapIndexed { index, item -> + checkReturningColumns(item.expression, schema) + columnLabel(item, index) + } + } + val resultRows = actionedRows.map { (row, action) -> + when (projection) { + is StarProjection -> row.values.toList() + is ExpressionProjection -> { + val evaluator = Evaluator(rowResolver(schema, row), subqueries, mergeAction = action) + projection.items.map { evaluator.evaluate(it.expression) } + } + } + } + return SelectResult(columns, resultRows) + } + + private fun applyMergeUpdate( + schema: TableSchema, + table: DiskTable, + txn: Long, + targetRid: RecordId, + targetRow: Row, + clause: MergeMatchedUpdate, + resolver: ColumnResolver, + subqueries: SubqueryRunner + ): Row { + val assignmentEvaluator = Evaluator(resolver, subqueries, this) + val values = targetRow.values.toMutableList() + for (assignment in clause.assignments) { + val index = schema.columnIndex(assignment.columnName) + ?: throw DiskExecutorException("no such column: ${assignment.columnName}") + val column = schema.columns[index] + rejectGeneratedAssignment(column, explicitValueGiven = assignment.value != null) + values[index] = if (assignment.value != null) { + coerceForColumn(assignmentEvaluator.evaluate(assignment.value), column) + } else { + column.default?.let { coerceForColumn(assignmentEvaluator.evaluate(it), column) } ?: Value.NullValue + } + } + applyGeneratedColumns(schema, values, subqueries) + val updatedRow = Row(values) + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Update, listOf(targetRow to updatedRow), txn) + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, updatedRow.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, updatedRow, subqueries) + for (group in schema.enforcedUniqueKeyGroups()) { + val columnIndices = group.map { schema.columnIndex(it)!! } + val oldKey = columnIndices.map { targetRow.values[it] } + val newKey = columnIndices.map { updatedRow.values[it] } + DiskForeignKeyEnforcer.cascadeUpdate(catalog, txn, schema.name, group, listOf(oldKey to newKey)) + } + table.update(txn, targetRid, updatedRow) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Update, listOf(targetRow to updatedRow), txn) + return updatedRow + } + + /** Returns [targetRow] itself - the pre-deletion image, matching [applyMergeDelete]'s in-memory counterpart. */ + private fun applyMergeDelete(schema: TableSchema, table: DiskTable, txn: Long, targetRid: RecordId, targetRow: Row): Row { + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Delete, listOf(targetRow to null), txn) + DiskForeignKeyEnforcer.cascadeDelete(catalog, txn, schema.name, listOf(targetRow)) + table.delete(txn, targetRid) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Delete, listOf(targetRow to null), txn) + return targetRow + } + + private fun applyMergeInsert( + schema: TableSchema, + table: DiskTable, + txn: Long, + clause: MergeNotMatchedInsert, + resolver: ColumnResolver, + subqueries: SubqueryRunner + ): Row { + val targetIndices = resolveInsertTargets(schema, clause.columns) + val evaluator = Evaluator(resolver, subqueries, this) + val values = MutableList(schema.columns.size) { Value.NullValue } + val touched = BooleanArray(schema.columns.size) + for ((position, expression) in clause.values.withIndex()) { + val columnIndex = targetIndices[position] + if (expression != null) { + rejectGeneratedAssignment(schema.columns[columnIndex], explicitValueGiven = true) + values[columnIndex] = coerceForColumn(evaluator.evaluate(expression), schema.columns[columnIndex]) + touched[columnIndex] = true + } + } + for ((index, column) in schema.columns.withIndex()) { + val default = column.default + if (!touched[index] && default != null) { + values[index] = coerceForColumn(evaluator.evaluate(default), column) + } + } + applyGeneratedColumns(schema, values, subqueries) + val row = Row(values) + fireRowTriggers(schema, TriggerTiming.Before, TriggerEvent.Insert, listOf(null to row), txn) + DiskForeignKeyEnforcer.validateRow(catalog, txn, schema, row.values, uncommittedSiblings = emptyList()) + enforceChecks(schema, row, subqueries) + table.insert(txn, row) + fireRowTriggers(schema, TriggerTiming.After, TriggerEvent.Insert, listOf(null to row), txn) + return row + } + + /** A snapshot (schema, rows) for a MERGE `USING` source - ported 1:1 from [kestrel.engine.SelectRunner.resolveMergeSource], just via [resolveFromItem] rather than a separate in-memory `resolveSource`. */ + private fun resolveMergeSource(item: FromItem, txn: Long): Pair> { + val source = resolveFromItem(item, txn) + return source.schema to source.scan(txn).map { Row(it) } + } + + /** + * Resolves a MERGE `ON`/`WHEN MATCHED` column reference against whichever of [targetRow]/ + * [sourceRow] its qualifier names ([targetLabel]/[sourceLabel] - alias if given, bare name + * otherwise); an unqualified reference must exist on exactly one side. Ported 1:1 from + * [kestrel.engine.Executor.mergeResolver]. + */ + private fun mergeResolver( + targetSchema: TableSchema, + targetLabel: String, + targetRow: Row, + sourceSchema: TableSchema, + sourceLabel: String, + sourceRow: Row + ): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + if (qualifier != null) { + return when { + qualifier.equals(sourceLabel, ignoreCase = true) -> + sourceRow.values[sourceSchema.columnIndex(expression.name) ?: throw DiskExecutorException("no such column: ${expression.name}")] + qualifier.equals(targetLabel, ignoreCase = true) -> + targetRow.values[targetSchema.columnIndex(expression.name) ?: throw DiskExecutorException("no such column: ${expression.name}")] + else -> throw DiskExecutorException("no such table or alias: $qualifier") + } + } + val targetIndex = targetSchema.columnIndex(expression.name) + val sourceIndex = sourceSchema.columnIndex(expression.name) + if (targetIndex != null && sourceIndex != null) { + throw DiskExecutorException("column reference '${expression.name}' is ambiguous between $targetLabel and $sourceLabel") + } + targetIndex?.let { return targetRow.values[it] } + sourceIndex?.let { return sourceRow.values[it] } + throw DiskExecutorException("no such column: ${expression.name}") + } + + override fun declaredCollation(expression: ColumnExpression): Collation? { + val qualifier = expression.qualifier + if (qualifier != null) { + return when { + qualifier.equals(sourceLabel, ignoreCase = true) -> sourceSchema.column(expression.name)?.collation + qualifier.equals(targetLabel, ignoreCase = true) -> targetSchema.column(expression.name)?.collation + else -> null + } + } + val targetColumn = targetSchema.column(expression.name) + val sourceColumn = sourceSchema.column(expression.name) + return if (targetColumn != null && sourceColumn == null) targetColumn.collation + else if (sourceColumn != null && targetColumn == null) sourceColumn.collation + else null + } + } + + /** A `WHEN NOT MATCHED` clause has no target row to read - a qualifier naming the target throws rather than silently resolving to nothing. Ported 1:1 from [kestrel.engine.Executor.notMatchedResolver]. */ + private fun notMatchedResolver( + sourceSchema: TableSchema, + sourceLabel: String, + sourceRow: Row, + targetLabel: String + ): ColumnResolver = object : ColumnResolver { + override fun resolve(expression: ColumnExpression): Value { + val qualifier = expression.qualifier + if (qualifier != null && qualifier.equals(targetLabel, ignoreCase = true)) { + throw DiskExecutorException("no target row is available in a WHEN NOT MATCHED clause: $targetLabel.${expression.name}") + } + if (qualifier != null && !qualifier.equals(sourceLabel, ignoreCase = true)) { + throw DiskExecutorException("no such table or alias: $qualifier") + } + val index = sourceSchema.columnIndex(expression.name) ?: throw DiskExecutorException("no such column: ${expression.name}") + return sourceRow.values[index] + } + + override fun declaredCollation(expression: ColumnExpression): Collation? = sourceSchema.column(expression.name)?.collation + } + + /** Upfront validation for every clause of a MERGE statement, run even when the source produces zero rows. Ported 1:1 from [kestrel.engine.Executor.checkMergeColumns]. */ + private fun checkMergeColumns( + statement: MergeStatement, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String + ) { + checkMergeExpressionColumns(statement.onCondition, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) + for (clause in statement.whenClauses) { + when (clause) { + is MergeMatchedUpdate -> { + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + for (assignment in clause.assignments) { + if (targetSchema.columnIndex(assignment.columnName) == null) { + throw DiskExecutorException("no such column: ${assignment.columnName}") + } + assignment.value?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + } + } + is MergeMatchedDelete -> + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = true) } + is MergeNotMatchedInsert -> { + clause.condition?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = false) } + val targetIndices = resolveInsertTargets(targetSchema, clause.columns) + if (clause.values.size != targetIndices.size) { + throw DiskExecutorException( + "table ${targetSchema.name} expects ${targetIndices.size} values but ${clause.values.size} were supplied" + ) + } + clause.values.forEach { value -> + value?.let { checkMergeExpressionColumns(it, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget = false) } + } + } + } + } + } + + /** Fifth near-parallel recursive [Expression] walker in this file (alongside [requireNoSubquery]/[checkColumns]/[checkReturningColumns]/[validateTriggerRowReferences]) - same deliberate non-abstraction as [kestrel.engine.Executor.checkMergeExpressionColumns], since this one needs two relations plus ambiguity detection none of the others do. */ + private fun checkMergeExpressionColumns( + expression: Expression, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String, + allowTarget: Boolean + ) { + fun recurse(inner: Expression) = + checkMergeExpressionColumns(inner, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget) + when (expression) { + is ColumnExpression -> checkMergeColumnReference(expression, targetSchema, targetLabel, sourceSchema, sourceLabel, allowTarget) + is BinaryExpression -> { recurse(expression.left); recurse(expression.right) } + is UnaryExpression -> recurse(expression.operand) + is FunctionExpression -> throw DiskExecutorException("aggregate functions are not allowed here") + is WindowExpression -> throw DiskExecutorException("window functions are not allowed here") + is LiteralExpression -> Unit + is CallExpression -> expression.arguments.forEach { recurse(it) } + is IsNullExpression -> recurse(expression.operand) + is InExpression -> { + recurse(expression.operand) + expression.values.forEach { recurse(it) } + } + is BetweenExpression -> { + recurse(expression.operand) + recurse(expression.low) + recurse(expression.high) + } + is LikeExpression -> { + recurse(expression.operand) + recurse(expression.pattern) + } + is CaseExpression -> { + expression.operand?.let { recurse(it) } + expression.branches.forEach { recurse(it.condition); recurse(it.result) } + expression.elseExpression?.let { recurse(it) } + } + is ScalarSubqueryExpression -> Unit + is ExistsExpression -> Unit + is InSubqueryExpression -> recurse(expression.operand) + is CastExpression -> recurse(expression.operand) + is CollateExpression -> recurse(expression.operand) + // Structurally harmless here too - see checkReturningColumns's own MERGE_ACTION() branch. + is NextValueExpression, is CurrentValueExpression, is MergeActionExpression, is ParameterExpression -> Unit + } + } + + private fun checkMergeColumnReference( + expression: ColumnExpression, + targetSchema: TableSchema, + targetLabel: String, + sourceSchema: TableSchema, + sourceLabel: String, + allowTarget: Boolean + ) { + val qualifier = expression.qualifier + if (qualifier != null) { + val matchesSource = qualifier.equals(sourceLabel, ignoreCase = true) + val matchesTargetLabel = qualifier.equals(targetLabel, ignoreCase = true) + if (matchesTargetLabel && !allowTarget) { + throw DiskExecutorException("no target row is available in a WHEN NOT MATCHED clause: $targetLabel.${expression.name}") + } + if (!matchesSource && !matchesTargetLabel) { + throw DiskExecutorException("no such table or alias: $qualifier") + } + val schema = if (matchesSource) sourceSchema else targetSchema + if (schema.columnIndex(expression.name) == null) throw DiskExecutorException("no such column: ${expression.name}") + return + } + val inTarget = allowTarget && targetSchema.columnIndex(expression.name) != null + val inSource = sourceSchema.columnIndex(expression.name) != null + if (inTarget && inSource) { + throw DiskExecutorException("column reference '${expression.name}' is ambiguous between $targetLabel and $sourceLabel") + } + if (!inTarget && !inSource) throw DiskExecutorException("no such column: ${expression.name}") + } + + /** Commits [txnId] if [block] returns normally, aborts it (then rethrows) if [block] throws. */ + private inline fun runInTransaction(txnId: Long, block: () -> T): T { + val result = try { + block() + } catch (failure: Exception) { + catalog.abort(txnId) + throw failure + } + catalog.commit(txnId) + return result + } + + private fun isTrue(value: Value): Boolean = value is Value.BooleanValue && value.value + + private fun coerceForColumn(value: Value, column: Column): Value { + if (value is Value.NullValue) return value + return when (column.type) { + ColumnType.Integer -> value as? Value.IntegerValue ?: throw typeMismatch(column, value) + ColumnType.Text -> value as? Value.TextValue ?: throw typeMismatch(column, value) + ColumnType.Boolean -> value as? Value.BooleanValue ?: throw typeMismatch(column, value) + ColumnType.Real -> when (value) { + is Value.RealValue -> value + is Value.IntegerValue -> Value.RealValue(value.value.toDouble()) + else -> throw typeMismatch(column, value) + } + ColumnType.Decimal -> { + val raw = when (value) { + is Value.DecimalValue -> value.value + is Value.IntegerValue -> BigDecimal(value.value) + is Value.RealValue -> BigDecimal.valueOf(value.value) + else -> throw typeMismatch(column, value) + } + Value.DecimalValue(DecimalColumns.normalize(raw, column.precision, column.scale, "column ${column.name}")) + } + ColumnType.Date -> value as? Value.DateValue ?: throw typeMismatch(column, value) + } + } + + private fun typeMismatch(column: Column, value: Value): DiskExecutorException = + DiskExecutorException("type mismatch: column ${column.name} expects ${column.type.displayName} but got ${value.typeName}") + + companion object { + private const val MAX_RECURSION_ITERATIONS = 10_000 + private const val MAX_TRIGGER_DEPTH = 100 + + /** Kinds whose result actually depends on which rows fall inside the frame. */ + private val FRAME_CAPABLE_KINDS = setOf(WindowFunctionKind.Aggregate, WindowFunctionKind.FirstValue, WindowFunctionKind.LastValue) + } +} diff --git a/src/main/kotlin/kestrel/store/sql/DiskForeignKeyEnforcer.kt b/src/main/kotlin/kestrel/store/sql/DiskForeignKeyEnforcer.kt new file mode 100644 index 0000000..4b1224e --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DiskForeignKeyEnforcer.kt @@ -0,0 +1,201 @@ +package kestrel.store.sql + +import kestrel.catalog.ForeignKeyAction +import kestrel.catalog.TableSchema +import kestrel.store.heap.RecordId +import kestrel.storage.Row +import kestrel.storage.Value + +/** + * Disk-backed counterpart of [kestrel.engine.ForeignKeyEnforcer]: existence checks on INSERT/UPDATE + * of the referencing (child) table, and ON DELETE/ON UPDATE actions on the referenced (parent) + * table, operating on [DiskCatalog]/[DiskTable] within an active disk transaction instead of the + * in-memory engine's `Table.rows` ArrayList. Deliberately duplicated rather than shared - same + * reasoning as every other validation [DiskExecutor] duplicates (see + * [DiskExecutor.validateUniqueDefinitions]). + * + * [validateRow]'s existence check is a full [DiskTable.scan] of the referenced table, exactly + * mirroring the in-memory engine's own full scan of `Table.rows` - a documented performance + * compromise (not correctness), same category as CTE materialization or the disk executor's UNIQUE + * constraint scan: a real index-driven point lookup here would need to first prove the referenced + * columns line up with the PRIMARY KEY or a secondary index before choosing a plan, which is scope + * this pass does not take on. + */ +internal object DiskForeignKeyEnforcer { + + /** + * [uncommittedSiblings] covers a self-referencing FK (`referencedTable == schema.name`, e.g. an + * employee/manager hierarchy) inside a multi-row INSERT/UPDATE statement: those sibling rows are + * being validated as part of the very same statement and have not been written to [txn] yet, so + * without this a later row could not reference an earlier row from the same batch (or vice + * versa - order within the batch must not matter). + * + * Follows MATCH SIMPLE (the SQL default): if *any* column of the FK is NULL in this row, the + * whole constraint is skipped for that row - not just the null column. + */ + fun validateRow( + catalog: DiskCatalog, + txn: Long, + schema: TableSchema, + values: List, + uncommittedSiblings: List = emptyList() + ) { + for (fk in schema.foreignKeys) { + val columnIndices = fk.columns.map { schema.columnIndex(it)!! } + val key = columnIndices.map { values[it] } + if (key.any { it is Value.NullValue }) continue + val referenced = catalog.tableOrNull(fk.referencedTable) + ?: throw DiskExecutorException( + "FOREIGN KEY constraint failed: table '${fk.referencedTable}' referenced by " + + "${schema.name}(${fk.columns.joinToString(", ")}) does not exist" + ) + val refIndices = fk.referencedColumns.map { referenced.schema.columnIndex(it)!! } + val siblings = if (fk.referencedTable.equals(schema.name, ignoreCase = true)) uncommittedSiblings else emptyList() + val exists = referenced.scan(txn).any { (_, row) -> refIndices.map { idx -> row.values[idx] } == key } || + siblings.any { row -> refIndices.map { idx -> row.values[idx] } == key } + if (!exists) { + throw DiskExecutorException( + "FOREIGN KEY constraint failed: ${schema.name}(${fk.columns.joinToString(", ")}) has no " + + "matching row in ${fk.referencedTable}(${fk.referencedColumns.joinToString(", ")})" + ) + } + } + } + + private sealed interface CascadeStep { + data class Delete(val table: DiskTable, val rows: List>, val next: List) : CascadeStep + data class SetToNull(val table: DiskTable, val columnIndices: List, val rows: List>) : CascadeStep + data class SetTo(val table: DiskTable, val columnIndices: List, val rows: List>, val values: List) : CascadeStep + } + + fun cascadeDelete(catalog: DiskCatalog, txn: Long, parentTableName: String, deletedRows: List) { + applySteps(txn, planDelete(catalog, txn, parentTableName, deletedRows)) + } + + /** + * [referencedColumns] is the full column group whose values changed together (a PRIMARY KEY or + * UNIQUE key group) and [changedPairs] holds the old/new key tuple for every row of the parent + * table, positionally aligned with [referencedColumns] - a matching FK may declare its own + * `referencedColumns` in a different order, so [planUpdate] re-projects each tuple by name. + */ + fun cascadeUpdate( + catalog: DiskCatalog, + txn: Long, + parentTableName: String, + referencedColumns: List, + changedPairs: List, List>> + ) { + applySteps(txn, planUpdate(catalog, txn, parentTableName, referencedColumns, changedPairs)) + } + + private fun planDelete(catalog: DiskCatalog, txn: Long, parentTableName: String, deletedRows: List): List { + if (deletedRows.isEmpty()) return emptyList() + val parent = catalog.table(parentTableName) + val steps = mutableListOf() + for (childName in catalog.tableNames) { + val child = catalog.table(childName) + for (fk in child.schema.foreignKeys) { + if (!fk.referencedTable.equals(parentTableName, ignoreCase = true)) continue + val refIndices = fk.referencedColumns.map { parent.schema.columnIndex(it)!! } + val deletedKeys = deletedRows.map { row -> refIndices.map { row.values[it] } }.toHashSet() + val childColumnIndices = fk.columns.map { child.schema.columnIndex(it)!! } + val matches = child.scan(txn).filter { (_, row) -> childColumnIndices.map { row.values[it] } in deletedKeys } + if (matches.isEmpty()) continue + when (fk.onDelete) { + ForeignKeyAction.Cascade -> + steps.add(CascadeStep.Delete(child, matches, planDelete(catalog, txn, child.schema.name, matches.map { it.second }))) + ForeignKeyAction.SetNull -> { + requireNullable(child.schema, childColumnIndices, "ON DELETE SET NULL") + steps.add(CascadeStep.SetToNull(child, childColumnIndices, matches)) + } + ForeignKeyAction.Restrict, ForeignKeyAction.NoAction -> throw DiskExecutorException( + "FOREIGN KEY constraint failed: ${matches.size} row(s) in ${child.schema.name} still " + + "reference $parentTableName (use ON DELETE CASCADE or remove them first)" + ) + } + } + } + return steps + } + + private fun planUpdate( + catalog: DiskCatalog, + txn: Long, + parentTableName: String, + referencedColumns: List, + changedPairs: List, List>> + ): List { + val changes = changedPairs.filter { (old, new) -> old != new } + if (changes.isEmpty()) return emptyList() + val steps = mutableListOf() + for (childName in catalog.tableNames) { + val child = catalog.table(childName) + for (fk in child.schema.foreignKeys) { + if (!fk.referencedTable.equals(parentTableName, ignoreCase = true)) continue + if (fk.referencedColumns.size != referencedColumns.size) continue + // Re-project the tuple by name: this FK's referencedColumns may list the same key + // group in a different order than the one changedPairs was built from. + val positions = fk.referencedColumns.map { refCol -> + referencedColumns.indexOfFirst { it.equals(refCol, ignoreCase = true) } + } + if (positions.any { it == -1 }) continue + val childColumnIndices = fk.columns.map { child.schema.columnIndex(it)!! } + for ((oldTuple, newTuple) in changes) { + val oldKey = positions.map { oldTuple[it] } + val newKey = positions.map { newTuple[it] } + if (oldKey == newKey) continue + val matches = child.scan(txn).filter { (_, row) -> childColumnIndices.map { row.values[it] } == oldKey } + if (matches.isEmpty()) continue + when (fk.onUpdate) { + ForeignKeyAction.Cascade -> + steps.add(CascadeStep.SetTo(child, childColumnIndices, matches, newKey)) + ForeignKeyAction.SetNull -> { + requireNullable(child.schema, childColumnIndices, "ON UPDATE SET NULL") + steps.add(CascadeStep.SetToNull(child, childColumnIndices, matches)) + } + ForeignKeyAction.Restrict, ForeignKeyAction.NoAction -> throw DiskExecutorException( + "FOREIGN KEY constraint failed: ${matches.size} row(s) in ${child.schema.name} still " + + "reference $parentTableName (use ON UPDATE CASCADE)" + ) + } + } + } + } + return steps + } + + private fun applySteps(txn: Long, steps: List) { + for (step in steps) { + when (step) { + is CascadeStep.Delete -> { + for ((rid, _) in step.rows) step.table.delete(txn, rid) + applySteps(txn, step.next) + } + is CascadeStep.SetToNull -> { + val nulls = step.columnIndices.map { Value.NullValue } + for ((rid, row) in step.rows) step.table.update(txn, rid, withColumns(row, step.columnIndices, nulls)) + } + is CascadeStep.SetTo -> { + for ((rid, row) in step.rows) step.table.update(txn, rid, withColumns(row, step.columnIndices, step.values)) + } + } + } + } + + private fun requireNullable(schema: TableSchema, columnIndices: List, action: String) { + for (columnIndex in columnIndices) { + val column = schema.columns[columnIndex] + if (column.notNull) { + throw DiskExecutorException( + "FOREIGN KEY constraint failed: cannot $action on NOT NULL column ${schema.name}.${column.name}" + ) + } + } + } + + private fun withColumns(row: Row, columnIndices: List, values: List): Row { + val result = row.values.toMutableList() + columnIndices.forEachIndexed { position, columnIndex -> result[columnIndex] = values[position] } + return Row(result) + } +} diff --git a/src/main/kotlin/kestrel/store/sql/DiskIndexAccessPlanner.kt b/src/main/kotlin/kestrel/store/sql/DiskIndexAccessPlanner.kt new file mode 100644 index 0000000..02d2c38 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DiskIndexAccessPlanner.kt @@ -0,0 +1,238 @@ +package kestrel.store.sql + +import kestrel.catalog.TableSchema +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.LiteralExpression +import kestrel.storage.Value + +/** + * How [DiskExecutor] should fetch candidate rows for a WHERE predicate on a + * [DiskTable]: a primary-key point/range lookup, a non-unique secondary-index + * lookup, or (when nothing in the predicate is indexed) a full heap scan. + * Mirrors [kestrel.engine.IndexAccessPlanner]'s role for the in-memory + * engine, but returns disk-backed lookup parameters instead of row indices, + * since [DiskTable] exposes [DiskTable.findByPrimaryKey]/[DiskTable.rangeByPrimaryKey]/ + * [DiskTable.findBySecondary] directly rather than an in-memory [kestrel.catalog.SortedIndex]. + * + * Whichever plan is chosen only NARROWS the candidate set - [DiskExecutor] + * still evaluates the full WHERE predicate against every candidate + * afterwards, exactly like the in-memory engine's `primaryRows()` + + * `combined.filter { ... }` split. This keeps the planner safe to extend + * (it never has to prove it captured every conjunct) and correct even for a + * predicate with extra conjuncts beyond the indexed column, e.g. + * `WHERE id = 5 AND status = 'active'`. + */ +sealed class DiskAccessPlan { + data object FullScan : DiskAccessPlan() + class PrimaryKeyPoint(val key: Value) : DiskAccessPlan() + class PrimaryKeyRange( + val lower: Value?, + val lowerInclusive: Boolean, + val upper: Value?, + val upperInclusive: Boolean + ) : DiskAccessPlan() + /** + * Equality on every column of a table-level composite PRIMARY KEY, e.g. + * `WHERE order_id = 1 AND line_no = 2` against `PRIMARY KEY (order_id, line_no)`. [keys] is + * ordered to match [kestrel.catalog.TableSchema.primaryKeyColumns], the order [DiskTable]'s + * composite key encoding expects. A partial match (equality on only some of the composite + * PRIMARY KEY's columns) is deliberately not planned as a prefix range scan - that scope cut + * falls back to [Secondary]/[FullScan] like any other unindexable predicate. + */ + class CompositePrimaryKeyPoint(val keys: List) : DiskAccessPlan() + /** + * Range predicate on the LAST column of a table-level composite PRIMARY KEY, after an + * equality prefix on every column before it, e.g. `WHERE order_id = 1 AND line_no > 2` against + * `PRIMARY KEY (order_id, line_no)`. [prefix] holds the equality values bound so far, in + * [kestrel.catalog.TableSchema.primaryKeyColumns] order (one shorter than the full column + * count, since the ranged column itself is excluded). Unlike [kestrel.store.sql.DiskAccessPlan.SecondaryRange], + * which may range on a column with further index columns still unconstrained after it (leaning + * on the secondary entry's own RID suffix to bound the ambiguity), a PRIMARY KEY entry has no + * such suffix - filling in [prefix] plus the range's own bound value must therefore always + * produce a COMPLETE, full-arity key, otherwise an inclusive upper bound could not distinguish + * "this exact prefix, nothing more" from every real row extending past it with further PRIMARY + * KEY columns. Ranging on any column before the last one - with columns after it left + * unconstrained - is deliberately not planned for this reason; that scope cut falls back to + * [Secondary]/[SecondaryRange]/[FullScan] like any other unindexable predicate. + */ + class CompositePrimaryKeyRange( + val prefix: List, + val lower: Value?, + val lowerInclusive: Boolean, + val upper: Value?, + val upperInclusive: Boolean + ) : DiskAccessPlan() + /** + * Equality on every column of a secondary index, single-column or composite - [keys] is + * ordered to match [SecondaryIndexHandle.columnNames]. A partial match (equality on only some + * of a composite index's columns) is deliberately not planned as a prefix range scan, the same + * scope cut as [CompositePrimaryKeyPoint] - it falls back to another index or [FullScan]. + */ + class Secondary(val indexName: String, val keys: List) : DiskAccessPlan() + /** + * Range predicate (`<`, `<=`, `>`, `>=`, or both bounds together) on the column right after the + * longest leading equality run of a secondary index, single-column or composite - mirrors + * [kestrel.engine.IndexAccessPlanner]'s combined "prefix + range" logic for the in-memory + * engine's [kestrel.catalog.SortedIndex]-backed indexes. [prefix] holds the equality values + * bound so far, in [SecondaryIndexHandle.columnNames] order - empty for a single-column index, + * or for a composite index whose very first column is the one being ranged (no column can be + * skipped over unbound: an index sorted by its first column first cannot narrow a range on its + * second column without first pinning the first one down). Columns past the ranged one are left + * entirely unconstrained, the same restriction [CompositePrimaryKeyPoint] documents for equality. + */ + class SecondaryRange( + val indexName: String, + val prefix: List, + val lower: Value?, + val lowerInclusive: Boolean, + val upper: Value?, + val upperInclusive: Boolean + ) : DiskAccessPlan() +} + +class DiskIndexAccessPlanner { + + /** + * Picks the most selective single access path for [predicate] over + * [schema]'s primary key and [secondaryIndexes]. Preference order: an + * equality match on the primary key (a single-row point lookup, the most + * selective possible plan), then a range predicate on the primary key, + * then (for a table-level composite primary key) an equality prefix on + * every column but the last plus a range on that last column, then an + * equality match on any secondary index, then a prefix-equality + * (possibly empty) plus a range predicate on the column right after it for any + * secondary index, then [DiskAccessPlan.FullScan] - equality always + * outranks a range on the very same column, since a range visits at least as many + * rows as the point it would otherwise narrow to. + * A composite predicate naming more than one indexable column (e.g. both + * the primary key and an indexed column) always prefers the primary key, + * since a point lookup can never examine more rows than any other plan. + */ + fun plan(schema: TableSchema, secondaryIndexes: List, predicate: Expression?, alias: String? = null): DiskAccessPlan { + if (predicate == null) return DiskAccessPlan.FullScan + val byColumn = LinkedHashMap>>() + for (conjunct in conjuncts(predicate)) { + val simple = asColumnPredicate(conjunct, alias, schema) ?: continue + byColumn.getOrPut(simple.first) { mutableListOf() }.add(simple.second to simple.third) + } + val pkColumnName = schema.primaryKeyIndex?.let { schema.columns[it].name } + if (pkColumnName != null) { + val predicates = byColumn[pkColumnName] + if (predicates != null) { + val equal = predicates.firstOrNull { it.first == BinaryOperator.Equal } + if (equal != null) return DiskAccessPlan.PrimaryKeyPoint(equal.second) + val range = predicates.filter { it.first != BinaryOperator.Equal } + if (range.isNotEmpty()) return rangePlan(range) + } + } + val pkColumns = schema.primaryKeyColumns + if (pkColumns.size > 1) { + val equalities = pkColumns.map { name -> byColumn[name]?.firstOrNull { it.first == BinaryOperator.Equal } } + if (equalities.all { it != null }) return DiskAccessPlan.CompositePrimaryKeyPoint(equalities.map { it!!.second }) + val prefix = mutableListOf() + for (name in pkColumns) { + val equal = byColumn[name]?.firstOrNull { it.first == BinaryOperator.Equal } ?: break + prefix.add(equal.second) + } + if (prefix.size == pkColumns.size - 1) { + val range = byColumn[pkColumns.last()]?.filter { it.first != BinaryOperator.Equal }.orEmpty() + if (range.isNotEmpty()) { + val bounds = rangeBounds(range) + return DiskAccessPlan.CompositePrimaryKeyRange(prefix.toList(), bounds.lower, bounds.lowerInclusive, bounds.upper, bounds.upperInclusive) + } + } + } + for (handle in secondaryIndexes) { + val equalities = handle.columnNames.map { name -> byColumn[name]?.firstOrNull { it.first == BinaryOperator.Equal } } + if (equalities.all { it != null }) return DiskAccessPlan.Secondary(handle.name, equalities.map { it!!.second }) + } + for (handle in secondaryIndexes) { + val prefix = mutableListOf() + for (columnName in handle.columnNames) { + val equal = byColumn[columnName]?.firstOrNull { it.first == BinaryOperator.Equal } ?: break + prefix.add(equal.second) + } + if (prefix.size == handle.columnNames.size) continue // full equality already tried above + val nextColumn = handle.columnNames[prefix.size] + val range = byColumn[nextColumn]?.filter { it.first != BinaryOperator.Equal }.orEmpty() + if (range.isEmpty()) continue + val bounds = rangeBounds(range) + return DiskAccessPlan.SecondaryRange(handle.name, prefix.toList(), bounds.lower, bounds.lowerInclusive, bounds.upper, bounds.upperInclusive) + } + return DiskAccessPlan.FullScan + } + + private class RangeBounds(val lower: Value?, val lowerInclusive: Boolean, val upper: Value?, val upperInclusive: Boolean) + + private fun rangeBounds(predicates: List>): RangeBounds { + var lowerValue: Value? = null + var lowerInclusive = false + var upperValue: Value? = null + var upperInclusive = false + for ((operator, literal) in predicates) { + when (operator) { + BinaryOperator.Greater -> { lowerValue = literal; lowerInclusive = false } + BinaryOperator.GreaterEqual -> { lowerValue = literal; lowerInclusive = true } + BinaryOperator.Less -> { upperValue = literal; upperInclusive = false } + BinaryOperator.LessEqual -> { upperValue = literal; upperInclusive = true } + else -> Unit + } + } + return RangeBounds(lowerValue, lowerInclusive, upperValue, upperInclusive) + } + + private fun rangePlan(predicates: List>): DiskAccessPlan { + val bounds = rangeBounds(predicates) + return DiskAccessPlan.PrimaryKeyRange(bounds.lower, bounds.lowerInclusive, bounds.upper, bounds.upperInclusive) + } + + private fun asColumnPredicate( + expression: Expression, + alias: String?, + schema: TableSchema + ): Triple? { + if (expression !is BinaryExpression) return null + val operator = expression.operator + if (operator !in RANGE_OPERATORS) return null + val left = expression.left + val right = expression.right + fun matches(column: ColumnExpression): String? { + if (column.qualifier != null && alias != null && !column.qualifier.equals(alias, ignoreCase = true)) return null + return if (schema.columnIndex(column.name) != null) column.name else null + } + if (left is ColumnExpression && right is LiteralExpression && right.value !is Value.NullValue) { + val name = matches(left) ?: return null + return Triple(name, operator, right.value) + } + if (right is ColumnExpression && left is LiteralExpression && left.value !is Value.NullValue) { + val name = matches(right) ?: return null + return Triple(name, flip(operator), left.value) + } + return null + } + + private fun conjuncts(expression: Expression): List = + if (expression is BinaryExpression && expression.operator == BinaryOperator.And) { + conjuncts(expression.left) + conjuncts(expression.right) + } else { + listOf(expression) + } + + private fun flip(operator: BinaryOperator): BinaryOperator = when (operator) { + BinaryOperator.Less -> BinaryOperator.Greater + BinaryOperator.LessEqual -> BinaryOperator.GreaterEqual + BinaryOperator.Greater -> BinaryOperator.Less + BinaryOperator.GreaterEqual -> BinaryOperator.LessEqual + else -> operator + } + + private companion object { + val RANGE_OPERATORS = setOf( + BinaryOperator.Equal, BinaryOperator.Less, BinaryOperator.LessEqual, + BinaryOperator.Greater, BinaryOperator.GreaterEqual + ) + } +} diff --git a/src/main/kotlin/kestrel/store/sql/DiskTable.kt b/src/main/kotlin/kestrel/store/sql/DiskTable.kt new file mode 100644 index 0000000..852ce7a --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/DiskTable.kt @@ -0,0 +1,511 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.catalog.TableSchema +import kestrel.engine.collationFoldedUniquenessKey +import kestrel.store.codec.ByteWriter +import kestrel.store.codec.KeyCodec +import kestrel.store.codec.TupleCodec +import kestrel.store.engine.TransactionalHeap +import kestrel.store.engine.TransactionalIndex +import kestrel.store.heap.RecordId +import kestrel.storage.Row +import kestrel.storage.Value + +class DiskTableException(message: String) : KestrelException(message) + +/** + * A live secondary index attached to a [DiskTable]: the B+Tree plus which column(s) it is keyed + * on - [columnNames]/[columnIndices] have more than one entry for a composite (multi-column) + * index, in the order the index was declared. [unique] mirrors a `CREATE UNIQUE INDEX` - see + * [DiskTable.rejectDuplicateUniqueSecondary]. + */ +class SecondaryIndexHandle( + val name: String, + val columnNames: List, + val columnIndices: List, + val index: TransactionalIndex, + val unique: Boolean = false +) + +/** + * The disk-resident counterpart of [kestrel.storage.Table]: a schema-aware + * row store backed by a [TransactionalHeap] for the row bodies, an optional + * unique [TransactionalIndex] on the primary key, and any number of + * non-unique secondary [TransactionalIndex]es - all driven by the same + * [kestrel.storage.Value]/[TableSchema] types the in-memory SQL engine uses, + * so this is the storage backend a future Catalog/Executor integration would + * sit on top of, not a parallel data model needing translation. + * + * [kestrel.store.btree.BPlusTree] only ever stores one value per key, so a + * non-unique secondary index cannot key its B+Tree entries by the indexed + * value(s) alone: two rows with the same value(s) would silently overwrite + * each other. Every secondary index instead keys on `encodeKey(values) ++ rid` + * (one or more columns, via [kestrel.store.codec.KeyCodec.encodeKey] - the + * same order-preserving multi-value encoding a composite PRIMARY KEY uses), a + * composite that is unique per row (see [compositeKey]); an equality lookup + * becomes a range scan bounded by that encoding's smallest and largest possible + * suffix (see [equalRangeFor]). This is the standard way an index built on a + * unique-key structure supports duplicates (e.g. how InnoDB appends the + * primary key to secondary index entries). + * + * Every mutating operation may change a primary or secondary index's root + * page (a B+Tree split or merge) purely in the caller process's memory - see + * [kestrel.store.engine.TransactionalIndex]'s own documentation. [onRootsChanged] + * is how a [DiskTable] tells its owning [DiskCatalog] to persist that new + * root into the directory row for this table, in the same transaction as the + * data write, so a fresh process reopening the catalog later sees the + * correct, committed root rather than the one recorded at table-creation time. + */ +class DiskTable internal constructor( + schema: TableSchema, + private val heap: TransactionalHeap, + private var primaryIndex: TransactionalIndex?, + private val secondaryIndexHandles: MutableList, + private val onRootsChanged: (txnId: Long) -> Unit +) { + /** + * Mutable so `ALTER TABLE` (see [replaceSchemaMetadata]/[renameColumnAndIndexes]/[rewriteRows]) + * can swap in a new [TableSchema] after CREATE TABLE - private setter because every mutation + * needs to happen alongside bookkeeping (secondary index column realignment, directory + * persistence) that only this class and [DiskCatalog] (its sole caller for these methods, + * same module) know how to do correctly. + */ + var schema: TableSchema = schema + private set + + val headerPageId: Long get() = heap.headerPageId + val secondaryIndexes: List get() = secondaryIndexHandles.toList() + + /** + * Access counters, incremented by [scan]/[findByPrimaryKey]/[rangeByPrimaryKey]/ + * [findBySecondary]. Exists so a caller (chiefly [DiskIndexAccessPlanner]-driven + * code and its tests) can confirm an index lookup actually happened instead of + * a full heap scan - correctness alone cannot distinguish the two, since both + * return the same rows. + */ + var scanCount: Int = 0; private set + var pointLookupCount: Int = 0; private set + var rangeScanCount: Int = 0; private set + var secondaryLookupCount: Int = 0; private set + + internal fun attachSecondaryIndex(handle: SecondaryIndexHandle) { + secondaryIndexHandles.add(handle) + } + + internal fun dropSecondaryIndexByName(name: String, txnId: Long) { + val handle = secondaryIndexHandles.firstOrNull { it.name.equals(name, ignoreCase = true) } ?: return + secondaryIndexHandles.remove(handle) + handle.index.destroy(txnId) + } + + /** + * Frees this table's own storage - its heap and every attached index (primary and secondary) - + * back to the shared page free list. Called by `DROP TABLE` once the directory row itself is + * already gone, since nothing will ever reopen this [DiskTable] again. + */ + internal fun destroyStorage(txnId: Long) { + for (handle in secondaryIndexHandles) handle.index.destroy(txnId) + primaryIndex?.destroy(txnId) + heap.destroy(txnId) + } + + /** + * Metadata-only schema swap used by `ALTER TABLE ... RENAME TO` (this table's own name/ + * self-referencing FK) and, on an unrelated table, by `RENAME TO`/`RENAME COLUMN`/`DROP COLUMN` + * rewriting a FOREIGN KEY that points at the table being altered - none of these change this + * table's own column count or positions, so no row rewrite or index realignment is needed. + */ + internal fun replaceSchemaMetadata(newSchema: TableSchema) { + schema = newSchema + } + + /** + * `ALTER TABLE ... RENAME COLUMN`: swaps in [newSchema] (column list already renamed by the + * caller) and renames [oldName] to [newName] wherever it appears in a secondary index's + * [SecondaryIndexHandle.columnNames] - the index's [SecondaryIndexHandle.columnIndices] stay + * exactly the same, since a pure rename never moves a column's position. + */ + internal fun renameColumnAndIndexes(newSchema: TableSchema, oldName: String, newName: String) { + schema = newSchema + for (i in secondaryIndexHandles.indices) { + val handle = secondaryIndexHandles[i] + if (handle.columnNames.none { it.equals(oldName, ignoreCase = true) }) continue + val renamedNames = handle.columnNames.map { if (it.equals(oldName, ignoreCase = true)) newName else it } + secondaryIndexHandles[i] = SecondaryIndexHandle(handle.name, renamedNames, handle.columnIndices, handle.index, handle.unique) + } + } + + /** + * `ALTER TABLE ... ADD/DROP COLUMN`: rewrites every row's stored bytes through [transform] and + * swaps in [newSchema] (already reflecting the added/removed column). Every surviving secondary + * index's [SecondaryIndexHandle.columnIndices] is recomputed fresh from [newSchema] right after + * the swap (mirrors [kestrel.catalog.Catalog.reindexOne] always recomputing positions from + * names rather than trusting a cached one) - required for DROP COLUMN, where every column after + * the dropped one shifts down by one position; a no-op for ADD COLUMN, which only ever appends + * at the end. Reuses the ordinary [update] for each row (not a hand-rolled unindex/reindex pair) + * so a row relocated to a new [RecordId] by a growing/shrinking body - see + * [kestrel.store.heap.HeapFile.update] - keeps every index entry correctly pointed at it, + * exactly like a normal SQL `UPDATE` already guarantees. The caller is responsible for having + * already dropped (via [dropSecondaryIndexByName]) any index that depends on a column being + * removed - a survivor's columns are always still present in [newSchema] by the time this runs. + */ + internal fun rewriteRows(txnId: Long, newSchema: TableSchema, transform: (Row) -> Row) { + val existing = rawScan(txnId) + schema = newSchema + for (i in secondaryIndexHandles.indices) { + val handle = secondaryIndexHandles[i] + val realignedIndices = handle.columnNames.map { + newSchema.columnIndex(it) ?: throw DiskTableException("index ${handle.name} references unknown column $it") + } + secondaryIndexHandles[i] = SecondaryIndexHandle(handle.name, handle.columnNames, realignedIndices, handle.index, handle.unique) + } + for ((rid, row) in existing) update(txnId, rid, transform(row)) + } + + fun insert(txnId: Long, row: Row): RecordId { + requireArity(row) + val pkValues = primaryKeyValuesOf(row) + rejectDuplicatePrimaryKey(txnId, pkValues) + rejectDuplicateUniqueSecondary(txnId, row, excludeRid = null) + rejectDuplicateUniqueConstraints(txnId, row, excludeRid = null) + val roots = currentRoots() + val rid = heap.insert(txnId, TupleCodec.encode(row.values)) + indexRow(txnId, rid, row, pkValues) + persistIfRootsChanged(txnId, roots) + return rid + } + + fun get(txnId: Long, rid: RecordId): Row = Row(TupleCodec.decode(heap.get(txnId, rid))) + + fun scan(txnId: Long): List> { + scanCount++ + return rawScan(txnId) + } + + private fun rawScan(txnId: Long): List> = + heap.scan(txnId).map { (rid, bytes) -> rid to Row(TupleCodec.decode(bytes)) } + + fun delete(txnId: Long, rid: RecordId) { + val row = get(txnId, rid) + val roots = currentRoots() + unindexRow(txnId, rid, row) + heap.delete(txnId, rid) + persistIfRootsChanged(txnId, roots) + } + + fun update(txnId: Long, rid: RecordId, newRow: Row): RecordId { + requireArity(newRow) + val oldRow = get(txnId, rid) + val newPk = primaryKeyValuesOf(newRow) + val oldPk = primaryKeyValuesOf(oldRow) + if (newPk != oldPk) rejectDuplicatePrimaryKey(txnId, newPk) + rejectDuplicateUniqueSecondary(txnId, newRow, excludeRid = rid) + rejectDuplicateUniqueConstraints(txnId, newRow, excludeRid = rid) + val roots = currentRoots() + unindexRow(txnId, rid, oldRow) + val newRid = heap.update(txnId, rid, TupleCodec.encode(newRow.values)) + indexRow(txnId, newRid, newRow, newPk) + persistIfRootsChanged(txnId, roots) + return newRid + } + + /** Convenience for a single-column PRIMARY KEY; delegates to the general, composite-capable overload below. */ + fun findByPrimaryKey(txnId: Long, key: Value): Pair? = findByPrimaryKey(txnId, listOf(key)) + + /** + * Point lookup by the table's PRIMARY KEY, whether inline-single-column or table-level + * composite - [keys] must be given in [TableSchema.primaryKeyColumns] order. [KeyCodec.encodeKey] + * concatenates each value's own order-preserving encoding, so a single-element list here + * produces byte-identical output to [KeyCodec.encode] on that same value, keeping this the + * one lookup path both PRIMARY KEY shapes share. [keys] is folded through + * [encodePrimaryKey] first, so a search literal on a `COLLATE NOCASE`/`RTRIM`-declared PRIMARY + * KEY column finds a case-/trailing-space-variant row the same way a plain `=` comparison does. + */ + fun findByPrimaryKey(txnId: Long, keys: List): Pair? { + pointLookupCount++ + val index = primaryIndex ?: throw DiskTableException("table ${schema.name} has no primary key index") + val ridBits = index.search(txnId, encodePrimaryKey(keys)) ?: return null + val rid = RecordId.decode(ridBits) + return rid to get(txnId, rid) + } + + /** + * [DiskIndexAccessPlanner] plans a range against either a single-column PRIMARY KEY ([prefix] + * empty, see [DiskAccessPlan.PrimaryKeyRange]'s own doc) or the LAST column of a table-level + * composite one, after an equality prefix on every column before it ([prefix] non-empty, see + * [DiskAccessPlan.CompositePrimaryKeyRange]'s own doc) - either way, [prefix] plus [lower]/ + * [upper] always names every PRIMARY KEY column, so [encodePrimaryKey]'s own per-position + * collation folding ([kestrel.engine.collationFoldedUniquenessKey]) applies directly and always + * produces a COMPLETE, full-arity key: unlike [rangeBySecondary], no RID-suffix bracketing + * trick is needed here, since a PRIMARY KEY entry is a bare `encodeKey(values)` with nothing + * after it to disambiguate. + */ + fun rangeByPrimaryKey( + txnId: Long, + lower: Value? = null, + lowerInclusive: Boolean = true, + upper: Value? = null, + upperInclusive: Boolean = true, + prefix: List = emptyList() + ): List> { + rangeScanCount++ + val index = primaryIndex ?: throw DiskTableException("table ${schema.name} has no primary key index") + fun encodeBound(value: Value): ByteArray = + KeyCodec.encodeKey(collationFoldedUniquenessKey(schema, schema.primaryKeyIndices, prefix + value)) + val lowerBytes = lower?.let(::encodeBound)?.let { if (lowerInclusive) it else strictlyAfter(it) } + val upperBytes = upper?.let(::encodeBound) + return index.rangeScan(txnId, lowerBytes, upperBytes, toInclusive = upperInclusive).map { (_, ridBits) -> + val rid = RecordId.decode(ridBits) + rid to get(txnId, rid) + } + } + + /** Convenience for a single-column secondary index; delegates to the general, composite-capable overload below. */ + fun findBySecondary(txnId: Long, indexName: String, key: Value): List> = + findBySecondary(txnId, indexName, listOf(key)) + + /** + * Equality lookup on a secondary index, whether single-column or composite - [keys] must be + * given in [SecondaryIndexHandle.columnNames] order. A single-element list here produces a + * byte-identical prefix to the single-[Value] overload, keeping this the one lookup path both + * shapes share (mirrors [findByPrimaryKey]). [keys] is folded per-column through + * [equalRangeFor], so equality against a declared-collation column finds a case-/trailing- + * space-variant row instead of missing it. + */ + fun findBySecondary(txnId: Long, indexName: String, keys: List): List> { + secondaryLookupCount++ + val handle = secondaryIndexHandles.firstOrNull { it.name.equals(indexName, ignoreCase = true) } + ?: throw DiskTableException("no such index: $indexName on table ${schema.name}") + val (lower, upper) = equalRangeFor(handle.columnIndices, keys) + return handle.index.rangeScan(txnId, lower, upper, toInclusive = true).map { (_, ridBits) -> + val rid = RecordId.decode(ridBits) + rid to get(txnId, rid) + } + } + + /** + * [DiskIndexAccessPlanner] plans a range against a secondary index's column right after + * [prefix]'s equality run (see [kestrel.store.sql.DiskAccessPlan.SecondaryRange]'s own doc) - + * [prefix] is empty for a single-column index or a composite one ranged on its very first + * column. Unlike [rangeByPrimaryKey] (whose B+Tree entries are bare `encodeKey(values)`, no RID + * suffix, since a PRIMARY KEY is unique), a secondary index entry is `encodeKey(values) ++ rid` + * (see the class doc), so a bound naming just [lower]/[upper]'s own encoded bytes would sit + * strictly *before* every real entry sharing that value (any real key, being longer, sorts + * after its own prefix) - reusing [equalRangeFor]'s (all-zero-suffix, all-one-suffix) pair for + * `prefix + value` sidesteps this: the all-zero-suffix bound is a valid inclusive lower + * bound for "this value or greater" (it sits at or before every RID-suffixed entry of that + * exact value), and the all-one-suffix bound is a valid inclusive upper bound for "this value + * or less" (it sits at or after every RID-suffixed entry of that exact value) - swapped for the + * opposite (exclusive) side of each bound, so e.g. `> X` starts scanning right after the last + * possible entry of exactly `X`. When [prefix] is non-empty and one side has no explicit bound + * (e.g. `WHERE a = 5 AND b < 10`, no lower bound on `b`), that side falls back to + * [equalRangeFor]'s own bound for [prefix] alone, so the scan still stays within the `a = 5` + * group instead of running off into rows from a smaller/larger `a` - [prefix]'s own encoding is + * a byte-string prefix of every real entry sharing it (each value's encoding is self-delimited), + * so this bound is exact regardless of how many values [prefix] holds relative to the index's + * full column count. + */ + fun rangeBySecondary( + txnId: Long, + indexName: String, + prefix: List = emptyList(), + lower: Value? = null, + lowerInclusive: Boolean = true, + upper: Value? = null, + upperInclusive: Boolean = true + ): List> { + rangeScanCount++ + val handle = secondaryIndexHandles.firstOrNull { it.name.equals(indexName, ignoreCase = true) } + ?: throw DiskTableException("no such index: $indexName on table ${schema.name}") + val prefixBounds = if (prefix.isNotEmpty()) equalRangeFor(handle.columnIndices, prefix) else null + val lowerBytes = if (lower != null) { + val (inclusiveBound, exclusiveBound) = equalRangeFor(handle.columnIndices, prefix + lower) + if (lowerInclusive) inclusiveBound else exclusiveBound + } else { + prefixBounds?.first + } + val upperBytes = if (upper != null) { + val (exclusiveBound, inclusiveBound) = equalRangeFor(handle.columnIndices, prefix + upper) + if (upperInclusive) inclusiveBound else exclusiveBound + } else { + prefixBounds?.second + } + return handle.index.rangeScan(txnId, lowerBytes, upperBytes, toInclusive = true).map { (_, ridBits) -> + val rid = RecordId.decode(ridBits) + rid to get(txnId, rid) + } + } + + /** + * When [handle] is [SecondaryIndexHandle.unique], a table with pre-existing duplicate values + * in that column must fail cleanly (like `CREATE UNIQUE INDEX` does in the in-memory engine) + * rather than silently building an index that lies about uniqueness - [seen] catches that + * before any B+Tree entry is written, folded through each column's declared `COLLATE` (see + * [collationFoldedUniquenessKey]) so e.g. two pre-existing rows `'Bob'`/`'bob'` under a + * `NOCASE`-declared column are caught here just like [rejectDuplicateUniqueConstraints] + * already catches them for a declarative `UNIQUE`. The B+Tree entry itself ([compositeKey]) + * is ALSO folded per-column through the same declared collation, so a value inserted *after* + * this backfill is caught too, by [rejectDuplicateUniqueSecondary]'s own range scan. + */ + internal fun backfillSecondaryIndex(txnId: Long, handle: SecondaryIndexHandle) { + val seen = if (handle.unique) HashSet>() else null + for ((rid, row) in scan(txnId)) { + val values = handle.columnIndices.map { row.values[it] } + if (values.any { it.isNull }) continue + if (seen != null && !seen.add(collationFoldedUniquenessKey(schema, handle.columnIndices, values))) { + throw DiskTableException("UNIQUE constraint failed: ${schema.name}.${handle.columnNames.joinToString(", ")}") + } + handle.index.insert(txnId, compositeKey(handle.columnIndices, values, rid), rid.encode()) + } + } + + fun snapshotDirectoryEntry(): TableDirectoryEntry = TableDirectoryEntry( + tableName = schema.name, + columns = schema.columns, + dataHeaderPageId = heap.headerPageId, + primaryKeyIndexRootPageId = primaryIndex?.rootPageId, + secondaryIndexes = secondaryIndexHandles.map { SecondaryIndexEntry(it.name, it.columnNames, it.index.rootPageId, it.unique) }, + compositePrimaryKey = schema.compositePrimaryKey, + uniqueConstraints = schema.uniqueConstraints, + checks = schema.checks.map { CheckConstraintEntry.from(it) } + ) + + private fun indexRow(txnId: Long, rid: RecordId, row: Row, pkValues: List?) { + primaryIndex?.let { index -> if (pkValues != null && pkValues.none { it.isNull }) index.insert(txnId, encodePrimaryKey(pkValues), rid.encode()) } + for (handle in secondaryIndexHandles) { + val values = handle.columnIndices.map { row.values[it] } + if (values.none { it.isNull }) handle.index.insert(txnId, compositeKey(handle.columnIndices, values, rid), rid.encode()) + } + } + + private fun unindexRow(txnId: Long, rid: RecordId, row: Row) { + primaryIndex?.let { index -> + val pkValues = primaryKeyValuesOf(row) + if (pkValues != null && pkValues.none { it.isNull }) index.delete(txnId, encodePrimaryKey(pkValues)) + } + for (handle in secondaryIndexHandles) { + val values = handle.columnIndices.map { row.values[it] } + if (values.none { it.isNull }) handle.index.delete(txnId, compositeKey(handle.columnIndices, values, rid)) + } + } + + private fun rejectDuplicatePrimaryKey(txnId: Long, pkValues: List?) { + val index = primaryIndex ?: return + if (pkValues == null || pkValues.any { it.isNull }) return + if (index.search(txnId, encodePrimaryKey(pkValues)) != null) { + throw DiskTableException("UNIQUE constraint failed: ${schema.name}.${schema.primaryKeyColumns.joinToString(", ")}") + } + } + + /** + * Rejects [row] if any [SecondaryIndexHandle.unique] index would end up with two rows sharing + * a non-null tuple of values across its column(s). Searches the index directly (like + * [rejectDuplicatePrimaryKey]) rather than through [findBySecondary], so this internal check + * never inflates [secondaryLookupCount]. [excludeRid] is the row being updated (so it does not + * collide with its own prior entry); `null` on insert. [equalRangeFor] folds [values] per + * column through its declared collation, so this range scan is collation-aware exactly like + * the B+Tree entries [compositeKey] wrote in the first place. + */ + private fun rejectDuplicateUniqueSecondary(txnId: Long, row: Row, excludeRid: RecordId?) { + for (handle in secondaryIndexHandles) { + if (!handle.unique) continue + val values = handle.columnIndices.map { row.values[it] } + if (values.any { it.isNull }) continue + val (lower, upper) = equalRangeFor(handle.columnIndices, values) + val conflict = handle.index.rangeScan(txnId, lower, upper, toInclusive = true).any { (_, ridBits) -> + RecordId.decode(ridBits) != excludeRid + } + if (conflict) throw DiskTableException("UNIQUE constraint failed: ${schema.name}.${handle.columnNames.joinToString(", ")}") + } + } + + /** + * [TableSchema.uniqueConstraints] have no backing index (unlike a `CREATE UNIQUE INDEX`), so + * they are checked with a full heap scan per write - the same approach [kestrel.engine.Executor] + * uses for the in-memory engine, and the only one that generalizes to a constraint spanning more + * than one column without inventing a composite-key index format. A row where any constrained + * column is NULL is exempt, per standard SQL (NULLs never collide with each other). Every + * comparison is folded through [collationFoldedUniquenessKey], so this scan - unlike + * [rejectDuplicateUniqueSecondary]'s B+Tree range scan - is fully collation-correct on every + * call, not just at index-creation time: there is no backing structure whose physical byte + * order would need to change to keep up. + */ + private fun rejectDuplicateUniqueConstraints(txnId: Long, row: Row, excludeRid: RecordId?) { + if (schema.uniqueConstraints.isEmpty()) return + val rows = rawScan(txnId) + for (unique in schema.uniqueConstraints) { + val indices = unique.columns.map { schema.columnIndex(it)!! } + val rawKey = indices.map { row.values[it] } + if (rawKey.any { it.isNull }) continue + val key = collationFoldedUniquenessKey(schema, indices, rawKey) + val conflict = rows.any { (candidateRid, candidateRow) -> + candidateRid != excludeRid && collationFoldedUniquenessKey(schema, indices, indices.map { candidateRow.values[it] }) == key + } + if (conflict) { + val label = unique.name?.let { " $it" } ?: "" + throw DiskTableException("UNIQUE constraint$label failed: ${schema.name}(${unique.columns.joinToString(", ")})") + } + } + } + + /** `null` when the table has no PRIMARY KEY at all; one value per [TableSchema.primaryKeyColumns] otherwise. */ + private fun primaryKeyValuesOf(row: Row): List? { + val indices = schema.primaryKeyIndices + if (indices.isEmpty()) return null + return indices.map { row.values[it] } + } + + private fun requireArity(row: Row) { + if (row.values.size != schema.columns.size) { + throw DiskTableException("table ${schema.name} expects ${schema.columns.size} values but got ${row.values.size}") + } + for ((index, column) in schema.columns.withIndex()) { + if (column.notNull && row.values[index].isNull) { + throw DiskTableException("NOT NULL constraint failed: ${schema.name}.${column.name}") + } + } + } + + private fun currentRoots(): List = + listOfNotNull(primaryIndex?.rootPageId) + secondaryIndexHandles.map { it.index.rootPageId } + + private fun persistIfRootsChanged(txnId: Long, before: List) { + if (before != currentRoots()) onRootsChanged(txnId) + } + + /** + * A secondary index's B+Tree key is `encodeKey(values) ++ rid` (see the class doc) - [values] + * is folded per-position through [columnIndices]' own declared collation first (mirrors + * [encodePrimaryKey]), so a `NOCASE`/`RTRIM`-declared column's index physically groups + * case-/trailing-space-variant values together instead of ordering/comparing them by raw bytes. + */ + private fun compositeKey(columnIndices: List, values: List, rid: RecordId): ByteArray = + ByteWriter().writeBytes(KeyCodec.encodeKey(collationFoldedUniquenessKey(schema, columnIndices, values))) + .writeFixedLong(rid.encode()).toByteArray() + + private fun equalRangeFor(columnIndices: List, values: List): Pair { + val prefix = KeyCodec.encodeKey(collationFoldedUniquenessKey(schema, columnIndices, values)) + val lower = ByteWriter().writeBytes(prefix).writeFixedLong(0L).toByteArray() + val upper = ByteWriter().writeBytes(prefix).writeFixedLong(-1L).toByteArray() + return lower to upper + } + + /** + * [KeyCodec.encodeKey] on [values] (in [TableSchema.primaryKeyColumns] order) after folding + * each position through that column's own declared collation - shared by every PRIMARY KEY + * B+Tree operation ([indexRow]/[unindexRow]/[findByPrimaryKey]/[rejectDuplicatePrimaryKey]) so + * insertion and lookup always agree on what bytes a given value encodes to. + */ + private fun encodePrimaryKey(values: List): ByteArray = + KeyCodec.encodeKey(collationFoldedUniquenessKey(schema, schema.primaryKeyIndices, values)) + + /** + * Smallest key ordered strictly after [bytes]. Correct for the fixed-width + * encodings (INTEGER, REAL, BOOLEAN): appending a byte cannot be + * overtaken by another same-type key, since two distinct fixed-width + * encodings always differ within their common, unextended length. TEXT + * range queries should prefer an inclusive lower bound instead. + */ + private fun strictlyAfter(bytes: ByteArray): ByteArray = bytes + byteArrayOf(0) +} diff --git a/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntry.kt b/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntry.kt new file mode 100644 index 0000000..4c14b64 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntry.kt @@ -0,0 +1,21 @@ +package kestrel.store.sql + +/** + * The durable, on-disk description of one SEQUENCE's static configuration - mirrors + * [kestrel.catalog.SequenceDefinition] field-for-field, plus [positionPageId]: the raw page (see + * [kestrel.store.engine.StorageEngine.allocateRawPage]) that holds its live counter. Deliberately + * split from the counter itself for the same reason [kestrel.catalog.Sequence] separates + * [kestrel.catalog.SequenceDefinition] from its own mutable `value`/`called` fields - CREATE/DROP/ + * ALTER SEQUENCE go through the normal transactional directory heap (see [DiskCatalog]) so they roll + * back like any other DDL, but a NEXTVAL advance must never be undone by a ROLLBACK, so it lives on + * its own page written outside any transaction (see [StorageEngine.writeRawPage][kestrel.store.engine.StorageEngine.writeRawPage]). + */ +data class SequenceDirectoryEntry( + val name: String, + val incrementBy: Long, + val minValue: Long, + val maxValue: Long, + val startValue: Long, + val cycle: Boolean, + val positionPageId: Long +) diff --git a/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntryCodec.kt b/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntryCodec.kt new file mode 100644 index 0000000..901962b --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/SequenceDirectoryEntryCodec.kt @@ -0,0 +1,38 @@ +package kestrel.store.sql + +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter + +/** + * Encodes a [SequenceDirectoryEntry] as a self-contained byte payload, mirroring + * [DirectoryEntryCodec]/[ViewDirectoryEntryCodec]. Every numeric field uses [ByteWriter.writeZigZag] + * rather than [ByteWriter.writeFixedLong] because MINVALUE/MAXVALUE/START WITH routinely hold + * negative values (a descending sequence's default MAXVALUE is `-1`, and either bound can be + * [Long.MIN_VALUE]/[Long.MAX_VALUE]) - zigzag keeps the common case (small increments, page ids) + * compact while still round-tripping the full `Long` range. + */ +object SequenceDirectoryEntryCodec { + fun encode(entry: SequenceDirectoryEntry): ByteArray { + val writer = ByteWriter() + writer.writeLengthPrefixed(entry.name.toByteArray(Charsets.UTF_8)) + writer.writeZigZag(entry.incrementBy) + writer.writeZigZag(entry.minValue) + writer.writeZigZag(entry.maxValue) + writer.writeZigZag(entry.startValue) + writer.writeByte(if (entry.cycle) 1 else 0) + writer.writeZigZag(entry.positionPageId) + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): SequenceDirectoryEntry { + val reader = ByteReader(bytes) + val name = String(reader.readLengthPrefixed(), Charsets.UTF_8) + val incrementBy = reader.readZigZag() + val minValue = reader.readZigZag() + val maxValue = reader.readZigZag() + val startValue = reader.readZigZag() + val cycle = reader.readByte() != 0 + val positionPageId = reader.readZigZag() + return SequenceDirectoryEntry(name, incrementBy, minValue, maxValue, startValue, cycle, positionPageId) + } +} diff --git a/src/main/kotlin/kestrel/store/sql/TableDirectoryEntry.kt b/src/main/kotlin/kestrel/store/sql/TableDirectoryEntry.kt new file mode 100644 index 0000000..6f5de9b --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/TableDirectoryEntry.kt @@ -0,0 +1,68 @@ +package kestrel.store.sql + +import kestrel.catalog.CheckConstraint +import kestrel.catalog.Column +import kestrel.catalog.ForeignKeyConstraint +import kestrel.catalog.UniqueConstraint +import kestrel.persistence.SqlWriter + +/** + * One secondary index recorded against a table's directory entry - [columnNames] has more than + * one entry for a composite (multi-column) index. [unique] mirrors a `CREATE UNIQUE INDEX`: + * [DiskTable] rejects an insert/update that would give two rows the same non-null tuple of + * values across [columnNames], the same way it already does for the primary key. + */ +data class SecondaryIndexEntry(val name: String, val columnNames: List, val rootPageId: Long, val unique: Boolean = false) + +/** + * The durable form of a [CheckConstraint]: its expression rendered to SQL text by + * [SqlWriter.expressionText] rather than a bespoke expression codec - mirrors [ViewDirectoryEntry] + * storing a view's query as `CREATE VIEW ... AS ...;` text, for the same reason (a CHECK expression + * can nest arbitrarily deep, so a dedicated binary codec would have to mirror the whole [kestrel.sql.ast.Expression] + * hierarchy). [DiskCatalog] re-derives the [Expression][kestrel.sql.ast.Expression] on load by parsing + * it back through the same [kestrel.sql.lexer.Lexer]/[kestrel.sql.parser.Parser] pair every other SQL + * entry point uses. + */ +data class CheckConstraintEntry(val name: String?, val expressionText: String) { + companion object { + fun from(check: CheckConstraint): CheckConstraintEntry = CheckConstraintEntry(check.name, SqlWriter.expressionText(check.expression)) + } +} + +/** + * The durable, on-disk description of one table: its schema plus every page + * id needed to reopen its storage after a process restart. [DiskCatalog] + * keeps one of these per table in its directory heap, re-encoding it (via + * [DirectoryEntryCodec]) whenever a page id it references changes - most + * often [primaryKeyIndexRootPageId] or a [SecondaryIndexEntry.rootPageId] + * after a B+Tree split or merge moves the tree's root. + */ +data class TableDirectoryEntry( + val tableName: String, + val columns: List, + val dataHeaderPageId: Long, + val primaryKeyIndexRootPageId: Long?, + val secondaryIndexes: List, + /** + * A table-level `PRIMARY KEY (a, b, ...)`, empty when the table instead declares its + * PRIMARY KEY inline on a single [Column] (tracked by [Column.primaryKey]) - mirrors + * [kestrel.catalog.TableSchema.compositePrimaryKey]. + */ + val compositePrimaryKey: List = emptyList(), + /** + * Table-level or inline `UNIQUE`/`UNIQUE (a, b, ...)` constraints, mirroring + * [kestrel.catalog.TableSchema.uniqueConstraints]. Unlike [secondaryIndexes] these are not + * backed by a B+Tree - [DiskTable] enforces them with a full heap scan per write, exactly like + * [kestrel.engine.Executor] does for the in-memory engine, which is what lets a single + * constraint span more than one column without a composite-key index format. + */ + val uniqueConstraints: List = emptyList(), + /** Table-level or column-level `CHECK (...)` constraints, mirroring [kestrel.catalog.TableSchema.checks]. */ + val checks: List = emptyList(), + /** + * Column-level `REFERENCES ...` or table-level `FOREIGN KEY (...) REFERENCES ...` constraints, + * mirroring [kestrel.catalog.TableSchema.foreignKeys]. Stored as plain typed fields (unlike + * [CheckConstraintEntry]) since [ForeignKeyConstraint] holds no arbitrary expression to render. + */ + val foreignKeys: List = emptyList() +) diff --git a/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntry.kt b/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntry.kt new file mode 100644 index 0000000..c11aa8c --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntry.kt @@ -0,0 +1,12 @@ +package kestrel.store.sql + +/** + * The durable, on-disk description of one trigger: its name plus the exact `CREATE TRIGGER ...;` + * text needed to rebuild its [kestrel.catalog.TriggerDefinition] after a process restart - mirrors + * [ViewDirectoryEntry]'s own reasoning: a trigger's WHEN clause and body can be arbitrarily deep AST + * (any INSERT/UPDATE/DELETE, each with its own WHERE/SET/VALUES), and [kestrel.persistence.SqlWriter. + * triggerText] already knows how to render any of that back to text, so re-parsing it with the normal + * [kestrel.sql.lexer.Lexer]/[kestrel.sql.parser.Parser] pair on [DiskCatalog.open] reuses + * already-tested machinery instead of a second, trigger-specific AST codec. + */ +data class TriggerDirectoryEntry(val triggerName: String, val statementText: String) diff --git a/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntryCodec.kt b/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntryCodec.kt new file mode 100644 index 0000000..7e1cefe --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/TriggerDirectoryEntryCodec.kt @@ -0,0 +1,21 @@ +package kestrel.store.sql + +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter + +/** Encodes a [TriggerDirectoryEntry] as a self-contained byte payload, mirroring [ViewDirectoryEntryCodec]. */ +object TriggerDirectoryEntryCodec { + fun encode(entry: TriggerDirectoryEntry): ByteArray { + val writer = ByteWriter() + writer.writeLengthPrefixed(entry.triggerName.toByteArray(Charsets.UTF_8)) + writer.writeLengthPrefixed(entry.statementText.toByteArray(Charsets.UTF_8)) + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): TriggerDirectoryEntry { + val reader = ByteReader(bytes) + val triggerName = String(reader.readLengthPrefixed(), Charsets.UTF_8) + val statementText = String(reader.readLengthPrefixed(), Charsets.UTF_8) + return TriggerDirectoryEntry(triggerName, statementText) + } +} diff --git a/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntry.kt b/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntry.kt new file mode 100644 index 0000000..2ecf7e3 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntry.kt @@ -0,0 +1,12 @@ +package kestrel.store.sql + +/** + * The durable, on-disk description of one view: its name plus the exact `CREATE VIEW ... AS ...;` + * text needed to rebuild its [kestrel.sql.ast.QueryStatement] after a process restart. Unlike + * [TableDirectoryEntry] this stores rendered SQL text rather than a structured payload - a view's + * defining query can be arbitrarily deep AST (joins, subqueries, set operations, CTEs), and + * [kestrel.persistence.SqlWriter] already knows how to render any of that back to text, so + * re-parsing that text with the normal [kestrel.sql.lexer.Lexer]/[kestrel.sql.parser.Parser] pair + * on [DiskCatalog.open] reuses already-tested machinery instead of a second, view-specific AST codec. + */ +data class ViewDirectoryEntry(val viewName: String, val statementText: String) diff --git a/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntryCodec.kt b/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntryCodec.kt new file mode 100644 index 0000000..36a9b89 --- /dev/null +++ b/src/main/kotlin/kestrel/store/sql/ViewDirectoryEntryCodec.kt @@ -0,0 +1,21 @@ +package kestrel.store.sql + +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter + +/** Encodes a [ViewDirectoryEntry] as a self-contained byte payload, mirroring [DirectoryEntryCodec]. */ +object ViewDirectoryEntryCodec { + fun encode(entry: ViewDirectoryEntry): ByteArray { + val writer = ByteWriter() + writer.writeLengthPrefixed(entry.viewName.toByteArray(Charsets.UTF_8)) + writer.writeLengthPrefixed(entry.statementText.toByteArray(Charsets.UTF_8)) + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): ViewDirectoryEntry { + val reader = ByteReader(bytes) + val viewName = String(reader.readLengthPrefixed(), Charsets.UTF_8) + val statementText = String(reader.readLengthPrefixed(), Charsets.UTF_8) + return ViewDirectoryEntry(viewName, statementText) + } +} diff --git a/src/main/kotlin/kestrel/store/txn/LockManager.kt b/src/main/kotlin/kestrel/store/txn/LockManager.kt new file mode 100644 index 0000000..9ab0cef --- /dev/null +++ b/src/main/kotlin/kestrel/store/txn/LockManager.kt @@ -0,0 +1,127 @@ +package kestrel.store.txn + +import kestrel.KestrelException +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +/** + * A resource-level shared/exclusive lock table for strict two-phase locking: + * transactions acquire locks as they go and are expected to release + * everything at once, via [releaseAll], on commit or abort. + * + * Blocked requests are detected for deadlock *before* they wait: acquiring a + * lock builds the tentative wait-for edges from the requester to the + * resource's current holders and walks the existing wait-for graph (formed + * by every other transaction currently blocked) to see whether any of those + * holders already, transitively, waits on the requester. This avoids ever + * parking a thread that could not possibly be woken. What it does not do is + * enforce waiter ordering, so a steady stream of compatible shared requests + * can starve a waiting exclusive request - a real risk this scope accepts in + * exchange for a single shared [java.util.concurrent.locks.Condition] + * instead of a priority queue. + */ +class LockManager { + enum class LockMode { SHARED, EXCLUSIVE } + + class DeadlockException(message: String) : KestrelException(message) + + private data class Waiter(val txnId: Long, val mode: LockMode) + + private class ResourceLock { + val holders = LinkedHashMap() + val waiters = ArrayList() + } + + private val guard = ReentrantLock() + private val condition = guard.newCondition() + private val resources = HashMap() + private val locksHeldByTxn = HashMap>() + + fun acquireShared(txnId: Long, resourceId: Long) = acquire(txnId, resourceId, LockMode.SHARED) + + fun acquireExclusive(txnId: Long, resourceId: Long) = acquire(txnId, resourceId, LockMode.EXCLUSIVE) + + fun release(txnId: Long, resourceId: Long) { + guard.withLock { + val lock = resources[resourceId] ?: return + if (lock.holders.remove(txnId) != null) { + locksHeldByTxn[txnId]?.remove(resourceId) + condition.signalAll() + } + } + } + + /** Releases every lock [txnId] holds. Called once, on commit or abort, under strict 2PL. */ + fun releaseAll(txnId: Long) { + guard.withLock { + val held = locksHeldByTxn.remove(txnId) ?: return + for (resourceId in held) { + resources[resourceId]?.holders?.remove(txnId) + } + condition.signalAll() + } + } + + /** Snapshot of current holders, e.g. for a `pg_locks`-style inspection view. */ + fun holders(resourceId: Long): Map = guard.withLock { + resources[resourceId]?.holders?.toMap() ?: emptyMap() + } + + fun waiters(resourceId: Long): List = guard.withLock { + resources[resourceId]?.waiters?.map { it.txnId } ?: emptyList() + } + + private fun acquire(txnId: Long, resourceId: Long, mode: LockMode) { + guard.withLock { + val lock = resources.getOrPut(resourceId) { ResourceLock() } + val current = lock.holders[txnId] + if (current == LockMode.EXCLUSIVE || current == mode) return@withLock + + if (!canGrant(lock, txnId, mode)) { + val blockers = lock.holders.keys.filter { it != txnId } + if (hasCycle(txnId, blockers)) { + throw DeadlockException( + "granting $mode on resource $resourceId to txn $txnId would deadlock with $blockers" + ) + } + val waiter = Waiter(txnId, mode) + lock.waiters.add(waiter) + try { + while (!canGrant(lock, txnId, mode)) { + condition.awaitUninterruptibly() + } + } finally { + lock.waiters.remove(waiter) + } + } + + lock.holders[txnId] = mode + locksHeldByTxn.getOrPut(txnId) { LinkedHashSet() }.add(resourceId) + } + } + + private fun canGrant(lock: ResourceLock, txnId: Long, mode: LockMode): Boolean { + val others = lock.holders.filterKeys { it != txnId } + if (others.isEmpty()) return true + return mode == LockMode.SHARED && others.values.all { it == LockMode.SHARED } + } + + /** True if any txn in [from] transitively waits, via the existing wait-for graph, on [target]. */ + private fun hasCycle(target: Long, from: List): Boolean { + val visited = HashSet() + val stack = ArrayDeque(from) + while (stack.isNotEmpty()) { + val current = stack.removeLast() + if (current == target) return true + if (!visited.add(current)) continue + for (lock in resources.values) { + if (lock.waiters.any { it.txnId == current }) { + for (holderTxn in lock.holders.keys) { + if (holderTxn != current) stack.addLast(holderTxn) + } + } + } + } + return false + } +} diff --git a/src/main/kotlin/kestrel/store/txn/TransactionManager.kt b/src/main/kotlin/kestrel/store/txn/TransactionManager.kt new file mode 100644 index 0000000..e2b9c84 --- /dev/null +++ b/src/main/kotlin/kestrel/store/txn/TransactionManager.kt @@ -0,0 +1,84 @@ +package kestrel.store.txn + +import kestrel.KestrelException +import kestrel.store.page.DiskManager +import kestrel.store.wal.RecoveryManager +import kestrel.store.wal.WriteAheadLog + +class TransactionException(message: String) : KestrelException(message) + +/** + * Ties [WriteAheadLog], [RecoveryManager] and [LockManager] into a single + * begin/commit/abort surface: a transaction acquires locks as it accesses + * resources, logs a before/after page image for every write, and on abort + * is unwound with the same [RecoveryManager.rollback] logic that undoes an + * unresolved transaction during crash recovery. Every lock a transaction + * holds is released in one shot on commit or abort, per strict 2PL. + */ +class TransactionManager(private val wal: WriteAheadLog, diskManager: DiskManager) { + enum class Status { ACTIVE, COMMITTED, ABORTED } + + private val lockManager = LockManager() + private val recoveryManager = RecoveryManager(wal, diskManager) + private val statusByTxn = HashMap() + + fun begin(): Long { + val txnId = wal.beginTransaction() + statusByTxn[txnId] = Status.ACTIVE + return txnId + } + + fun acquireShared(txnId: Long, resourceId: Long) { + requireActive(txnId) + lockManager.acquireShared(txnId, resourceId) + } + + fun acquireExclusive(txnId: Long, resourceId: Long) { + requireActive(txnId) + lockManager.acquireExclusive(txnId, resourceId) + } + + fun logUpdate(txnId: Long, pageId: Long, before: ByteArray, after: ByteArray): Long { + requireActive(txnId) + return wal.logUpdate(txnId, pageId, before, after) + } + + /** The LSN a savepoint on [txnId] should capture right now - see [rollbackTo]. */ + fun currentLsn(txnId: Long): Long { + requireActive(txnId) + return wal.lastLsnOf(txnId) + } + + /** + * Undoes [txnId]'s updates back to a [savepointLsn] earlier captured via [currentLsn], without + * resolving the transaction - unlike [abort], it remains ACTIVE and keeps every lock it holds (2PL + * only releases locks at the transaction's own commit or abort, never at a savepoint boundary, so + * an equally-isolated concurrent transaction can't observe a savepoint rollback happening). + */ + fun rollbackTo(txnId: Long, savepointLsn: Long) { + requireActive(txnId) + recoveryManager.rollbackTo(txnId, savepointLsn) + } + + fun commit(txnId: Long) { + requireActive(txnId) + wal.commit(txnId) + lockManager.releaseAll(txnId) + statusByTxn[txnId] = Status.COMMITTED + } + + fun abort(txnId: Long) { + requireActive(txnId) + recoveryManager.rollback(txnId) + lockManager.releaseAll(txnId) + statusByTxn[txnId] = Status.ABORTED + } + + fun statusOf(txnId: Long): Status = statusByTxn[txnId] ?: throw TransactionException("unknown transaction $txnId") + + private fun requireActive(txnId: Long) { + if (statusByTxn[txnId] != Status.ACTIVE) { + throw TransactionException("transaction $txnId is not active") + } + } +} diff --git a/src/main/kotlin/kestrel/store/wal/LogRecord.kt b/src/main/kotlin/kestrel/store/wal/LogRecord.kt new file mode 100644 index 0000000..04f4f0a --- /dev/null +++ b/src/main/kotlin/kestrel/store/wal/LogRecord.kt @@ -0,0 +1,50 @@ +package kestrel.store.wal + +/** + * A single write-ahead log entry. [Update] carries whole-page before/after + * images rather than a delta, so redoing it is just overwriting a page with + * [Update.after]; undoing it is overwriting with [Update.before]. `prevLsn` + * threads a transaction's updates into a backward-traversable chain, the + * same role it plays in ARIES. + */ +sealed class LogRecord { + class Begin(val txnId: Long) : LogRecord() + + class Commit(val txnId: Long) : LogRecord() + + class Abort(val txnId: Long) : LogRecord() + + class Update( + val txnId: Long, + val prevLsn: Long, + val pageId: Long, + val before: ByteArray, + val after: ByteArray + ) : LogRecord() + + /** + * A compensation log record (CLR), ARIES terminology for "this page was just + * restored to an earlier image by an undo". Written by [kestrel.store.wal.RecoveryManager.rollbackTo] + * for every [Update] it undoes on the way to a savepoint, so that: + * - forward redo ([kestrel.store.wal.RecoveryManager.recover]) can reproduce the undo's effect + * just like any other logged write, by reapplying [undoImage]. + * - a later undo walk (a full [kestrel.store.wal.RecoveryManager.rollback], or a rollback to an + * earlier savepoint) that reaches this record never re-undoes the [Update] it compensates for; + * it jumps straight to [undoNextLsn] instead, which is that [Update]'s own `prevLsn` - the + * transaction's undo chain skips the already-undone range as if it had never happened, exactly + * the property that makes partial rollback safe to repeat and safe to crash through. + * Chains into the transaction's `prevLsn` thread the same way [Update] does, via [prevLsn] - so a + * write logged after a savepoint rollback correctly points back at this CLR, not at the undone + * [Update] it replaced. + */ + class CompensationUpdate( + val txnId: Long, + val prevLsn: Long, + val pageId: Long, + val undoImage: ByteArray, + val undoNextLsn: Long + ) : LogRecord() + + /** Marks a quiescent checkpoint: written only when no transaction is active. */ + class Checkpoint(val activeTxnIds: List) : LogRecord() +} diff --git a/src/main/kotlin/kestrel/store/wal/LogRecordCodec.kt b/src/main/kotlin/kestrel/store/wal/LogRecordCodec.kt new file mode 100644 index 0000000..7f0f555 --- /dev/null +++ b/src/main/kotlin/kestrel/store/wal/LogRecordCodec.kt @@ -0,0 +1,85 @@ +package kestrel.store.wal + +import kestrel.KestrelException +import kestrel.store.codec.ByteReader +import kestrel.store.codec.ByteWriter + +class WalCorruptionException(message: String) : KestrelException(message) + +/** Serializes [LogRecord]s to and from the byte payload stored in each log frame. */ +object LogRecordCodec { + private const val TYPE_BEGIN = 1 + private const val TYPE_COMMIT = 2 + private const val TYPE_ABORT = 3 + private const val TYPE_UPDATE = 4 + private const val TYPE_CHECKPOINT = 5 + private const val TYPE_COMPENSATION_UPDATE = 6 + + fun encode(record: LogRecord): ByteArray { + val writer = ByteWriter() + when (record) { + is LogRecord.Begin -> { + writer.writeByte(TYPE_BEGIN) + writer.writeFixedLong(record.txnId) + } + is LogRecord.Commit -> { + writer.writeByte(TYPE_COMMIT) + writer.writeFixedLong(record.txnId) + } + is LogRecord.Abort -> { + writer.writeByte(TYPE_ABORT) + writer.writeFixedLong(record.txnId) + } + is LogRecord.Update -> { + writer.writeByte(TYPE_UPDATE) + writer.writeFixedLong(record.txnId) + writer.writeFixedLong(record.prevLsn) + writer.writeFixedLong(record.pageId) + writer.writeLengthPrefixed(record.before) + writer.writeLengthPrefixed(record.after) + } + is LogRecord.Checkpoint -> { + writer.writeByte(TYPE_CHECKPOINT) + writer.writeVarInt(record.activeTxnIds.size.toLong()) + for (id in record.activeTxnIds) writer.writeFixedLong(id) + } + is LogRecord.CompensationUpdate -> { + writer.writeByte(TYPE_COMPENSATION_UPDATE) + writer.writeFixedLong(record.txnId) + writer.writeFixedLong(record.prevLsn) + writer.writeFixedLong(record.pageId) + writer.writeLengthPrefixed(record.undoImage) + writer.writeFixedLong(record.undoNextLsn) + } + } + return writer.toByteArray() + } + + fun decode(bytes: ByteArray): LogRecord { + val reader = ByteReader(bytes) + return when (val type = reader.readByte()) { + TYPE_BEGIN -> LogRecord.Begin(reader.readFixedLong()) + TYPE_COMMIT -> LogRecord.Commit(reader.readFixedLong()) + TYPE_ABORT -> LogRecord.Abort(reader.readFixedLong()) + TYPE_UPDATE -> LogRecord.Update( + txnId = reader.readFixedLong(), + prevLsn = reader.readFixedLong(), + pageId = reader.readFixedLong(), + before = reader.readLengthPrefixed(), + after = reader.readLengthPrefixed() + ) + TYPE_CHECKPOINT -> { + val count = reader.readVarInt().toInt() + LogRecord.Checkpoint((0 until count).map { reader.readFixedLong() }) + } + TYPE_COMPENSATION_UPDATE -> LogRecord.CompensationUpdate( + txnId = reader.readFixedLong(), + prevLsn = reader.readFixedLong(), + pageId = reader.readFixedLong(), + undoImage = reader.readLengthPrefixed(), + undoNextLsn = reader.readFixedLong() + ) + else -> throw WalCorruptionException("unknown log record type $type") + } + } +} diff --git a/src/main/kotlin/kestrel/store/wal/RecoveryManager.kt b/src/main/kotlin/kestrel/store/wal/RecoveryManager.kt new file mode 100644 index 0000000..5a3b233 --- /dev/null +++ b/src/main/kotlin/kestrel/store/wal/RecoveryManager.kt @@ -0,0 +1,163 @@ +package kestrel.store.wal + +import kestrel.KestrelException +import kestrel.store.page.DiskManager +import kestrel.store.page.Page + +class RecoveryException(message: String) : KestrelException(message) + +/** + * ARIES-lite recovery over a [WriteAheadLog] of whole-page physical images. + * + * Because every [LogRecord.Update] carries the exact after-image a page had + * once that write completed, replaying every update from the last + * checkpoint forward in LSN order reproduces the pre-crash contents exactly + * - no page-LSN comparison is needed to make redo idempotent, unlike + * classic ARIES with incremental (delta) logging. The tradeoff is a bigger + * log: every logged write carries a full page image rather than a diff. + * + * Undo walks each unresolved transaction's [LogRecord.Update] chain + * backward via `prevLsn`, restoring the before-image at each step. A crash + * partway through undoing a transaction requires re-running undo for it + * from the start on the next recovery pass ([recover]'s forward redo pass + * reproduces whatever partial undo already happened before the crash, via + * any [LogRecord.CompensationUpdate] records it wrote, and the backward undo + * pass then continues from there) rather than resuming partway through a + * still-in-flight undo. + * + * [rollbackTo] undoes only a *suffix* of a transaction's updates, back to a + * savepoint LSN captured earlier in the same transaction, and writes a + * [LogRecord.CompensationUpdate] for each [LogRecord.Update] it undoes. A + * later full [rollback] (or a rollback to an earlier savepoint) that walks + * back through a CLR does not undo it again - it follows [LogRecord.CompensationUpdate.undoNextLsn] + * straight past the range it already compensated for. This is the classic + * ARIES technique for making undo of undo safe without needing a full CLR + * chain of its own. + * + * Redo only ever targets pages already allocated in [diskManager]; page + * allocation itself is not logged in this scope, so the log can only + * restore content, not resurrect a page directory lost mid-crash. + * + * Because [rollback] restores before-images with plain physical writes + * rather than logging its own compensation records, a transaction that + * finished with an [LogRecord.Abort] before this recovery pass ran already + * has the correct (pre-write) bytes on disk. Blindly redoing every + * [LogRecord.Update] forward in LSN order - as classic ARIES redo does, + * since it always has CLRs to make that safe - would reapply that + * transaction's after-images and resurrect data it rolled back. [recover] + * therefore runs an analysis pass first to learn each transaction's final + * disposition, then skips redoing updates belonging to one that aborted. + */ +class RecoveryManager(private val wal: WriteAheadLog, private val diskManager: DiskManager) { + + class Result(val redoneUpdates: Int, val rolledBackTxns: List) + + fun recover(): Result { + val startLsn = if (wal.lastCheckpointLsn == WriteAheadLog.NO_LSN) 0L else wal.lastCheckpointLsn + val history = wal.readFrom(startLsn) + + val aborted = HashSet() + for ((_, record) in history) { + if (record is LogRecord.Abort) aborted.add(record.txnId) + } + + val activeTxns = LinkedHashMap() + var redone = 0 + + for ((lsn, record) in history) { + when (record) { + is LogRecord.Begin -> activeTxns[record.txnId] = lsn + is LogRecord.Update -> { + activeTxns[record.txnId] = lsn + if (record.txnId !in aborted) { + applyImage(record.pageId, record.after) + redone++ + } + } + is LogRecord.CompensationUpdate -> { + activeTxns[record.txnId] = lsn + if (record.txnId !in aborted) { + applyImage(record.pageId, record.undoImage) + redone++ + } + } + is LogRecord.Commit -> activeTxns.remove(record.txnId) + is LogRecord.Abort -> activeTxns.remove(record.txnId) + is LogRecord.Checkpoint -> Unit + } + } + + val rolledBack = activeTxns.keys.toList() + for (txnId in rolledBack) rollback(txnId, activeTxns.getValue(txnId)) + diskManager.fsync() + return Result(redone, rolledBack) + } + + /** + * Rolls back a transaction by walking its update chain backward from + * [fromLsn] (defaulting to the tail this [wal] instance has tracked for + * a still-live transaction) and restoring each before-image. Also used + * by [recover] to undo transactions left active by a crash. + */ + fun rollback(txnId: Long, fromLsn: Long = wal.lastLsnOf(txnId)) { + require(fromLsn != WriteAheadLog.NO_LSN) { "transaction $txnId has no recorded updates" } + var lsn = fromLsn + while (lsn != WriteAheadLog.NO_LSN) { + when (val record = wal.readAt(lsn)) { + is LogRecord.Update -> { + applyImage(record.pageId, record.before) + lsn = record.prevLsn + } + is LogRecord.CompensationUpdate -> lsn = record.undoNextLsn + is LogRecord.Begin -> lsn = WriteAheadLog.NO_LSN + else -> throw RecoveryException("unexpected record in undo chain for txn $txnId at lsn $lsn") + } + } + wal.markAborted(txnId) + } + + /** + * Undoes [txnId]'s updates back to [savepointLsn] - the LSN [WriteAheadLog.lastLsnOf] reported for + * [txnId] at the moment the savepoint was established - without resolving the transaction: it + * remains active, can accumulate further writes, and can later be committed, fully rolled back, or + * rolled back to another savepoint. Each undone [LogRecord.Update] gets a + * [LogRecord.CompensationUpdate] so this is safe to call again (e.g. a second rollback to the same + * savepoint after more writes) and safe to crash through (see the class doc). + * + * Walking backward from the transaction's current tail, a [LogRecord.CompensationUpdate] already on + * the chain (written by an earlier, deeper savepoint rollback) is skipped via [LogRecord.CompensationUpdate.undoNextLsn] + * rather than undone again - the range it compensates for is already at its pre-savepoint bytes. + */ + fun rollbackTo(txnId: Long, savepointLsn: Long) { + var lsn = wal.lastLsnOf(txnId) + while (lsn != savepointLsn) { + if (lsn == WriteAheadLog.NO_LSN) { + throw RecoveryException("savepoint lsn $savepointLsn is not on transaction $txnId's undo chain") + } + when (val record = wal.readAt(lsn)) { + is LogRecord.Update -> { + applyImage(record.pageId, record.before) + wal.logCompensation(txnId, record.pageId, record.before, record.prevLsn) + lsn = record.prevLsn + } + is LogRecord.CompensationUpdate -> lsn = record.undoNextLsn + is LogRecord.Begin -> throw RecoveryException( + "savepoint lsn $savepointLsn is not on transaction $txnId's undo chain" + ) + else -> throw RecoveryException("unexpected record in undo chain for txn $txnId at lsn $lsn") + } + } + } + + private fun applyImage(pageId: Long, image: ByteArray) { + if (pageId >= diskManager.allocatedPageCount()) { + throw RecoveryException( + "page $pageId referenced by the log was never allocated; " + + "this recovery scope only redoes updates to already-allocated pages" + ) + } + val page = Page(pageId) + System.arraycopy(image, 0, page.data, 0, image.size) + diskManager.writePage(page) + } +} diff --git a/src/main/kotlin/kestrel/store/wal/WriteAheadLog.kt b/src/main/kotlin/kestrel/store/wal/WriteAheadLog.kt new file mode 100644 index 0000000..2d86f47 --- /dev/null +++ b/src/main/kotlin/kestrel/store/wal/WriteAheadLog.kt @@ -0,0 +1,172 @@ +package kestrel.store.wal + +import kestrel.KestrelException +import kestrel.store.codec.Crc32C +import java.io.File +import java.io.RandomAccessFile + +class WalException(message: String) : KestrelException(message) + +/** + * Append-only binary log file. Every record is framed as + * `[4-byte length][payload][4-byte CRC32C]`, and a record's LSN is simply + * the file offset its frame starts at, which lets [readAt] seek straight to + * a `prevLsn` during undo instead of needing an in-memory LSN index. + * + * Tracks, for transactions begun through this instance, the LSN of their + * most recent record so [logUpdate] can chain `prevLsn` and callers can look + * up a transaction's tail for rollback. Reconstructing that table for + * transactions that were active before a crash is [RecoveryManager]'s job, + * not this class's - on reopen only `nextTxnId` and [lastCheckpointLsn] are + * recovered by scanning the existing file once. + */ +class WriteAheadLog(file: File) : AutoCloseable { + companion object { + const val NO_LSN = -1L + } + + private val raf = RandomAccessFile(file, "rw") + private val lastLsnByTxn = HashMap() + private var nextTxnId: Long + + var lastCheckpointLsn: Long = NO_LSN + private set + + init { + var maxTxnId = 0L + var pos = 0L + val end = raf.length() + while (pos < end) { + val (record, next) = readFrameAt(pos) + val txnId = when (record) { + is LogRecord.Begin -> record.txnId + is LogRecord.Update -> record.txnId + is LogRecord.CompensationUpdate -> record.txnId + is LogRecord.Commit -> record.txnId + is LogRecord.Abort -> record.txnId + is LogRecord.Checkpoint -> { + lastCheckpointLsn = pos + 0L + } + } + if (txnId > maxTxnId) maxTxnId = txnId + pos = next + } + nextTxnId = maxTxnId + 1 + } + + fun beginTransaction(): Long { + val txnId = nextTxnId++ + val lsn = appendRaw(LogRecord.Begin(txnId)) + lastLsnByTxn[txnId] = lsn + return txnId + } + + fun logUpdate(txnId: Long, pageId: Long, before: ByteArray, after: ByteArray): Long { + val prevLsn = lastLsnByTxn[txnId] ?: throw WalException("transaction $txnId is not active") + val lsn = appendRaw(LogRecord.Update(txnId, prevLsn, pageId, before, after)) + lastLsnByTxn[txnId] = lsn + return lsn + } + + /** + * Logs a compensation record for an undo step taken against [txnId] (see + * [LogRecord.CompensationUpdate]) and threads it into the transaction's `prevLsn` chain exactly + * like [logUpdate] does for a forward write, so the transaction's tail always points at the most + * recent record regardless of whether it was a normal write or an undo. + */ + fun logCompensation(txnId: Long, pageId: Long, undoImage: ByteArray, undoNextLsn: Long): Long { + val prevLsn = lastLsnByTxn[txnId] ?: throw WalException("transaction $txnId is not active") + val lsn = appendRaw(LogRecord.CompensationUpdate(txnId, prevLsn, pageId, undoImage, undoNextLsn)) + lastLsnByTxn[txnId] = lsn + return lsn + } + + fun commit(txnId: Long): Long { + if (txnId !in lastLsnByTxn) throw WalException("transaction $txnId is not active") + val lsn = appendRaw(LogRecord.Commit(txnId)) + flush() + lastLsnByTxn.remove(txnId) + return lsn + } + + /** Records that [txnId] is resolved after its updates were undone. Used by [RecoveryManager]. */ + fun markAborted(txnId: Long): Long { + val lsn = appendRaw(LogRecord.Abort(txnId)) + flush() + lastLsnByTxn.remove(txnId) + return lsn + } + + /** Requires no transaction to be active (a quiescent checkpoint). */ + fun checkpoint(): Long { + check(lastLsnByTxn.isEmpty()) { "checkpoint requires no active transactions" } + val lsn = appendRaw(LogRecord.Checkpoint(emptyList())) + flush() + lastCheckpointLsn = lsn + return lsn + } + + fun lastLsnOf(txnId: Long): Long = lastLsnByTxn[txnId] ?: NO_LSN + + fun flush() { + raf.fd.sync() + } + + /** Sequentially reads every record from [startLsn] to the end of the log. */ + fun readFrom(startLsn: Long): List> { + val result = ArrayList>() + var pos = startLsn + val end = raf.length() + while (pos < end) { + val (record, next) = readFrameAt(pos) + result.add(pos to record) + pos = next + } + return result + } + + fun readAt(lsn: Long): LogRecord = readFrameAt(lsn).first + + override fun close() { + flush() + raf.close() + } + + private fun readFrameAt(lsn: Long): Pair { + raf.seek(lsn) + val length = readInt() + val payload = ByteArray(length) + raf.readFully(payload) + val expected = readInt() + val actual = Crc32C.of(payload) + if (actual != expected) throw WalCorruptionException("checksum mismatch for log frame at lsn $lsn") + return LogRecordCodec.decode(payload) to raf.filePointer + } + + private fun appendRaw(record: LogRecord): Long { + val payload = LogRecordCodec.encode(record) + val lsn = raf.length() + raf.seek(lsn) + writeInt(payload.size) + raf.write(payload) + writeInt(Crc32C.of(payload)) + return lsn + } + + private fun writeInt(value: Int) { + raf.write(value ushr 24) + raf.write(value ushr 16) + raf.write(value ushr 8) + raf.write(value) + } + + private fun readInt(): Int { + val a = raf.read() + val b = raf.read() + val c = raf.read() + val d = raf.read() + if (a < 0 || b < 0 || c < 0 || d < 0) throw WalCorruptionException("truncated log frame") + return (a shl 24) or (b shl 16) or (c shl 8) or d + } +} diff --git a/src/test/kotlin/kestrel/AggregateTest.kt b/src/test/kotlin/kestrel/AggregateTest.kt index 8a5f919..686a5a7 100644 --- a/src/test/kotlin/kestrel/AggregateTest.kt +++ b/src/test/kotlin/kestrel/AggregateTest.kt @@ -68,6 +68,13 @@ class AggregateTest { assertEquals(listOf(text("ops"), int(120)), result.rows[1]) } + @Test + fun offsetSkipsGroupsAfterOrderingAggregatedResults() { + seed() + val result = select("SELECT dept, SUM(amount) AS total FROM sales GROUP BY dept ORDER BY dept ASC LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("ops"), int(120))), result.rows) + } + @Test fun groupByMultipleColumns() { seed() @@ -79,6 +86,22 @@ class AggregateTest { assertEquals(listOf(text("ops"), text("eu"), int(2)), result.rows[2]) } + @Test + fun orderByMultipleColumnsOnGroupedResults() { + seed() + val result = select( + "SELECT dept, region, SUM(amount) AS total FROM sales GROUP BY dept, region ORDER BY dept ASC, region DESC" + ) + assertEquals( + listOf( + listOf(text("eng"), text("us"), int(200)), + listOf(text("eng"), text("eu"), int(100)), + listOf(text("ops"), text("eu"), int(120)) + ), + result.rows + ) + } + @Test fun havingFiltersGroups() { seed() @@ -120,4 +143,212 @@ class AggregateTest { assertEquals(listOf(text("eng"), int(303)), result.rows[0]) assertEquals(listOf(text("ops"), int(122)), result.rows[1]) } + + @Test + fun stringAggJoinsWithTheGivenSeparatorInRowOrderWhenThereIsNoOrderByClause() { + run("CREATE TABLE t (label TEXT)") + run("INSERT INTO t VALUES ('c'), ('a'), ('b')") + assertEquals(listOf(listOf(text("c, a, b"))), select("SELECT STRING_AGG(label, ', ') FROM t").rows) + } + + @Test + fun stringAggOrderByInsideTheCallSortsIndependentlyOfInsertionOrder() { + run("CREATE TABLE t (label TEXT, rank INTEGER)") + run("INSERT INTO t VALUES ('c', 3), ('a', 1), ('b', 2)") + val result = select("SELECT STRING_AGG(label, '-' ORDER BY rank) FROM t") + assertEquals(listOf(listOf(text("a-b-c"))), result.rows) + } + + @Test + fun stringAggOrderByDescendingReversesTheConcatenationOrder() { + run("CREATE TABLE t (label TEXT, rank INTEGER)") + run("INSERT INTO t VALUES ('a', 1), ('b', 2), ('c', 3)") + val result = select("SELECT STRING_AGG(label, '-' ORDER BY rank DESC) FROM t") + assertEquals(listOf(listOf(text("c-b-a"))), result.rows) + } + + @Test + fun stringAggSkipsNullValuesButKeepsNonNullOnesInOrder() { + run("CREATE TABLE t (label TEXT)") + run("INSERT INTO t VALUES ('a'), (NULL), ('b')") + assertEquals(listOf(listOf(text("a, b"))), select("SELECT STRING_AGG(label, ', ') FROM t").rows) + } + + @Test + fun stringAggOverOnlyNullValuesIsNull() { + run("CREATE TABLE t (label TEXT)") + run("INSERT INTO t VALUES (NULL), (NULL)") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT STRING_AGG(label, ', ') FROM t").rows) + } + + @Test + fun stringAggOverAnEmptyTableIsNull() { + run("CREATE TABLE t (label TEXT)") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT STRING_AGG(label, ', ') FROM t").rows) + } + + @Test + fun stringAggPerGroupWithGroupBy() { + seed() + val result = select("SELECT dept, STRING_AGG(region, '/' ORDER BY region) FROM sales GROUP BY dept ORDER BY dept ASC") + assertEquals(listOf(text("eng"), text("eu/us/us")), result.rows[0]) + assertEquals(listOf(text("ops"), text("eu/eu")), result.rows[1]) + } + + @Test + fun stringAggRejectsANonTextValueArgument() { + seed() + val failure = assertFailsWith { select("SELECT STRING_AGG(amount, ', ') FROM sales") } + assertEquals("function STRING_AGG requires a TEXT argument but got INTEGER", failure.message) + } + + @Test + fun stringAggRejectsANonTextSeparator() { + run("CREATE TABLE t (label TEXT)") + run("INSERT INTO t VALUES ('a')") + assertFailsWith { select("SELECT STRING_AGG(label, 1) FROM t") } + } + + @Test + fun stringAggCannotBeUsedAsAWindowFunction() { + assertFailsWith { + Parser(Lexer("SELECT STRING_AGG(label, ', ') OVER () FROM t").tokenize()).parseProgram() + } + } + + @Test + fun stringAggRequiresBothAValueAndASeparatorArgument() { + assertFailsWith { + Parser(Lexer("SELECT STRING_AGG(label) FROM t").tokenize()).parseProgram() + } + } + + private fun seedDistinct() { + run("CREATE TABLE t (dept TEXT, amount INTEGER)") + run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + } + + @Test + fun countDistinctCountsEachDistinctNonNullValueOnce() { + seedDistinct() + assertEquals(listOf(listOf(int(4))), select("SELECT COUNT(amount) FROM t").rows) + assertEquals(listOf(listOf(int(2))), select("SELECT COUNT(DISTINCT amount) FROM t").rows) + } + + @Test + fun sumDistinctAddsEachDistinctValueOnce() { + seedDistinct() + assertEquals(listOf(listOf(int(500))), select("SELECT SUM(amount) FROM t").rows) + assertEquals(listOf(listOf(int(300))), select("SELECT SUM(DISTINCT amount) FROM t").rows) + } + + @Test + fun avgDistinctAveragesEachDistinctValueOnce() { + seedDistinct() + assertEquals(listOf(listOf(real(125.0))), select("SELECT AVG(amount) FROM t").rows) + assertEquals(listOf(listOf(real(150.0))), select("SELECT AVG(DISTINCT amount) FROM t").rows) + } + + @Test + fun minAndMaxDistinctAreUnaffectedByDeduplication() { + seedDistinct() + val result = select("SELECT MIN(DISTINCT amount) AS lo, MAX(DISTINCT amount) AS hi FROM t") + assertEquals(listOf(int(100), int(200)), result.rows[0]) + } + + @Test + fun countDistinctIgnoresNulls() { + seedDistinct() + assertEquals(listOf(listOf(int(0))), select("SELECT COUNT(DISTINCT amount) FROM t WHERE dept = 'nope'").rows) + } + + @Test + fun countDistinctPerGroupWithGroupBy() { + seedDistinct() + val result = select("SELECT dept, COUNT(DISTINCT amount) FROM t GROUP BY dept ORDER BY dept ASC") + assertEquals(listOf(listOf(text("eng"), int(2)), listOf(text("ops"), int(1))), result.rows) + } + + @Test + fun countDistinctStarAndDistinctOverAWindowAreRejectedAtParseTime() { + assertFailsWith { Parser(Lexer("SELECT COUNT(DISTINCT *) FROM t").tokenize()).parseProgram() } + assertFailsWith { + Parser(Lexer("SELECT SUM(DISTINCT amount) OVER () FROM t").tokenize()).parseProgram() + } + } + + @Test + fun countStarFilterCountsOnlyRowsMatchingThePredicate() { + seedDistinct() + assertEquals(listOf(listOf(int(5))), select("SELECT COUNT(*) FROM t").rows) + assertEquals(listOf(listOf(int(1))), select("SELECT COUNT(*) FILTER (WHERE amount > 100) FROM t").rows) + } + + @Test + fun countColumnFilterCombinesNullSkippingWithThePredicate() { + seedDistinct() + assertEquals(listOf(listOf(int(3))), select("SELECT COUNT(amount) FILTER (WHERE dept = 'eng') FROM t").rows) + assertEquals(listOf(listOf(int(1))), select("SELECT COUNT(amount) FILTER (WHERE dept = 'ops') FROM t").rows) + } + + @Test + fun sumFilterSumsOnlyRowsMatchingThePredicate() { + seedDistinct() + assertEquals(listOf(listOf(int(400))), select("SELECT SUM(amount) FILTER (WHERE dept = 'eng') FROM t").rows) + assertEquals(listOf(listOf(int(100))), select("SELECT SUM(amount) FILTER (WHERE dept = 'ops') FROM t").rows) + } + + @Test + fun avgMinMaxFilterOperateOnlyOnRowsMatchingThePredicate() { + seedDistinct() + val result = select( + "SELECT AVG(amount) FILTER (WHERE amount >= 150) AS a, " + + "MIN(amount) FILTER (WHERE dept = 'eng') AS lo, " + + "MAX(amount) FILTER (WHERE dept = 'eng') AS hi FROM t" + ) + assertEquals(listOf(real(200.0), int(100), int(200)), result.rows[0]) + } + + @Test + fun filterExcludingEveryRowLeavesCountZeroAndSumAvgMinMaxNull() { + seedDistinct() + val result = select( + "SELECT COUNT(*) FILTER (WHERE dept = 'nope') AS c, " + + "SUM(amount) FILTER (WHERE dept = 'nope') AS s, " + + "AVG(amount) FILTER (WHERE dept = 'nope') AS a FROM t" + ) + assertEquals(listOf(int(0), Value.NullValue, Value.NullValue), result.rows[0]) + } + + @Test + fun filterCombinesWithDistinct() { + seedDistinct() + assertEquals( + listOf(listOf(int(2))), + select("SELECT COUNT(DISTINCT amount) FILTER (WHERE dept = 'eng') FROM t").rows + ) + } + + @Test + fun filterAppliesIndependentlyPerAggregatePerGroup() { + seedDistinct() + val result = select( + "SELECT dept, COUNT(*) FILTER (WHERE amount > 50) AS matching, COUNT(*) AS total " + + "FROM t GROUP BY dept ORDER BY dept ASC" + ) + assertEquals( + listOf(listOf(text("eng"), int(3), int(3)), listOf(text("ops"), int(1), int(2))), + result.rows + ) + } + + @Test + fun filterIsRejectedOnNonAggregateFunctionsAndOnWindowFunctions() { + assertFailsWith { + Parser(Lexer("SELECT UPPER(dept) FILTER (WHERE amount > 0) FROM t").tokenize()).parseProgram() + } + assertFailsWith { + Parser(Lexer("SELECT SUM(amount) FILTER (WHERE amount > 0) OVER () FROM t").tokenize()).parseProgram() + } + } } diff --git a/src/test/kotlin/kestrel/AlterTableTest.kt b/src/test/kotlin/kestrel/AlterTableTest.kt new file mode 100644 index 0000000..60f03f4 --- /dev/null +++ b/src/test/kotlin/kestrel/AlterTableTest.kt @@ -0,0 +1,1089 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.AlterTableResult +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.persistence.SqlWriter +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.math.BigDecimal +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `ALTER TABLE ... ADD/DROP/RENAME COLUMN` and `DEFAULT` - the engine previously had no way to + * change a table's shape after `CREATE TABLE` at all, and no way to fill a column omitted from an + * INSERT column list with anything but NULL. + */ +class AlterTableTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun explain(sql: String): List = (exec(sql) as ExplainResult).plan + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private fun real(value: Double) = Value.RealValue(value) + private fun decimal(value: String) = Value.DecimalValue(BigDecimal(value)) + + // ---- ADD COLUMN ---- + + @Test + fun addColumnAppendsNullToExistingRowsWhenNoDefaultIsGiven() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO t VALUES (1, 'ada')") + val result = exec("ALTER TABLE t ADD COLUMN age INTEGER") + assertEquals(AlterTableResult("t"), result) + assertEquals(listOf(listOf(int(1), text("ada"), Value.NullValue)), select("SELECT * FROM t").rows) + } + + @Test + fun addColumnBackfillsExistingRowsWithDefaultValue() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO t VALUES (1, 'ada'), (2, 'linus')") + run("ALTER TABLE t ADD COLUMN status TEXT DEFAULT 'active'") + assertEquals( + listOf(listOf(text("active")), listOf(text("active"))), + select("SELECT status FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun addColumnRejectsNotNullWithoutDefaultWhenTableHasRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + val error = assertFailsWith { run("ALTER TABLE t ADD COLUMN name TEXT NOT NULL") } + assertTrue(error.message!!.contains("NOT NULL")) + } + + @Test + fun addColumnAllowsNotNullWithoutDefaultWhenTableIsEmpty() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("ALTER TABLE t ADD COLUMN name TEXT NOT NULL") + run("INSERT INTO t VALUES (1, 'ada')") + assertEquals(listOf(listOf(text("ada"))), select("SELECT name FROM t").rows) + assertFailsWith { run("INSERT INTO t (id) VALUES (2)") } + } + + @Test + fun addColumnWithNotNullAndDefaultBackfillsAndKeepsEnforcingGoingForward() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + run("ALTER TABLE t ADD COLUMN score INTEGER NOT NULL DEFAULT 0") + assertEquals(listOf(listOf(int(0))), select("SELECT score FROM t").rows) + assertFailsWith { run("INSERT INTO t (id, score) VALUES (2, NULL)") } + } + + @Test + fun addColumnRejectsDuplicateColumnName() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { run("ALTER TABLE t ADD COLUMN name TEXT") } + assertTrue(error.message!!.contains("duplicate column")) + } + + @Test + fun addColumnRequiresTableToExist() { + assertFailsWith { run("ALTER TABLE missing ADD COLUMN x INTEGER") } + } + + @Test + fun newColumnParticipatesInInsertUpdateWhereAndSelect() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO t VALUES (1, 'ada')") + run("ALTER TABLE t ADD COLUMN age INTEGER DEFAULT 0") + run("INSERT INTO t VALUES (2, 'linus', 54)") + run("UPDATE t SET age = 36 WHERE id = 1") + assertEquals( + listOf(listOf(text("linus"))), + select("SELECT name FROM t WHERE age > 40").rows + ) + } + + @Test + fun defaultExpressionCanBeAnArithmeticConstantNotJustALiteral() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + run("ALTER TABLE t ADD COLUMN adjustment INTEGER DEFAULT -1 + 10") + assertEquals(listOf(listOf(int(9))), select("SELECT adjustment FROM t").rows) + } + + @Test + fun addColumnRejectsPrimaryKeyReferencesUniqueAndCheckConstraints() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + assertFailsWith { run("ALTER TABLE t ADD COLUMN id2 INTEGER PRIMARY KEY") } + assertFailsWith { run("ALTER TABLE t ADD COLUMN dept_id INTEGER REFERENCES departments(id)") } + assertFailsWith { run("ALTER TABLE t ADD COLUMN email TEXT UNIQUE") } + assertFailsWith { run("ALTER TABLE t ADD COLUMN age INTEGER CHECK (age > 0)") } + } + + // ---- DROP COLUMN ---- + + @Test + fun dropColumnRemovesColumnFromSchemaAndData() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, bio TEXT)") + run("INSERT INTO t VALUES (1, 'ada', 'mathematician')") + run("ALTER TABLE t DROP COLUMN bio") + val result = select("SELECT * FROM t") + assertEquals(listOf("id", "name"), result.columns) + assertEquals(listOf(listOf(int(1), text("ada"))), result.rows) + } + + @Test + fun dropColumnRejectsWhenItIsTheOnlyColumn() { + run("CREATE TABLE t (id INTEGER)") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN id") } + assertTrue(error.message!!.contains("no columns")) + } + + @Test + fun dropColumnRejectsPrimaryKeyColumnEvenWithCascade() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN id CASCADE") } + assertTrue(error.message!!.contains("PRIMARY KEY")) + } + + @Test + fun dropColumnRestrictsWhenReferencedByAnotherTablesForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT UNIQUE)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT REFERENCES departments(code))") + val error = assertFailsWith { run("ALTER TABLE departments DROP COLUMN code") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropColumnCascadeRemovesTheReferencingForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT UNIQUE)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT REFERENCES departments(code))") + run("INSERT INTO departments VALUES (1, 'eng')") + run("ALTER TABLE departments DROP COLUMN code CASCADE") + // The FK is gone, so an employee can now reference a nonexistent department code freely. + run("INSERT INTO employees VALUES (1, 'ghost')") + assertEquals(listOf(listOf(text("ghost"))), select("SELECT dept_code FROM employees").rows) + } + + @Test + fun dropColumnRestrictsWhenUsedByItsOwnForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + val error = assertFailsWith { run("ALTER TABLE employees DROP COLUMN dept_id") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropColumnCascadeRemovesItsOwnForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("ALTER TABLE employees DROP COLUMN dept_id CASCADE") + assertEquals(listOf("id"), select("SELECT * FROM employees").columns) + } + + @Test + fun dropColumnRestrictsWhenUsedByAUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN email") } + assertTrue(error.message!!.contains("UNIQUE")) + } + + @Test + fun dropColumnCascadeRemovesTheUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO t VALUES (1, 'a@x.com')") + run("ALTER TABLE t DROP COLUMN email CASCADE") + run("INSERT INTO t VALUES (2)") + assertEquals(2, select("SELECT * FROM t").rows.size) + } + + @Test + fun dropColumnRestrictsWhenUsedByACheckConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN age") } + assertTrue(error.message!!.contains("CHECK")) + } + + @Test + fun dropColumnCascadeRemovesTheCheckConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + run("ALTER TABLE t DROP COLUMN age CASCADE") + run("INSERT INTO t VALUES (1)") + assertEquals(1, select("SELECT * FROM t").rows.size) + } + + @Test + fun dropColumnRestrictsWhenUsedByAnIndex() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("CREATE INDEX idx_age ON t (age)") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN age") } + assertTrue(error.message!!.contains("idx_age")) + } + + @Test + fun dropColumnCascadeRemovesTheIndex() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("CREATE INDEX idx_age ON t (age)") + run("ALTER TABLE t DROP COLUMN age CASCADE") + assertTrue(catalog.indexNames.isEmpty()) + } + + // ---- RENAME COLUMN ---- + + @Test + fun renameColumnRenamesSchemaAndPreservesData() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, bio TEXT)") + run("INSERT INTO t VALUES (1, 'mathematician')") + run("ALTER TABLE t RENAME COLUMN bio TO biography") + val result = select("SELECT * FROM t") + assertEquals(listOf("id", "biography"), result.columns) + assertEquals(listOf(listOf(int(1), text("mathematician"))), result.rows) + } + + @Test + fun renameColumnRejectsUnknownColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + assertFailsWith { run("ALTER TABLE t RENAME COLUMN missing TO whatever") } + } + + @Test + fun renameColumnRejectsWhenNewNameAlreadyExists() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { run("ALTER TABLE t RENAME COLUMN name TO id") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameColumnUpdatesCompositePrimaryKeyListing() { + run("CREATE TABLE t (region TEXT, seq INTEGER, PRIMARY KEY (region, seq))") + run("INSERT INTO t VALUES ('eu', 1)") + run("ALTER TABLE t RENAME COLUMN region TO area") + assertFailsWith { run("INSERT INTO t VALUES ('eu', 1)") } + run("INSERT INTO t VALUES ('eu', 2)") + assertEquals(2, select("SELECT * FROM t").rows.size) + } + + @Test + fun renameColumnUpdatesItsOwnForeignKeyColumns() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("INSERT INTO departments VALUES (1)") + run("ALTER TABLE employees RENAME COLUMN dept_id TO department_id") + assertFailsWith { run("INSERT INTO employees VALUES (1, 99)") } + run("INSERT INTO employees VALUES (1, 1)") + assertEquals(1, select("SELECT * FROM employees").rows.size) + } + + @Test + fun renameColumnPropagatesToOtherTablesForeignKeyReferencedColumns() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE)") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + run("ALTER TABLE departments RENAME COLUMN id TO dept_pk") + run("DELETE FROM departments WHERE dept_pk = 1") + assertTrue(select("SELECT * FROM employees").rows.isEmpty(), "cascade must still work through the renamed referenced column") + } + + @Test + fun renameColumnUpdatesAUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO t VALUES (1, 'a@x.com')") + run("ALTER TABLE t RENAME COLUMN email TO contact_email") + assertFailsWith { run("INSERT INTO t VALUES (2, 'a@x.com')") } + } + + @Test + fun renameColumnUpdatesACheckConstraintAndStillEnforcesIt() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + run("ALTER TABLE t RENAME COLUMN age TO years") + assertFailsWith { run("INSERT INTO t VALUES (1, -5)") } + run("INSERT INTO t VALUES (1, 30)") + assertEquals(listOf(listOf(int(30))), select("SELECT years FROM t").rows) + } + + @Test + fun renameColumnUpdatesAnIndexAndThePlannerStillUsesIt() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("INSERT INTO t VALUES (1, 30), (2, 40)") + run("CREATE INDEX idx_age ON t (age)") + run("ALTER TABLE t RENAME COLUMN age TO years") + val plan = explain("EXPLAIN SELECT id FROM t WHERE years = 30").joinToString("\n") + assertTrue(plan.contains("Index Scan using idx_age"), plan) + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t WHERE years = 30").rows) + } + + // ---- RENAME TABLE ---- + + @Test + fun renameTableRenamesAndPreservesData() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO t VALUES (1, 'ada')") + val result = exec("ALTER TABLE t RENAME TO people") + assertEquals(AlterTableResult("people"), result) + assertEquals(listOf(listOf(int(1), text("ada"))), select("SELECT * FROM people").rows) + assertFailsWith { run("SELECT * FROM t") } + assertEquals(listOf("people"), catalog.tableNames) + } + + @Test + fun renameTableRejectsWhenNewNameAlreadyExistsAsTable() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { run("ALTER TABLE t RENAME TO other") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameTableRejectsWhenNewNameAlreadyExistsAsView() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + run("CREATE VIEW v AS SELECT * FROM other") + val error = assertFailsWith { run("ALTER TABLE t RENAME TO v") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameTableRejectsRenamingAView() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE VIEW v AS SELECT * FROM t") + val error = assertFailsWith { run("ALTER TABLE v RENAME TO w") } + assertTrue(error.message!!.contains("cannot modify view")) + } + + @Test + fun renameTablePropagatesToOtherTablesForeignKeyReference() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE)") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + run("ALTER TABLE departments RENAME TO divisions") + run("INSERT INTO divisions VALUES (2)") + run("INSERT INTO employees VALUES (2, 2)") + assertFailsWith { run("INSERT INTO employees VALUES (3, 99)") } + run("DELETE FROM divisions WHERE id = 1") + assertEquals(listOf(listOf(int(2))), select("SELECT id FROM employees").rows) + } + + @Test + fun renameTableUpdatesItsOwnSelfReferencingForeignKey() { + run( + "CREATE TABLE staff (id INTEGER PRIMARY KEY, manager_id INTEGER REFERENCES staff(id))" + ) + run("INSERT INTO staff VALUES (1, NULL)") + run("ALTER TABLE staff RENAME TO personnel") + run("INSERT INTO personnel VALUES (2, 1)") + assertFailsWith { run("INSERT INTO personnel VALUES (3, 99)") } + assertEquals(2, select("SELECT * FROM personnel").rows.size) + } + + @Test + fun renameTableUpdatesAnIndexAndThePlannerStillUsesIt() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("INSERT INTO t VALUES (1, 30), (2, 40)") + run("CREATE INDEX idx_age ON t (age)") + run("ALTER TABLE t RENAME TO people") + val plan = explain("EXPLAIN SELECT id FROM people WHERE age = 30").joinToString("\n") + assertTrue(plan.contains("Index Scan using idx_age on people"), plan) + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM people WHERE age = 30").rows) + } + + @Test + fun renameTablePropagatesIntoASimpleViewDefinition() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO t VALUES (1, 'ada')") + run("CREATE VIEW v AS SELECT * FROM t") + run("ALTER TABLE t RENAME TO people") + assertEquals(listOf(listOf(int(1), text("ada"))), select("SELECT * FROM v").rows) + } + + @Test + fun renameTablePropagatesIntoAViewJoinAndASubqueryExpression() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1)") + run( + "CREATE VIEW v AS SELECT e.id FROM employees e JOIN departments d ON e.dept_id = d.id " + + "WHERE d.id IN (SELECT id FROM departments)" + ) + run("ALTER TABLE departments RENAME TO divisions") + assertEquals(listOf(listOf(int(1))), select("SELECT * FROM v").rows) + } + + @Test + fun renameTablePropagatesIntoADerivedTableAndAUnion() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1), (2)") + run( + "CREATE VIEW v AS SELECT * FROM (SELECT id FROM t) sub " + + "UNION SELECT id FROM t WHERE id = 2" + ) + run("ALTER TABLE t RENAME TO renamed") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), select("SELECT * FROM v ORDER BY id").rows) + } + + @Test + fun renameTableDoesNotTouchATableSourceShadowedByACteOfTheSameName() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + run("CREATE VIEW v AS WITH t AS (SELECT 99 AS id FROM t) SELECT * FROM t") + run("ALTER TABLE t RENAME TO renamed") + // The CTE named "t" shadows the base table for the rest of that WITH clause, so the view + // must still yield the CTE's own row (99), not read from the (now renamed) base table. + assertEquals(listOf(listOf(int(99))), select("SELECT * FROM v").rows) + assertEquals(listOf(listOf(int(1))), select("SELECT * FROM renamed").rows) + } + + @Test + fun renameTableRenamingToItsOwnNameIsANoOp() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + run("ALTER TABLE t RENAME TO t") + assertEquals(listOf(listOf(int(1))), select("SELECT * FROM t").rows) + } + + // ---- DEFAULT (CREATE TABLE + INSERT) ---- + + @Test + fun insertUsesColumnDefaultWhenColumnOmittedFromColumnList() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t (id) VALUES (1)") + assertEquals(listOf(listOf(text("pending"))), select("SELECT status FROM t").rows) + } + + @Test + fun insertDoesNotUseDefaultWhenExplicitNullIsGiven() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t (id, status) VALUES (1, NULL)") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT status FROM t").rows) + } + + @Test + fun defaultAppliesIndependentlyToEachOmittedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'new', score INTEGER DEFAULT 0)") + run("INSERT INTO t (id) VALUES (1)") + val result = exec("INSERT INTO t (id) VALUES (2)") + assertEquals(InsertResult(1), result) + assertEquals( + listOf(listOf(text("new"), int(0)), listOf(text("new"), int(0))), + select("SELECT status, score FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun createTableRejectsColumnReferenceInDefault() { + val error = assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, twice INTEGER DEFAULT id)") + } + assertTrue(error.message!!.contains("DEFAULT")) + } + + @Test + fun createTableRejectsSubqueryInDefault() { + run("CREATE TABLE other (x INTEGER)") + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER DEFAULT (SELECT COUNT(*) FROM other))") + } + } + + @Test + fun createTableRejectsAggregateInDefault() { + val error = assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER DEFAULT COUNT(*))") + } + assertTrue(error.message!!.contains("aggregate")) + } + + @Test + fun createTableValidatesDefaultTypeEagerly() { + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER DEFAULT 'not a number')") + } + } + + // ---- DEFAULT (UPDATE) ---- + + @Test + fun updateSetToDefaultResetsTheColumnToItsDeclaredDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t VALUES (1, 'shipped')") + val result = exec("UPDATE t SET status = DEFAULT WHERE id = 1") + assertEquals(UpdateResult(1), result) + assertEquals(listOf(listOf(text("pending"))), select("SELECT status FROM t").rows) + } + + @Test + fun updateSetToDefaultFallsBackToNullWhenTheColumnHasNoDeclaredDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, note TEXT)") + run("INSERT INTO t VALUES (1, 'keep me')") + run("UPDATE t SET note = DEFAULT WHERE id = 1") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT note FROM t").rows) + } + + @Test + fun updateSetToDefaultIsEvaluatedFreshPerRowJustLikeInsert() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, score INTEGER DEFAULT 0)") + run("INSERT INTO t VALUES (1, 10), (2, 20), (3, 30)") + val result = exec("UPDATE t SET score = DEFAULT WHERE id IN (1, 2)") + assertEquals(UpdateResult(2), result) + assertEquals( + listOf(listOf(int(1), int(0)), listOf(int(2), int(0)), listOf(int(3), int(30))), + select("SELECT id, score FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun updateCanMixADefaultAssignmentWithARegularAssignmentInTheSameStatement() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, status TEXT DEFAULT 'pending')") + run("INSERT INTO t VALUES (1, 'ada', 'shipped')") + run("UPDATE t SET name = 'grace', status = DEFAULT WHERE id = 1") + assertEquals(listOf(listOf(text("grace"), text("pending"))), select("SELECT name, status FROM t").rows) + } + + @Test + fun updateSetToDefaultFailsNotNullValidationWhenThereIsNoDefaultToFallBackOn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT NOT NULL)") + run("INSERT INTO t VALUES (1, 'shipped')") + val error = assertFailsWith { + run("UPDATE t SET status = DEFAULT WHERE id = 1") + } + assertTrue(error.message!!.contains("status"), error.message!!) + } + + @Test + fun updateSetToDefaultRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t VALUES (1, 'shipped')") + val statement = Parser(Lexer("UPDATE t SET status = DEFAULT WHERE id = 1").tokenize()).parseProgram().single() + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("status = DEFAULT"), rendered) + run(rendered) + assertEquals(listOf(listOf(text("pending"))), select("SELECT status FROM t").rows) + } + + // ---- DEFAULT (INSERT VALUES) ---- + + @Test + fun insertValuesWithLiteralDefaultUsesTheColumnsDeclaredDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + val result = exec("INSERT INTO t VALUES (1, DEFAULT)") + assertEquals(InsertResult(1), result) + assertEquals(listOf(listOf(text("pending"))), select("SELECT status FROM t").rows) + } + + @Test + fun insertValuesWithLiteralDefaultFallsBackToNullWhenTheColumnHasNoDeclaredDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, note TEXT)") + run("INSERT INTO t VALUES (1, DEFAULT)") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT note FROM t").rows) + } + + @Test + fun insertValuesLiteralDefaultIsDistinctFromAnExplicitNullLiteral() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t VALUES (1, NULL)") + assertEquals(listOf(listOf(Value.NullValue)), select("SELECT status FROM t").rows) + } + + @Test + fun insertValuesLiteralDefaultAppliesIndependentlyToEachRowInAMultiRowInsert() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, score INTEGER DEFAULT 0)") + run("INSERT INTO t VALUES (1, DEFAULT), (2, 20), (3, DEFAULT)") + assertEquals( + listOf(listOf(int(1), int(0)), listOf(int(2), int(20)), listOf(int(3), int(0))), + select("SELECT id, score FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun insertValuesLiteralDefaultWorksWithAnExplicitColumnList() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, status TEXT DEFAULT 'pending')") + run("INSERT INTO t (id, status, name) VALUES (1, DEFAULT, 'ada')") + assertEquals(listOf(listOf(text("ada"), text("pending"))), select("SELECT name, status FROM t").rows) + } + + @Test + fun insertValuesLiteralDefaultFailsNotNullValidationWhenThereIsNoDefaultToFallBackOn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT NOT NULL)") + val error = assertFailsWith { + run("INSERT INTO t VALUES (1, DEFAULT)") + } + assertTrue(error.message!!.contains("status"), error.message!!) + } + + @Test + fun insertValuesLiteralDefaultRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + val statement = Parser(Lexer("INSERT INTO t VALUES (1, DEFAULT)").tokenize()).parseProgram().single() + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("DEFAULT"), rendered) + run(rendered) + assertEquals(listOf(listOf(text("pending"))), select("SELECT status FROM t").rows) + } + + // ---- ADD CONSTRAINT / DROP CONSTRAINT ---- + + @Test + fun addConstraintCheckSucceedsWhenEveryExistingRowSatisfiesIt() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("INSERT INTO t VALUES (1, 30), (2, 40)") + val result = exec("ALTER TABLE t ADD CONSTRAINT age_positive CHECK (age > 0)") + assertEquals(AlterTableResult("t"), result) + val error = assertFailsWith { run("INSERT INTO t VALUES (3, -1)") } + assertTrue(error.message!!.contains("age_positive"), error.message!!) + } + + @Test + fun addConstraintCheckRejectsWhenAnExistingRowAlreadyViolatesIt() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("INSERT INTO t VALUES (1, 30), (2, -5)") + val error = assertFailsWith { run("ALTER TABLE t ADD CONSTRAINT age_positive CHECK (age > 0)") } + assertTrue(error.message!!.contains("age_positive"), error.message!!) + // Rejected, so never installed: a fresh negative row must still be free to insert. + run("INSERT INTO t VALUES (3, -1)") + assertEquals(2, select("SELECT id FROM t WHERE age < 0").rows.size) + } + + @Test + fun addConstraintCheckRejectsAReferenceToAnUnknownColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + val error = assertFailsWith { run("ALTER TABLE t ADD CONSTRAINT c1 CHECK (bogus > 0)") } + assertTrue(error.message!!.contains("bogus"), error.message!!) + } + + @Test + fun addConstraintCheckNameRoundTripsThroughSqlWriterDumpAndReplay() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + run("ALTER TABLE t ADD CONSTRAINT age_positive CHECK (age > 0)") + val dumped = SqlWriter.dump(catalog) + assertTrue(dumped.contains("CONSTRAINT age_positive CHECK"), dumped) + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + for (statement in Parser(Lexer(dumped).tokenize()).parseProgram()) freshExecutor.execute(statement) + val error = assertFailsWith { + freshExecutor.execute(Parser(Lexer("INSERT INTO t VALUES (5, -1)").tokenize()).parseProgram().single()) + } + assertTrue(error.message!!.contains("age_positive"), error.message!!) + } + + @Test + fun addConstraintUniqueSucceedsWhenNoExistingRowsAreDuplicates() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT)") + run("INSERT INTO t VALUES (1, 'a@x.com'), (2, 'b@x.com')") + exec("ALTER TABLE t ADD CONSTRAINT uq_email UNIQUE (email)") + run("INSERT INTO t VALUES (3, 'c@x.com')") + val error = assertFailsWith { run("INSERT INTO t VALUES (4, 'c@x.com')") } + assertTrue(error.message!!.contains("uq_email"), error.message!!) + } + + @Test + fun addConstraintUniqueRejectsWhenExistingRowsAreAlreadyDuplicated() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT)") + run("INSERT INTO t VALUES (1, 'a@x.com'), (2, 'a@x.com')") + val error = assertFailsWith { run("ALTER TABLE t ADD CONSTRAINT uq_email UNIQUE (email)") } + assertTrue(error.message!!.contains("uq_email"), error.message!!) + } + + @Test + fun addConstraintUniqueIgnoresNullValuesAmongExistingRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT)") + run("INSERT INTO t VALUES (1, NULL), (2, NULL)") + exec("ALTER TABLE t ADD CONSTRAINT uq_email UNIQUE (email)") + run("INSERT INTO t VALUES (3, NULL)") + assertEquals(3, select("SELECT id FROM t").rows.size) + } + + @Test + fun addConstraintUniqueRejectsAReferenceToAnUnknownColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { run("ALTER TABLE t ADD CONSTRAINT uq_bogus UNIQUE (bogus)") } + assertTrue(error.message!!.contains("bogus"), error.message!!) + } + + @Test + fun addConstraintForeignKeySucceedsWhenEveryExistingRowHasAMatchOrIsNull() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1), (2, NULL)") + exec("ALTER TABLE employees ADD CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id)") + val error = assertFailsWith { run("INSERT INTO employees VALUES (3, 99)") } + assertTrue(error.message!!.contains("FOREIGN KEY constraint failed"), error.message!!) + } + + @Test + fun addConstraintForeignKeyRejectsWhenAnExistingRowIsAlreadyOrphaned() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1), (2, 99)") + val error = assertFailsWith { + run("ALTER TABLE employees ADD CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id)") + } + assertTrue(error.message!!.contains("FOREIGN KEY constraint failed"), error.message!!) + } + + @Test + fun addConstraintForeignKeySelfReferenceValidatesAgainstExistingSiblingRows() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, manager_id INTEGER)") + run("INSERT INTO employees VALUES (1, NULL), (2, 1), (3, 99)") + assertFailsWith { + run("ALTER TABLE employees ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES employees (id)") + } + run("UPDATE employees SET manager_id = 1 WHERE id = 3") + exec("ALTER TABLE employees ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES employees (id)") + val error = assertFailsWith { run("INSERT INTO employees VALUES (4, 100)") } + assertTrue(error.message!!.contains("FOREIGN KEY constraint failed"), error.message!!) + } + + @Test + fun addConstraintForeignKeyRejectsWhenReferencedColumnsAreNotEnforcedUnique() { + run("CREATE TABLE departments (id INTEGER, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + val error = assertFailsWith { + run("ALTER TABLE employees ADD CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id)") + } + assertTrue(error.message!!.contains("PRIMARY KEY or UNIQUE"), error.message!!) + } + + @Test + fun addConstraintForeignKeyRejectsAColumnCountMismatch() { + // Caught by the parser itself (parseInlineForeignKey, shared with CREATE TABLE's own + // table-level FOREIGN KEY clause) before Executor.executeAddForeignKeyConstraint's own + // redundant arity check would even run - same as CREATE TABLE's identical redundancy. + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + val error = assertFailsWith { + run("ALTER TABLE employees ADD CONSTRAINT fk_bad FOREIGN KEY (dept_id) REFERENCES departments (id, name)") + } + assertTrue(error.message!!.contains("column(s)"), error.message!!) + } + + @Test + fun addConstraintForeignKeyOnDeleteCascadeAppliesAfterBeingAdded() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1), (2, 1)") + exec("ALTER TABLE employees ADD CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id) ON DELETE CASCADE") + run("DELETE FROM departments WHERE id = 1") + assertEquals(emptyList(), select("SELECT id FROM employees").rows) + } + + @Test + fun addConstraintRejectsANameAlreadyUsedByADifferentKindOfConstraintOnTheSameTable() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER, email TEXT)") + run("ALTER TABLE t ADD CONSTRAINT c1 CHECK (age > 0)") + val error = assertFailsWith { run("ALTER TABLE t ADD CONSTRAINT c1 UNIQUE (email)") } + assertTrue(error.message!!.contains("c1"), error.message!!) + } + + @Test + fun createTableAcceptsANamedTableLevelCheckConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER, CONSTRAINT chk_age CHECK (age > 0))") + val error = assertFailsWith { run("INSERT INTO t VALUES (1, -1)") } + assertTrue(error.message!!.contains("chk_age"), error.message!!) + } + + @Test + fun createTableAcceptsANamedTableLevelUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT, CONSTRAINT uq_email UNIQUE (email))") + run("INSERT INTO t VALUES (1, 'a@x.com')") + val error = assertFailsWith { run("INSERT INTO t VALUES (2, 'a@x.com')") } + assertTrue(error.message!!.contains("uq_email"), error.message!!) + } + + @Test + fun createTableAcceptsANamedTableLevelForeignKeyConstraint() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, " + + "CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id))" + ) + run("INSERT INTO departments VALUES (1)") + val error = assertFailsWith { run("INSERT INTO employees VALUES (1, 99)") } + assertTrue(error.message!!.contains("FOREIGN KEY constraint failed"), error.message!!) + } + + @Test + fun dropConstraintRemovesANamedCheckConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER, CONSTRAINT chk_age CHECK (age > 0))") + exec("ALTER TABLE t DROP CONSTRAINT chk_age") + run("INSERT INTO t VALUES (1, -5)") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun dropConstraintRemovesANamedUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT, CONSTRAINT uq_email UNIQUE (email))") + run("INSERT INTO t VALUES (1, 'a@x.com')") + exec("ALTER TABLE t DROP CONSTRAINT uq_email") + run("INSERT INTO t VALUES (2, 'a@x.com')") + assertEquals(2, select("SELECT id FROM t").rows.size) + } + + @Test + fun dropConstraintRemovesANamedForeignKeyConstraint() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, " + + "CONSTRAINT fk_dept FOREIGN KEY (dept_id) REFERENCES departments (id))" + ) + exec("ALTER TABLE employees DROP CONSTRAINT fk_dept") + run("INSERT INTO employees VALUES (1, 99)") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM employees").rows) + } + + @Test + fun dropConstraintFailsForAnUnknownName() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { run("ALTER TABLE t DROP CONSTRAINT nope") } + assertTrue(error.message!!.contains("no such constraint"), error.message!!) + } + + @Test + fun dropConstraintRestrictsWhenAnotherTablesForeignKeyDependsOnTheUniqueConstraint() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT, CONSTRAINT uq_code UNIQUE (code))") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT, " + + "CONSTRAINT fk_dept_code FOREIGN KEY (dept_code) REFERENCES departments (code))" + ) + val error = assertFailsWith { run("ALTER TABLE departments DROP CONSTRAINT uq_code") } + assertTrue(error.message!!.contains("employees"), error.message!!) + } + + @Test + fun dropConstraintCascadeRemovesTheDependentForeignKeyOnAnotherTable() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT, CONSTRAINT uq_code UNIQUE (code))") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT, " + + "CONSTRAINT fk_dept_code FOREIGN KEY (dept_code) REFERENCES departments (code))" + ) + exec("ALTER TABLE departments DROP CONSTRAINT uq_code CASCADE") + run("INSERT INTO employees VALUES (1, 'nonexistent')") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM employees").rows) + } + + @Test + fun dropConstraintRestrictsASelfReferencingForeignKeyDependentOnItsOwnUniqueConstraint() { + run( + "CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT, parent_code TEXT, " + + "CONSTRAINT uq_code UNIQUE (code), CONSTRAINT fk_parent FOREIGN KEY (parent_code) REFERENCES t (code))" + ) + val error = assertFailsWith { run("ALTER TABLE t DROP CONSTRAINT uq_code") } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message!!) + } + + @Test + fun dropConstraintCascadeRemovesASelfReferencingForeignKeyThatDependedOnTheDroppedUniqueConstraint() { + run( + "CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT, parent_code TEXT, " + + "CONSTRAINT uq_code UNIQUE (code), CONSTRAINT fk_parent FOREIGN KEY (parent_code) REFERENCES t (code))" + ) + exec("ALTER TABLE t DROP CONSTRAINT uq_code CASCADE") + run("INSERT INTO t VALUES (1, 'a', 'nonexistent')") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun dropConstraintAllowsDroppingAUniqueConstraintWhenAnotherRemainingGroupStillCoversTheSameColumns() { + run( + "CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT, " + + "CONSTRAINT uq_code_a UNIQUE (code), CONSTRAINT uq_code_b UNIQUE (code))" + ) + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT, " + + "CONSTRAINT fk_dept_code FOREIGN KEY (dept_code) REFERENCES departments (code))" + ) + exec("ALTER TABLE departments DROP CONSTRAINT uq_code_a") + val error = assertFailsWith { run("INSERT INTO employees VALUES (1, 'nonexistent')") } + assertTrue(error.message!!.contains("FOREIGN KEY constraint failed"), error.message!!) + } + + // ---- ALTER COLUMN TYPE ---- + + @Test + fun changeColumnTypeWideningIntegerToRealCoercesExistingRowsWithNoExplicitUsing() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, score INTEGER)") + run("INSERT INTO t VALUES (1, 10), (2, 20)") + val result = exec("ALTER TABLE t ALTER COLUMN score TYPE REAL") + assertEquals(AlterTableResult("t"), result) + assertEquals(listOf(listOf(real(10.0)), listOf(real(20.0))), select("SELECT score FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeWideningIntegerToDecimalCoercesExistingRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO t VALUES (1, 42)") + exec("ALTER TABLE t ALTER COLUMN amount TYPE DECIMAL(10, 2)") + assertEquals(listOf(listOf(decimal("42.00"))), select("SELECT amount FROM t").rows) + } + + @Test + fun changeColumnTypeWithUsingConvertsTextToIntegerViaCast() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT)") + run("INSERT INTO t VALUES (1, '100'), (2, '200')") + exec("ALTER TABLE t ALTER COLUMN code TYPE INTEGER USING CAST(code AS INTEGER)") + assertEquals(listOf(listOf(int(100)), listOf(int(200))), select("SELECT code FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeWithUsingCanReferenceTheRowsOtherColumns() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, price INTEGER, quantity INTEGER)") + run("INSERT INTO t VALUES (1, 10, 3)") + exec("ALTER TABLE t ALTER COLUMN price TYPE INTEGER USING price * quantity") + assertEquals(listOf(listOf(int(30))), select("SELECT price FROM t").rows) + } + + @Test + fun changeColumnTypeRejectsImplicitCoercionWhenTypesAreIncompatibleAndNoUsingIsGiven() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT)") + run("INSERT INTO t VALUES (1, 'abc')") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN code TYPE INTEGER") } + assertTrue(error.message!!.contains("type mismatch"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsUnknownColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN missing TYPE TEXT") } + assertTrue(error.message!!.contains("no such column"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsGeneratedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, price INTEGER, doubled INTEGER GENERATED ALWAYS AS (price * 2) STORED)") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN doubled TYPE REAL") } + assertTrue(error.message!!.contains("GENERATED"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsPrimaryKeyColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN id TYPE TEXT") } + assertTrue(error.message!!.contains("PRIMARY KEY"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnUsedByItsOwnForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, FOREIGN KEY (dept_id) REFERENCES departments (id))") + val error = assertFailsWith { run("ALTER TABLE employees ALTER COLUMN dept_id TYPE TEXT") } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnReferencedByAnotherTablesForeignKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT, CONSTRAINT uq_code UNIQUE (code))") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT, " + + "FOREIGN KEY (dept_code) REFERENCES departments (code))" + ) + val error = assertFailsWith { run("ALTER TABLE departments ALTER COLUMN code TYPE INTEGER") } + assertTrue(error.message!!.contains("employees"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnUsedByAUniqueConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT, CONSTRAINT uq_email UNIQUE (email))") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN email TYPE TEXT") } + assertTrue(error.message!!.contains("UNIQUE"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnWithASecondaryIndex() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE INDEX idx_name ON t (name)") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN name TYPE TEXT") } + assertTrue(error.message!!.contains("idx_name"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsWhenUsingProducesNullForANotNullColumnAndLeavesTheTableUnchanged() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT NOT NULL)") + run("INSERT INTO t VALUES (1, '100'), (2, '')") + val error = assertFailsWith { + run("ALTER TABLE t ALTER COLUMN code TYPE INTEGER USING CASE WHEN code = '' THEN NULL ELSE CAST(code AS INTEGER) END") + } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + assertEquals(listOf(listOf(text("100")), listOf(text(""))), select("SELECT code FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeRevalidatesACheckConstraintAgainstTheNewValuesAndLeavesTheTableUnchangedOnFailure() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER, CHECK (amount <= 100))") + run("INSERT INTO t VALUES (1, 10), (2, 60)") + val error = assertFailsWith { + run("ALTER TABLE t ALTER COLUMN amount TYPE INTEGER USING amount * 2") + } + assertTrue(error.message!!.contains("CHECK constraint"), error.message!!) + assertEquals( + listOf(listOf(int(1), int(10)), listOf(int(2), int(60))), + select("SELECT id, amount FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun changeColumnTypeSucceedsAndAppliesUsingToEveryRowWhenAllSatisfyAnExistingCheckConstraint() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER, CHECK (amount <= 100))") + run("INSERT INTO t VALUES (1, 10), (2, 20)") + exec("ALTER TABLE t ALTER COLUMN amount TYPE INTEGER USING amount * 2") + assertEquals(listOf(listOf(int(20)), listOf(int(40))), select("SELECT amount FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeRejectsAnIncompatibleDefaultExpressionUpFront() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + val error = assertFailsWith { run("ALTER TABLE t ALTER COLUMN status TYPE INTEGER USING 0") } + assertTrue(error.message!!.contains("DEFAULT"), error.message!!) + } + + @Test + fun changeColumnTypePreservesNotNullAndStillEnforcesItGoingForward() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER NOT NULL)") + run("INSERT INTO t VALUES (1, 10)") + exec("ALTER TABLE t ALTER COLUMN amount TYPE REAL") + val error = assertFailsWith { run("INSERT INTO t VALUES (2, NULL)") } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + } + + @Test + fun changeColumnTypeAcceptsAnOptionalColumnKeyword() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO t VALUES (1, 5)") + exec("ALTER TABLE t ALTER amount TYPE REAL") + assertEquals(listOf(listOf(real(5.0))), select("SELECT amount FROM t").rows) + } + + @Test + fun changeColumnTypeDumpReflectsTheNewDeclaredType() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO t VALUES (1, 5)") + exec("ALTER TABLE t ALTER COLUMN amount TYPE DECIMAL(10, 2)") + val dumped = SqlWriter.dump(catalog) + assertTrue(dumped.contains("amount DECIMAL(10, 2)"), dumped) + } + + @Test + fun changeColumnTypeRedoLogTextRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO t VALUES (1, 5)") + val statement = Parser(Lexer("ALTER TABLE t ALTER COLUMN amount TYPE REAL USING amount").tokenize()).parseProgram().single() + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ALTER COLUMN amount TYPE REAL USING amount"), rendered) + run(rendered) + assertEquals(listOf(listOf(real(5.0))), select("SELECT amount FROM t").rows) + } +} diff --git a/src/test/kotlin/kestrel/CheckConstraintTest.kt b/src/test/kotlin/kestrel/CheckConstraintTest.kt new file mode 100644 index 0000000..2198e92 --- /dev/null +++ b/src/test/kotlin/kestrel/CheckConstraintTest.kt @@ -0,0 +1,111 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class CheckConstraintTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + @Test + fun columnLevelCheckAcceptsSatisfyingRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + assertEquals(InsertResult(1), exec("INSERT INTO t VALUES (1, 30)")) + } + + @Test + fun columnLevelCheckRejectsViolatingRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + val error = assertFailsWith { run("INSERT INTO t VALUES (1, -5)") } + assertTrue(error.message!!.contains("CHECK")) + } + + @Test + fun checkPassesWhenExpressionIsUnknownBecauseOfNull() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + // three-valued logic: NULL > 0 is UNKNOWN, not FALSE, so the row is accepted + assertEquals(InsertResult(1), exec("INSERT INTO t VALUES (1, NULL)")) + } + + @Test + fun tableLevelCheckCanSpanMultipleColumns() { + run("CREATE TABLE ranges (id INTEGER PRIMARY KEY, low INTEGER, high INTEGER, CHECK (low <= high))") + assertEquals(InsertResult(1), exec("INSERT INTO ranges VALUES (1, 1, 10)")) + assertFailsWith { run("INSERT INTO ranges VALUES (2, 10, 1)") } + } + + @Test + fun multipleCheckConstraintsAllMustBeSatisfied() { + run( + "CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age >= 0), " + + "score INTEGER CHECK (score BETWEEN 0 AND 100))" + ) + assertEquals(InsertResult(1), exec("INSERT INTO t VALUES (1, 30, 90)")) + assertFailsWith { run("INSERT INTO t VALUES (2, 30, 150)") } + assertFailsWith { run("INSERT INTO t VALUES (3, -1, 90)") } + } + + @Test + fun checkIsEnforcedOnUpdateToo() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + run("INSERT INTO t VALUES (1, 30)") + assertFailsWith { run("UPDATE t SET age = -1 WHERE id = 1") } + assertEquals(UpdateResult(1), exec("UPDATE t SET age = 40 WHERE id = 1")) + } + + @Test + fun checkRejectsAggregateFunctions() { + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER CHECK (SUM(amount) > 0))") + } + } + + @Test + fun checkRejectsSubqueries() { + run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY CHECK (id IN (SELECT id FROM other)))") + } + } + + @Test + fun checkRejectsUnknownColumn() { + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, CHECK (nope > 0))") + } + } + + @Test + fun checkCanReferenceStringAndCallExpressions() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT CHECK (LENGTH(name) > 0))") + assertEquals(InsertResult(1), exec("INSERT INTO t VALUES (1, 'ada')")) + assertFailsWith { run("INSERT INTO t VALUES (2, '')") } + } + + @Test + fun checkConstraintSurvivesRollback() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + run("BEGIN") + run("INSERT INTO t VALUES (1, 30)") + run("ROLLBACK") + assertFailsWith { run("INSERT INTO t VALUES (1, -5)") } + assertEquals(0, select("SELECT * FROM t").rows.size) + } +} diff --git a/src/test/kotlin/kestrel/CollateTest.kt b/src/test/kotlin/kestrel/CollateTest.kt new file mode 100644 index 0000000..232eff4 --- /dev/null +++ b/src/test/kotlin/kestrel/CollateTest.kt @@ -0,0 +1,184 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * `expr COLLATE BINARY|NOCASE|RTRIM` (SQL:1999), the explicit, expression-level form - see + * [kestrel.ColumnCollationTest] for a column's own declared `COLLATE` (`CREATE TABLE ... col TYPE + * COLLATE name`), which only ever changes what a *bare* column reference compares with and is + * always overridden by an explicit annotation like the one covered here. Covers every comparison + * site [kestrel.engine.Evaluator] threads a resolved [kestrel.storage.Collation] through (`=`/`<>`/ + * `<`/`<=`/`>`/`>=`, `IN`, `BETWEEN`, a searched-`CASE`'s operand match) plus every sort that shares + * [kestrel.engine.compareOrderKeyValues] (top-level ORDER BY, a window's `OVER (... ORDER BY + * ...)`), and a persistence round-trip through [SqlWriter]. [kestrel.store.sql.DiskExecutorCollateTest] + * mirrors the comparison and ORDER BY cases against the disk engine, proving the shared + * [kestrel.engine.Evaluator]/[kestrel.engine.compareOrderKeyValues] code is genuinely shared, not + * reimplemented per engine. + */ +class CollateTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun parseOne(sql: String): Statement = Parser(Lexer(sql).tokenize()).parseProgram().single() + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private fun bool(value: Boolean) = Value.BooleanValue(value) + + private fun seedNames() { + run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO people VALUES (1, 'Bob'), (2, 'bob'), (3, 'Alice'), (4, 'alice '), (5, 'Zoe')") + } + + @Test + fun equalityWithNocaseIgnoresAsciiCaseButPlainEqualityDoesNot() { + seedNames() + assertEquals(bool(true), select("SELECT name = 'BOB' COLLATE NOCASE FROM people WHERE id = 1").rows.single().single()) + assertEquals(bool(false), select("SELECT name = 'BOB' FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun explicitCollateOnTheLeftOperandTakesPrecedenceOverTheRight() { + seedNames() + assertEquals( + bool(true), + select("SELECT name COLLATE NOCASE = 'BOB' COLLATE BINARY FROM people WHERE id = 1").rows.single().single() + ) + } + + @Test + fun explicitCollateOnTheRightOperandAppliesWhenTheLeftHasNone() { + seedNames() + assertEquals(bool(true), select("SELECT 'BOB' = name COLLATE NOCASE FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun rtrimIgnoresOnlyTrailingSpacesNotLeadingOrInteriorOnes() { + seedNames() + assertEquals( + bool(true), + select("SELECT name = 'alice' COLLATE RTRIM FROM people WHERE id = 4").rows.single().single() + ) + assertEquals( + bool(false), + select("SELECT name = ' alice' COLLATE RTRIM FROM people WHERE id = 4").rows.single().single() + ) + } + + @Test + fun lessThanAndGreaterThanHonorNocase() { + seedNames() + assertEquals(bool(false), select("SELECT 'APPLE' COLLATE NOCASE < 'apple' FROM people WHERE id = 1").rows.single().single()) + assertEquals(bool(true), select("SELECT 'APPLE' COLLATE NOCASE < 'banana' FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun orderByCollateNocaseInterleavesUpperAndLowerCaseNamesByAsciiFoldedOrder() { + seedNames() + val result = select("SELECT name FROM people ORDER BY name COLLATE NOCASE, id") + assertEquals( + listOf("Alice", "alice ", "Bob", "bob", "Zoe").map { listOf(text(it)) }, + result.rows + ) + } + + @Test + fun plainOrderByWithoutCollateStaysAsciiCodepointOrder() { + seedNames() + val result = select("SELECT name FROM people ORDER BY name") + assertEquals( + listOf("Alice", "Bob", "Zoe", "alice ", "bob").map { listOf(text(it)) }, + result.rows + ) + } + + @Test + fun windowFunctionOverOrderByHonorsAnExplicitCollateClause() { + seedNames() + val result = select( + "SELECT id, ROW_NUMBER() OVER (ORDER BY name COLLATE NOCASE) AS rn FROM people ORDER BY id" + ) + assertEquals( + listOf( + listOf(int(1), int(3)), listOf(int(2), int(4)), listOf(int(3), int(1)), + listOf(int(4), int(2)), listOf(int(5), int(5)) + ), + result.rows + ) + } + + @Test + fun inListHonorsTheOperandsExplicitCollate() { + seedNames() + assertEquals( + bool(true), + select("SELECT name COLLATE NOCASE IN ('BOB', 'other') FROM people WHERE id = 1").rows.single().single() + ) + assertEquals( + bool(false), + select("SELECT name IN ('BOB', 'other') FROM people WHERE id = 1").rows.single().single() + ) + } + + @Test + fun betweenHonorsCollateIndependentlyOnEachSide() { + run("CREATE TABLE words (id INTEGER PRIMARY KEY, w TEXT)") + run("INSERT INTO words VALUES (1, 'M')") + assertEquals( + bool(true), + select("SELECT w COLLATE NOCASE BETWEEN 'a' AND 'z' FROM words WHERE id = 1").rows.single().single() + ) + assertEquals( + bool(false), + select("SELECT w BETWEEN 'a' AND 'z' FROM words WHERE id = 1").rows.single().single() + ) + } + + @Test + fun searchedCaseOperandMatchHonorsCollate() { + run("CREATE TABLE words (id INTEGER PRIMARY KEY, w TEXT)") + run("INSERT INTO words VALUES (1, 'YES')") + assertEquals( + listOf(listOf(text("matched"))), + select("SELECT CASE w COLLATE NOCASE WHEN 'yes' THEN 'matched' ELSE 'no' END FROM words").rows + ) + assertEquals( + listOf(listOf(text("no"))), + select("SELECT CASE w WHEN 'yes' THEN 'matched' ELSE 'no' END FROM words").rows + ) + } + + @Test + fun collateOnANonTextValueIsAcceptedButHasNoEffect() { + run("CREATE TABLE nums (id INTEGER PRIMARY KEY, n INTEGER)") + run("INSERT INTO nums VALUES (1, 5)") + assertEquals(bool(true), select("SELECT n COLLATE NOCASE = 5 FROM nums").rows.single().single()) + } + + @Test + fun collateExpressionInAWhereClauseRoundTripsThroughSqlWriterAndKeepsItsComparisonBehavior() { + seedNames() + val statement = parseOne("UPDATE people SET name = 'MATCHED' WHERE name COLLATE NOCASE = 'BOB'") + val rendered = SqlWriter.write(statement) + executor.execute(parseOne(rendered)) + assertEquals( + listOf(listOf(int(1), text("MATCHED")), listOf(int(2), text("MATCHED"))), + select("SELECT id, name FROM people WHERE name = 'MATCHED' ORDER BY id").rows + ) + } +} diff --git a/src/test/kotlin/kestrel/ColumnCollationTest.kt b/src/test/kotlin/kestrel/ColumnCollationTest.kt new file mode 100644 index 0000000..96355b8 --- /dev/null +++ b/src/test/kotlin/kestrel/ColumnCollationTest.kt @@ -0,0 +1,639 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `col TYPE COLLATE name` - a column's own declared collating sequence (see + * [kestrel.catalog.Column.collation]), which a *bare* reference to it compares with whenever the + * query itself writes no explicit `COLLATE` - see [kestrel.CollateTest] for that explicit, + * expression-level form, which always overrides this one. Threaded through + * [kestrel.engine.ColumnResolver.declaredCollation] into every resolver built directly over a + * table's own schema: plain `WHERE`/`SELECT`/`HAVING` (via [kestrel.engine.SelectRunner]'s + * per-source lookup, including across a JOIN), `INSERT ... ON CONFLICT DO UPDATE` (EXCLUDED-aware), + * `MERGE`'s `ON`/`WHEN` clauses, and a trigger's `NEW`/`OLD`. [kestrel.store.sql. + * DiskExecutorColumnCollationTest] mirrors the core cases against the disk engine, including a + * real file close/reopen (the declared collation is persisted in [kestrel.store.sql. + * DirectoryEntryCodec], not just this engine's text dump). + */ +class ColumnCollationTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun parseOne(sql: String): Statement = Parser(Lexer(sql).tokenize()).parseProgram().single() + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private fun bool(value: Boolean) = Value.BooleanValue(value) + + private fun seedNames() { + run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO people VALUES (1, 'Bob'), (2, 'bob'), (3, 'Alice'), (4, 'Zoe')") + } + + @Test + fun bareEqualityHonorsTheColumnsDeclaredCollationWithoutAnExplicitClause() { + seedNames() + assertEquals(bool(true), select("SELECT name = 'BOB' FROM people WHERE id = 1").rows.single().single()) + assertEquals( + listOf(int(1), int(2)), + select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows.map { it.single() } + ) + } + + @Test + fun explicitCollateInTheQueryOverridesTheColumnsDeclaredCollation() { + seedNames() + assertEquals(bool(false), select("SELECT name COLLATE BINARY = 'BOB' FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun explicitCollateOnTheOtherOperandStillOverridesADeclaredCollationOnThisOne() { + seedNames() + // 'BOB' COLLATE BINARY on the right wins over people.name's own declared NOCASE on the left. + assertEquals(bool(false), select("SELECT name = 'BOB' COLLATE BINARY FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun columnWithNoDeclaredCollationStaysBinaryByDefault() { + run("CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO plain VALUES (1, 'Bob')") + assertEquals(bool(false), select("SELECT name = 'BOB' FROM plain WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInAnInList() { + seedNames() + assertEquals(bool(true), select("SELECT name IN ('BOB', 'other') FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInBetween() { + run("CREATE TABLE words (id INTEGER PRIMARY KEY, w TEXT COLLATE NOCASE)") + run("INSERT INTO words VALUES (1, 'm')") + assertEquals(bool(true), select("SELECT w BETWEEN 'A' AND 'Z' FROM words").rows.single().single()) + } + + @Test + fun declaredCollationAppliesToASearchedCaseOperandMatch() { + run("CREATE TABLE words (id INTEGER PRIMARY KEY, w TEXT COLLATE NOCASE)") + run("INSERT INTO words VALUES (1, 'YES')") + assertEquals( + listOf(listOf(text("matched"))), + select("SELECT CASE w WHEN 'yes' THEN 'matched' ELSE 'no' END FROM words").rows + ) + } + + @Test + fun declaredCollationAppliesToDefaultOrderBySortOrder() { + seedNames() + // No explicit `ORDER BY name COLLATE NOCASE` needed - the column's own declared NOCASE + // already interleaves case. 'Bob' (id 1) and 'bob' (id 2) compare equal under NOCASE, so + // the sort (stable) keeps them in their original scan-order relative position. + val result = select("SELECT name FROM people ORDER BY name") + assertEquals(listOf("Alice", "Bob", "bob", "Zoe").map { listOf(text(it)) }, result.rows) + } + + @Test + fun explicitCollateInOrderByOverridesTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name FROM people ORDER BY name COLLATE BINARY") + assertEquals(listOf("Alice", "Bob", "Zoe", "bob").map { listOf(text(it)) }, result.rows) + } + + @Test + fun declaredCollationAppliesInAWindowFunctionsOrderBy() { + seedNames() + val result = select("SELECT name, ROW_NUMBER() OVER (ORDER BY name) AS rn FROM people ORDER BY rn") + assertEquals( + listOf("Alice" to 1L, "Bob" to 2L, "bob" to 3L, "Zoe" to 4L).map { (name, rn) -> listOf(text(name), int(rn)) }, + result.rows + ) + } + + @Test + fun declaredCollationAppliesInGroupBysOwnOrderBy() { + seedNames() + // GROUP BY itself now also dedups through name's declared NOCASE (closing the scope cut this + // test used to document), so 'Bob' and 'bob' collapse into a single group - the group keeps + // whichever row it saw first in scan order ('Bob', id 1). Only three groups remain, and the + // ordering of those groups (GroupAggregation's own ORDER BY handling) is still under test. + val result = select("SELECT name FROM people GROUP BY name ORDER BY name") + assertEquals(listOf("Alice", "Bob", "Zoe").map { listOf(text(it)) }, result.rows) + } + + @Test + fun groupByDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name, COUNT(*) FROM people GROUP BY name ORDER BY name") + assertEquals( + listOf("Alice" to 1L, "Bob" to 2L, "Zoe" to 1L).map { (name, count) -> listOf(text(name), int(count)) }, + result.rows + ) + } + + @Test + fun groupByWithNoDeclaredCollationStillDedupsByBinaryEquality() { + run("CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO plain VALUES (1, 'Bob'), (2, 'bob')") + val result = select("SELECT name, COUNT(*) FROM plain GROUP BY name ORDER BY name") + assertEquals(listOf("Bob" to 1L, "bob" to 1L).map { (name, count) -> listOf(text(name), int(count)) }, result.rows) + } + + @Test + fun explicitGroupByCollateOverridesTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name COLLATE BINARY, COUNT(*) FROM people GROUP BY name COLLATE BINARY ORDER BY name COLLATE BINARY") + assertEquals( + listOf("Alice" to 1L, "Bob" to 1L, "Zoe" to 1L, "bob" to 1L).map { (name, count) -> listOf(text(name), int(count)) }, + result.rows + ) + } + + @Test + fun selectDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT DISTINCT name FROM people ORDER BY name") + assertEquals(listOf("Alice", "Bob", "Zoe").map { listOf(text(it)) }, result.rows) + } + + @Test + fun selectDistinctStarDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + // A single-column table so DISTINCT *'s whole-row key is driven entirely by name's own + // declared NOCASE - with more columns involved, any other differing column (like people's + // own id) would already make every row distinct regardless of collation. + run("CREATE TABLE tags (label TEXT COLLATE NOCASE)") + run("INSERT INTO tags VALUES ('Bob'), ('bob'), ('Alice')") + val result = select("SELECT DISTINCT * FROM tags ORDER BY label") + assertEquals(listOf("Alice", "Bob").map { listOf(text(it)) }, result.rows) + } + + @Test + fun selectDistinctWithNoDeclaredCollationStaysBinary() { + run("CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO plain VALUES (1, 'Bob'), (2, 'bob')") + val result = select("SELECT DISTINCT name FROM plain ORDER BY name") + assertEquals(listOf("Bob", "bob").map { listOf(text(it)) }, result.rows) + } + + @Test + fun selectDistinctOnDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, hits INTEGER)") + run("INSERT INTO people VALUES (1, 'Bob', 10), (2, 'bob', 20), (3, 'Alice', 5)") + val result = select("SELECT DISTINCT ON (name) name, hits FROM people ORDER BY name, id") + assertEquals( + listOf("Alice" to 5L, "Bob" to 10L).map { (name, hits) -> listOf(text(name), int(hits)) }, + result.rows + ) + } + + @Test + fun countDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + assertEquals(int(3), select("SELECT COUNT(DISTINCT name) FROM people").rows.single().single()) + } + + /** + * `<>` forces [kestrel.engine.SelectRunner]'s nested-loop join path (only a `BinaryExpression` + * `=` condition can hash-join at all), which reuses the same collation-aware + * [kestrel.engine.Evaluator] as `WHERE`. See the `...InAnEqualityHashJoin` tests below for the + * `=` counterpart, which takes the hash-join fast path instead. + */ + @Test + fun joiningOnColumnsWhereOnlyTheLeftSideDeclaresCollationHonorsItInANonEqualityJoinCondition() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO left_t VALUES (1, 'Bob'), (2, 'Zoe')") + run("INSERT INTO right_t VALUES (1, 'BOB')") + assertEquals( + listOf(listOf(int(2))), + select("SELECT left_t.id FROM left_t JOIN right_t ON left_t.name <> right_t.name").rows + ) + } + + @Test + fun joiningOnColumnsWhereOnlyTheRightSideDeclaresCollationFallsBackToTheRightWhenTheLeftHasNone() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO left_t VALUES (1, 'Bob'), (2, 'Zoe')") + run("INSERT INTO right_t VALUES (1, 'BOB')") + assertEquals( + listOf(listOf(int(2))), + select("SELECT left_t.id FROM left_t JOIN right_t ON left_t.name <> right_t.name").rows + ) + } + + /** + * `=` takes [kestrel.engine.SelectRunner]'s hash-join fast path (`hashJoinKeys`/`executeJoin`), + * which must bucket [Value.TextValue] keys through the exact same [kestrel.engine.pairCollation] + * the nested-loop `<>` tests above resolve through [kestrel.engine.Evaluator] - otherwise the + * "optimization" would silently change which rows match. `right_t`'s undeclared `name` still + * only matches its own literal case ('bob'), so this also proves the fast path is not simply + * ignoring case altogether. + */ + @Test + fun joiningOnColumnsWhereOnlyTheLeftSideDeclaresCollationHonorsItInAnEqualityHashJoin() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO left_t VALUES (1, 'Bob'), (2, 'Zoe')") + run("INSERT INTO right_t VALUES (1, 'BOB'), (2, 'bob')") + assertEquals( + listOf(listOf(int(1), int(1)), listOf(int(1), int(2))), + select("SELECT left_t.id, right_t.id FROM left_t JOIN right_t ON left_t.name = right_t.name ORDER BY right_t.id").rows + ) + } + + @Test + fun joiningOnColumnsWhereOnlyTheRightSideDeclaresCollationHonorsItInAnEqualityHashJoin() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO left_t VALUES (1, 'BOB'), (2, 'Zoe')") + run("INSERT INTO right_t VALUES (1, 'Bob')") + assertEquals( + listOf(listOf(int(1))), + select("SELECT left_t.id FROM left_t JOIN right_t ON left_t.name = right_t.name").rows + ) + } + + @Test + fun explicitCollateOnAnEqualityHashJoinConditionOverridesTheColumnsDeclaredCollation() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO left_t VALUES (1, 'Bob')") + run("INSERT INTO right_t VALUES (1, 'BOB')") + assertEquals( + emptyList(), + select("SELECT left_t.id FROM left_t JOIN right_t ON left_t.name COLLATE BINARY = right_t.name").rows + ) + } + + @Test + fun equalityHashJoinWithNoDeclaredCollationOnEitherSideStaysBinary() { + run("CREATE TABLE left_t (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE right_t (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO left_t VALUES (1, 'Bob')") + run("INSERT INTO right_t VALUES (1, 'BOB')") + assertEquals( + emptyList(), + select("SELECT left_t.id FROM left_t JOIN right_t ON left_t.name = right_t.name").rows + ) + } + + @Test + fun declaredCollationAppliesInsideAnOnConflictDoUpdateAssignment() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, was_bob BOOLEAN)") + run("INSERT INTO t VALUES (1, 'Bob', FALSE)") + run("INSERT INTO t (id, name) VALUES (1, 'x') ON CONFLICT (id) DO UPDATE SET was_bob = (name = 'BOB')") + assertEquals(bool(true), select("SELECT was_bob FROM t WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInAMergeOnCondition() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, hits INTEGER)") + run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO accounts VALUES (1, 'Bob', 0)") + run("INSERT INTO staging VALUES (2, 'BOB')") + run("MERGE INTO accounts USING staging ON accounts.name = staging.name WHEN MATCHED THEN UPDATE SET hits = hits + 1") + assertEquals(int(1), select("SELECT hits FROM accounts WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInATriggerWhenClauseOnNew() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("CREATE TABLE hits (id INTEGER PRIMARY KEY, count INTEGER)") + run("INSERT INTO t VALUES (1, 'Bob')") + run("INSERT INTO hits VALUES (1, 0)") + run( + "CREATE TRIGGER bump AFTER UPDATE ON t FOR EACH ROW WHEN NEW.name = 'BOB' " + + "BEGIN UPDATE hits SET count = count + 100 WHERE id = NEW.id; END" + ) + run("UPDATE t SET name = 'bob' WHERE id = 1") + assertEquals(int(100), select("SELECT count FROM hits WHERE id = 1").rows.single().single()) + } + + /** + * `UNION`/`INTERSECT`/`EXCEPT`'s own `ORDER BY` ([kestrel.engine.SelectRunner. + * declaredSetOperationCollation]) - the last named gap left by #66 ("multiKeyComparator-owa + * sciezka UNION/INTERSECT/EXCEPT ORDER BY dalej binarna"). A `SELECT *` branch's declared + * collation is traced back to the real table each output column comes from. + */ + @Test + fun declaredCollationAppliesToASetOperationsOrderByThroughAStarProjection() { + seedNames() + val result = select( + "SELECT * FROM people WHERE id <= 2 UNION ALL SELECT * FROM people WHERE id > 2 ORDER BY name" + ) + // Combined pre-sort order is (1,Bob),(2,bob),(3,Alice),(4,Zoe) - NOCASE ranks Bob/bob equal, + // so the stable sort keeps Bob (scanned first) ahead of bob. + assertEquals( + listOf(int(3) to "Alice", int(1) to "Bob", int(2) to "bob", int(4) to "Zoe").map { (id, name) -> listOf(id, text(name)) }, + result.rows + ) + } + + @Test + fun declaredCollationAppliesToASetOperationsOrderByThroughAnAliasedProjection() { + seedNames() + val result = select( + "SELECT name AS n FROM people WHERE id <= 2 UNION ALL SELECT name AS n FROM people WHERE id > 2 ORDER BY n" + ) + assertEquals(listOf("Alice", "Bob", "bob", "Zoe").map { listOf(text(it)) }, result.rows) + } + + @Test + fun explicitCollateInASetOperationsOrderByOverridesTheColumnsDeclaredCollation() { + seedNames() + val result = select( + "SELECT * FROM people WHERE id <= 2 UNION ALL SELECT * FROM people WHERE id > 2 ORDER BY name COLLATE BINARY" + ) + assertEquals(listOf("Alice", "Bob", "Zoe", "bob"), result.rows.map { (it[1] as Value.TextValue).value }) + } + + /** + * The declared collation of an output column joined from a second `FROM` source is found past + * the first source's own column count - proves `declaredSetOperationCollation`'s per-source + * offset walk, not just the single-table case above. + */ + @Test + fun declaredCollationAppliesToASetOperationsOrderByAcrossAJoinedSecondTable() { + run("CREATE TABLE ids (id INTEGER PRIMARY KEY)") + run("CREATE TABLE names (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO ids VALUES (1), (2), (3), (4)") + run("INSERT INTO names VALUES (1, 'Bob'), (2, 'bob'), (3, 'Alice'), (4, 'Zoe')") + val result = select( + "SELECT * FROM ids JOIN names ON ids.id = names.id WHERE ids.id <= 2 " + + "UNION ALL SELECT * FROM ids JOIN names ON ids.id = names.id WHERE ids.id > 2 " + + "ORDER BY name" + ) + assertEquals(listOf("Alice", "Bob", "bob", "Zoe"), result.rows.map { (it[2] as Value.TextValue).value }) + } + + /** + * Still-open scope cut (see [kestrel.engine.SelectRunner.declaredSetOperationCollation]'s own + * doc comment): a set operation's left branch reading through a derived table rather than a + * real table directly reports no declared collation, so its ORDER BY stays binary even though + * the underlying table declares NOCASE - symmetric with the pre-existing view/CTE scope cut for + * a plain SELECT's own ORDER BY. + */ + @Test + fun declaredCollationDoesNotReachThroughADerivedTableInASetOperationsLeftBranch() { + seedNames() + val result = select( + "SELECT * FROM (SELECT * FROM people) sub WHERE id <= 2 " + + "UNION ALL SELECT * FROM (SELECT * FROM people) sub2 WHERE id > 2 " + + "ORDER BY name" + ) + assertEquals(listOf("Alice", "Bob", "Zoe", "bob"), result.rows.map { (it[1] as Value.TextValue).value }) + } + + /** + * `UNION`/`INTERSECT`/`EXCEPT`'s own multiset dedup ([kestrel.engine.SelectRunner. + * runSetOperation]) - the largest piece of the scope-cut #67/#68 left open ("UNION/CTE dedup ... + * dalej uzywaja bajtowej rownosci"). Reuses [kestrel.engine.SelectRunner.declaredSetOperationCollation], + * the same resolver the ORDER BY tests above already exercise, but now folded into the dedup key + * itself rather than only the comparator. + */ + @Test + fun plainUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name FROM people WHERE id = 1 UNION SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(text("Bob"))), result.rows) + } + + @Test + fun plainUnionWithNoDeclaredCollationKeepsCaseVariantsDistinct() { + run("CREATE TABLE plain2 (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO plain2 VALUES (1, 'Bob'), (2, 'bob')") + val result = select("SELECT name FROM plain2 WHERE id = 1 UNION SELECT name FROM plain2 WHERE id = 2") + assertEquals(listOf(listOf(text("Bob")), listOf(text("bob"))), result.rows) + } + + @Test + fun intersectMatchesCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name FROM people WHERE id = 1 INTERSECT SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(text("Bob"))), result.rows) + } + + @Test + fun exceptExcludesCaseVariantsThroughTheColumnsDeclaredCollation() { + seedNames() + val result = select("SELECT name FROM people WHERE id IN (1, 4) EXCEPT SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(text("Zoe"))), result.rows) + } + + /** + * `INTERSECT ALL`/`EXCEPT ALL` preserve duplicate counts through [kestrel.engine.SelectRunner. + * multisetFilter], which grouped its right-hand side by raw `Value` equality before this session + * - proving the count itself (not just presence/absence) is now grouped through the declared + * collation: 3 case-variant 'Bob' rows on the left, 2 on the right, min(3,2)=2 kept. + */ + @Test + fun intersectAllMatchesCaseVariantsThroughTheColumnsDeclaredCollationAndPreservesDuplicateCounts() { + run("CREATE TABLE dup (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO dup VALUES (1, 'Bob'), (2, 'Bob'), (3, 'bob'), (4, 'Zoe')") + val result = select( + "SELECT name FROM dup WHERE id IN (1, 2, 3) INTERSECT ALL SELECT name FROM dup WHERE id IN (2, 3)" + ) + assertEquals(listOf(listOf(text("Bob")), listOf(text("Bob"))), result.rows) + } + + /** + * A recursive CTE's own fixed-point dedup ([kestrel.engine.SelectRunner.buildCteScope]), folded + * through [kestrel.engine.SelectRunner.declaredRecursiveCteCollation] resolved against the + * anchor (base) term - mirrors [declaredSetOperationCollation] but for the base/recursive-term + * split instead of a set operation's left/right. `steps` chains 'Bob' -> 'Carol' -> 'bob'; without + * collation-aware dedup 'bob' would be treated as a new row (it differs from 'Bob' byte-for-byte) + * and the walk would keep going instead of recognizing it as the already-visited start node. + */ + @Test + fun recursiveCteWithUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + run("CREATE TABLE origin (name TEXT COLLATE NOCASE)") + run("INSERT INTO origin VALUES ('Bob')") + run("CREATE TABLE steps (name TEXT COLLATE NOCASE, next_name TEXT COLLATE NOCASE)") + run("INSERT INTO steps VALUES ('Bob', 'Carol'), ('Carol', 'bob')") + val result = select( + "WITH RECURSIVE walk(name) AS (" + + "SELECT name FROM origin " + + "UNION " + + "SELECT steps.next_name FROM steps JOIN walk ON steps.name = walk.name" + + ") SELECT name FROM walk ORDER BY name" + ) + assertEquals(listOf(listOf(text("Bob")), listOf(text("Carol"))), result.rows) + } + + @Test + fun createTableColumnCollationRoundTripsThroughSqlWriter() { + val createStatement = parseOne("CREATE TABLE people2 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") as CreateTableStatement + val rendered = SqlWriter.write(createStatement) + check("COLLATE NOCASE" in rendered) { "expected COLLATE NOCASE in rendered CREATE TABLE: $rendered" } + executor.execute(parseOne(rendered)) + run("INSERT INTO people2 VALUES (1, 'Bob')") + assertEquals(bool(true), select("SELECT name = 'BOB' FROM people2").rows.single().single()) + } + + /** + * A real correctness bug, not a documented scope-cut: [kestrel.catalog.SortedIndex] used to + * sort/compare its keys with plain byte order regardless of the indexed column's own declared + * `COLLATE` (see [kestrel.catalog.Catalog.collationsFor]/[kestrel.catalog.IndexKeyOrdering]), + * so once [kestrel.engine.IndexAccessPlanner] chose to serve `WHERE name = 'BOB'` from an index + * on a `NOCASE`-declared `name` column, the lookup silently missed a stored `'bob'` row - unlike + * every other collation gap in this codebase (all documented scope-cuts, never wrong answers), + * this one returned fewer rows than a full scan of the exact same table/predicate would, because + * [kestrel.engine.SelectRunner.materialize]'s post-scan `WHERE` filter only re-checks candidates + * the chosen access path already produced - it cannot recover a row the index never handed back. + */ + @Test + fun indexedEqualityHonorsTheColumnsDeclaredCollation() { + seedNames() + val scanned = select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows + run("CREATE INDEX idxName ON people (name)") + val plan = explain("EXPLAIN SELECT id FROM people WHERE name = 'BOB'") + assertTrue(plan.any { it.contains("Index Scan using idxName") }, "expected index scan, got $plan") + val indexed = select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows + assertEquals(scanned, indexed) + assertEquals(listOf(listOf(int(1)), listOf(int(2))), indexed) + } + + /** Mirrors [indexedEqualityHonorsTheColumnsDeclaredCollation] for a trailing-space collation instead of case. */ + @Test + fun indexedEqualityHonorsAnRtrimDeclaredCollation() { + run("CREATE TABLE tags (id INTEGER PRIMARY KEY, tag TEXT COLLATE RTRIM)") + run("INSERT INTO tags VALUES (1, 'a'), (2, 'a '), (3, 'b')") + run("CREATE INDEX idxTag ON tags (tag)") + val plan = explain("EXPLAIN SELECT id FROM tags WHERE tag = 'a'") + assertTrue(plan.any { it.contains("Index Scan using idxTag") }, "expected index scan, got $plan") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), select("SELECT id FROM tags WHERE tag = 'a' ORDER BY id").rows) + } + + /** + * A composite index's [kestrel.catalog.IndexKeyOrdering] carries one [kestrel.storage.Collation] + * per column position - `first` is `NOCASE` (matches both rows) but `last` stays plain `BINARY` + * (only `'Smith'` exactly matches), so only row 1 should survive both conjuncts. This only passes + * if each column position is folded through its OWN collation independently rather than, say, the + * whole composite key sharing a single collation. + */ + @Test + fun compositeIndexEqualityHonorsPerColumnDeclaredCollationIndependently() { + run("CREATE TABLE contacts (id INTEGER PRIMARY KEY, first TEXT COLLATE NOCASE, last TEXT)") + run("INSERT INTO contacts VALUES (1, 'Bob', 'Smith'), (2, 'bob', 'SMITH')") + run("CREATE INDEX idxContact ON contacts (first, last)") + val plan = explain("EXPLAIN SELECT id FROM contacts WHERE first = 'BOB' AND last = 'Smith'") + assertTrue(plan.any { it.contains("Index Scan using idxContact") }, "got $plan") + assertEquals( + listOf(listOf(int(1))), + select("SELECT id FROM contacts WHERE first = 'BOB' AND last = 'Smith'").rows + ) + } + + /** + * An explicit `COLLATE` at the query level overrides the column's own declared collation (see + * [kestrel.engine.pairCollation]) - [kestrel.engine.IndexAccessPlanner.asColumnPredicate] only + * recognizes a bare [kestrel.sql.ast.ColumnExpression], so a `COLLATE`-wrapped operand is simply + * excluded from pushdown rather than being (wrongly) served by an index built for the column's + * OWN collation. Proves that safety net: falling back to `Seq Scan` here, not silently pushing + * the query's `BINARY` override through a `NOCASE`-sorted index. + */ + @Test + fun explicitCollateOverrideOnAnIndexedColumnFallsBackToASeqScanInsteadOfMisusingTheIndex() { + seedNames() + run("CREATE INDEX idxName ON people (name)") + val plan = explain("EXPLAIN SELECT id FROM people WHERE name COLLATE BINARY = 'BOB'") + assertTrue(plan.any { it.contains("Seq Scan") }, "expected seq scan fallback, got $plan") + assertEquals(emptyList(), select("SELECT id FROM people WHERE name COLLATE BINARY = 'BOB'").rows) + } + + /** Control negative: an index over a column with no declared `COLLATE` behaves exactly as before this session. */ + @Test + fun indexedEqualityWithNoDeclaredCollationStaysBinary() { + run("CREATE TABLE plain3 (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO plain3 VALUES (1, 'Bob'), (2, 'bob')") + run("CREATE INDEX idxPlain3 ON plain3 (name)") + val plan = explain("EXPLAIN SELECT id FROM plain3 WHERE name = 'Bob'") + assertTrue(plan.any { it.contains("Index Scan using idxPlain3") }, "got $plan") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM plain3 WHERE name = 'Bob'").rows) + } + + /** + * PRIMARY KEY uniqueness ([kestrel.engine.Executor.enforcePrimaryKeyUniqueness]) folded through + * [kestrel.engine.collationFoldedUniquenessKey] - 'Bob'/'bob' now collide under a NOCASE-declared + * PRIMARY KEY exactly like they already do for a plain equality comparison. + */ + @Test + fun primaryKeyUniquenessHonorsTheColumnsDeclaredCollation() { + run("CREATE TABLE pk1 (name TEXT COLLATE NOCASE PRIMARY KEY)") + run("INSERT INTO pk1 VALUES ('Bob')") + assertFailsWith { run("INSERT INTO pk1 VALUES ('bob')") } + } + + /** [kestrel.engine.Executor.enforceUniqueConstraints], the declarative `UNIQUE` counterpart of the PK test above. */ + @Test + fun uniqueConstraintHonorsTheColumnsDeclaredCollation() { + run("CREATE TABLE u1 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE UNIQUE)") + run("INSERT INTO u1 VALUES (1, 'Bob')") + assertFailsWith { run("INSERT INTO u1 VALUES (2, 'bob')") } + } + + /** + * `CREATE UNIQUE INDEX` ([kestrel.engine.Executor.enforceUniqueIndexes]) checked against + * pre-existing rows before the index is registered - two rows already differing only by case + * under a `NOCASE`-declared column must now be rejected, matching what inserting either of them + * fresh under an already-`UNIQUE`-constrained column would do. + */ + @Test + fun createUniqueIndexHonorsTheColumnsDeclaredCollationAgainstPreexistingRows() { + run("CREATE TABLE u2 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO u2 VALUES (1, 'Bob'), (2, 'bob')") + assertFailsWith { run("CREATE UNIQUE INDEX idxU2 ON u2 (name)") } + } + + /** + * [kestrel.engine.Executor.enforceUniqueIndexes] guarding an ordinary `INSERT` against an + * ALREADY-REGISTERED `CREATE UNIQUE INDEX` - distinct from + * [createUniqueIndexHonorsTheColumnsDeclaredCollationAgainstPreexistingRows]'s backfill-time + * check: here the index already exists and a fresh case-variant row is what must be rejected, + * BEFORE the row is appended to the table (see `enforceUniqueIndexes`'s own doc comment on why + * it does not just rely on [kestrel.catalog.SortedIndex.add]'s own, likewise collation-aware, + * uniqueness check to catch this instead). + */ + @Test + fun uniqueIndexHonorsTheColumnsDeclaredCollationOnInsertAfterCreation() { + run("CREATE TABLE u5 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("CREATE UNIQUE INDEX idxU5 ON u5 (name)") + run("INSERT INTO u5 VALUES (1, 'Bob')") + assertFailsWith { run("INSERT INTO u5 VALUES (2, 'bob')") } + // Proves enforceUniqueIndexes itself rejected the row BEFORE table.rows.addAll ran, not + // merely that some exception surfaced - u5 has no trigger, so withStatementAtomicity takes + // no catalog snapshot here and could not have rolled back a row that got appended first. + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM u5").rows) + } + + /** Control negative: a `UNIQUE` column with no declared `COLLATE` behaves exactly as before this session. */ + @Test + fun uniqueConstraintWithNoDeclaredCollationStaysBinary() { + run("CREATE TABLE u3 (id INTEGER PRIMARY KEY, name TEXT UNIQUE)") + run("INSERT INTO u3 VALUES (1, 'Bob')") + run("INSERT INTO u3 VALUES (2, 'bob')") + assertEquals(2, select("SELECT id FROM u3").rows.size) + } +} diff --git a/src/test/kotlin/kestrel/CompositeIndexTest.kt b/src/test/kotlin/kestrel/CompositeIndexTest.kt new file mode 100644 index 0000000..05bdc1a --- /dev/null +++ b/src/test/kotlin/kestrel/CompositeIndexTest.kt @@ -0,0 +1,199 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** Covers multi-column `CREATE [UNIQUE] INDEX idx ON t (a, b, ...)`: parsing, uniqueness enforcement over the whole tuple, and query-planner pushdown (full match, equality prefix, and range-after-prefix scans). */ +class CompositeIndexTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + + private fun ids(result: SelectResult): List = result.rows.map { (it.single() as kestrel.storage.Value.IntegerValue).value } + + private fun seedPairs() { + run("CREATE TABLE pairs (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER)") + run( + "INSERT INTO pairs VALUES " + + "(1, 1, 10), (2, 1, 20), (3, 1, 30), (4, 2, 10), (5, 2, 20), (6, 3, 10)" + ) + } + + private fun seedRanked() { + run("CREATE TABLE ranked (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + run( + "INSERT INTO ranked VALUES " + + "(1, 30, 'amy'), (2, 30, 'bob'), (3, 40, 'cy'), (4, 40, 'dan'), (5, 50, 'eve')" + ) + } + + private fun seedSales() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, note TEXT)") + run( + "INSERT INTO sales VALUES " + + "(1, 'EU', 50, 'x'), (2, 'EU', 100, 'y'), (3, 'EU', 150, 'z'), (4, 'EU', 200, 'w'), " + + "(5, 'US', 150, 'v'), (6, 'US', 300, 'u')" + ) + } + + @Test + fun parsesMultiColumnCreateIndexAndCreateUniqueIndex() { + run("CREATE TABLE t (a INTEGER, b INTEGER, c INTEGER)") + run("CREATE INDEX idx ON t (a, b, c)") + run("CREATE UNIQUE INDEX uidx ON t (b, a)") + val schema = explain("EXPLAIN SELECT a FROM t").joinToString("\n") + assertTrue(schema.isNotEmpty()) + } + + @Test + fun compositeUniqueIndexRejectsDuplicateTupleButAllowsPartialReuse() { + seedPairs() + run("CREATE UNIQUE INDEX uidx ON pairs (a, b)") + // (1, 10) already exists as row 1; same 'a' with a different 'b', or same 'b' with a + // different 'a', must both be allowed - only the exact tuple collides. + run("INSERT INTO pairs VALUES (7, 1, 99)") + run("INSERT INTO pairs VALUES (8, 9, 10)") + assertFailsWith { run("INSERT INTO pairs VALUES (9, 1, 10)") } + } + + @Test + fun compositeUniqueIndexRejectsDuplicateTupleOnUpdate() { + seedPairs() + run("CREATE UNIQUE INDEX uidx ON pairs (a, b)") + assertFailsWith { run("UPDATE pairs SET a = 1, b = 20 WHERE id = 1") } + } + + @Test + fun compositeUniqueIndexAllowsMultipleNullsInEitherColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER)") + run("CREATE UNIQUE INDEX uidx ON t (a, b)") + run("INSERT INTO t VALUES (1, NULL, 5), (2, NULL, 5), (3, 5, NULL), (4, 5, NULL)") + assertEquals(4, select("SELECT id FROM t").rows.size) + } + + @Test + fun creatingACompositeUniqueIndexOverExistingDuplicatesFailsWithoutLeavingHalfState() { + seedPairs() + run("INSERT INTO pairs VALUES (7, 1, 10)") + assertFailsWith { run("CREATE UNIQUE INDEX uidx ON pairs (a, b)") } + run("CREATE INDEX uidx ON pairs (a, b)") + } + + @Test + fun indexOnUnknownColumnInCompositeListIsRejected() { + seedPairs() + val error = assertFailsWith { run("CREATE INDEX bad ON pairs (a, missing)") } + assertTrue(error.message!!.contains("missing"), error.message) + } + + @Test + fun plannerUsesCompositeIndexForFullEqualityMatch() { + seedPairs() + val scanned = select("SELECT id FROM pairs WHERE a = 1 AND b = 20").rows + run("CREATE INDEX idx ON pairs (a, b)") + val plan = explain("EXPLAIN SELECT id FROM pairs WHERE a = 1 AND b = 20") + assertTrue(plan.any { it.contains("Index Scan using idx") }, "got $plan") + val indexed = select("SELECT id FROM pairs WHERE a = 1 AND b = 20").rows + assertEquals(scanned, indexed) + assertEquals(listOf(2L), ids(select("SELECT id FROM pairs WHERE a = 1 AND b = 20"))) + } + + @Test + fun plannerUsesCompositeIndexAsEqualityPrefixScan() { + seedPairs() + val scanned = select("SELECT id FROM pairs WHERE a = 1").rows + run("CREATE INDEX idx ON pairs (a, b)") + val plan = explain("EXPLAIN SELECT id FROM pairs WHERE a = 1") + assertTrue(plan.any { it.contains("Index Scan using idx") }, "got $plan") + val indexed = select("SELECT id FROM pairs WHERE a = 1").rows + assertEquals(scanned, indexed) + assertEquals(listOf(1L, 2L, 3L), ids(select("SELECT id FROM pairs WHERE a = 1"))) + } + + @Test + fun plannerIgnoresCompositeIndexWhenOnlyTrailingColumnIsBound() { + seedPairs() + run("CREATE INDEX idx ON pairs (a, b)") + val plan = explain("EXPLAIN SELECT id FROM pairs WHERE b = 10") + assertTrue(plan.any { it.contains("Seq Scan on pairs") }, "got $plan") + assertFalse(plan.any { it.contains("Index Scan") }, "got $plan") + } + + @Test + fun plannerUsesCompositeIndexForRangeOnLeadingColumnAlone() { + seedRanked() + val scanned = select("SELECT id FROM ranked WHERE age > 30").rows + run("CREATE INDEX idxAgeName ON ranked (age, name)") + val plan = explain("EXPLAIN SELECT id FROM ranked WHERE age > 30") + assertTrue(plan.any { it.contains("Index Scan using idxAgeName") }, "got $plan") + val indexed = select("SELECT id FROM ranked WHERE age > 30").rows + assertEquals(scanned, indexed) + assertEquals(listOf(3L, 4L, 5L), ids(select("SELECT id FROM ranked WHERE age > 30"))) + } + + @Test + fun plannerUsesCompositeIndexForTwoSidedRangeOnLeadingColumn() { + seedRanked() + val scanned = select("SELECT id FROM ranked WHERE age > 30 AND age < 50").rows + run("CREATE INDEX idxAgeName ON ranked (age, name)") + val indexed = select("SELECT id FROM ranked WHERE age > 30 AND age < 50").rows + assertEquals(scanned, indexed) + assertEquals(listOf(3L, 4L), ids(select("SELECT id FROM ranked WHERE age > 30 AND age < 50"))) + } + + @Test + fun plannerUsesCompositeIndexForEqualityPrefixPlusRangeOnNextColumnWithFreeTail() { + seedSales() + val scanned = select("SELECT id FROM sales WHERE region = 'EU' AND amount > 100").rows + run("CREATE INDEX idxRegionAmount ON sales (region, amount, note)") + val plan = explain("EXPLAIN SELECT id FROM sales WHERE region = 'EU' AND amount > 100") + assertTrue(plan.any { it.contains("Index Scan using idxRegionAmount") }, "got $plan") + val indexed = select("SELECT id FROM sales WHERE region = 'EU' AND amount > 100").rows + assertEquals(scanned, indexed) + // Must exclude the US row with a bigger amount (300) and the EU row at exactly 100. + assertEquals(listOf(3L, 4L), ids(select("SELECT id FROM sales WHERE region = 'EU' AND amount > 100"))) + } + + @Test + fun plannerUsesCompositeIndexForEqualityPrefixPlusInclusiveUpperBound() { + seedSales() + run("CREATE INDEX idxRegionAmount ON sales (region, amount, note)") + val indexed = ids(select("SELECT id FROM sales WHERE region = 'EU' AND amount <= 100")) + assertEquals(listOf(1L, 2L), indexed) + } + + @Test + fun plannerPrefersTheMoreSelectiveCompositeIndexOverASingleColumnIndex() { + seedSales() + run("CREATE INDEX idxAmount ON sales (amount)") + run("CREATE INDEX idxRegionAmount ON sales (region, amount)") + val plan = explain("EXPLAIN SELECT id FROM sales WHERE region = 'EU' AND amount = 100") + assertTrue(plan.any { it.contains("Index Scan using idxRegionAmount") }, "got $plan") + assertFalse(plan.any { it.contains("idxAmount") }, "got $plan") + } + + @Test + fun dropCompositeIndexRemovesItFromPlanner() { + seedPairs() + run("CREATE INDEX idx ON pairs (a, b)") + run("DROP INDEX idx") + val plan = explain("EXPLAIN SELECT id FROM pairs WHERE a = 1 AND b = 20") + assertTrue(plan.any { it.contains("Seq Scan on pairs") }, "got $plan") + } +} diff --git a/src/test/kotlin/kestrel/CompositeKeyTest.kt b/src/test/kotlin/kestrel/CompositeKeyTest.kt new file mode 100644 index 0000000..56285e0 --- /dev/null +++ b/src/test/kotlin/kestrel/CompositeKeyTest.kt @@ -0,0 +1,316 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * Multi-column PRIMARY KEY (`PRIMARY KEY (a, b, ...)`) and multi-column FOREIGN KEY + * (`FOREIGN KEY (a, b) REFERENCES parent(x, y)`), the last documented "not yet supported" gap + * left in TableSchema/ForeignKeyEnforcer after single-column UNIQUE landed. + */ +class CompositeKeyTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun parseOne(sql: String): Statement = Parser(Lexer(sql).tokenize()).parseProgram().single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + + // ---- parsing ---- + + @Test + fun parsesTableLevelCompositePrimaryKey() { + val statement = parseOne( + "CREATE TABLE line_items (order_id INTEGER, sku TEXT, qty INTEGER, PRIMARY KEY (order_id, sku))" + ) as CreateTableStatement + assertEquals(listOf("order_id", "sku"), statement.compositePrimaryKey) + assertTrue(statement.columns.none { it.primaryKey }) + } + + @Test + fun parsesTableLevelCompositeForeignKey() { + val statement = parseOne( + "CREATE TABLE t (a INTEGER, b INTEGER, " + + "FOREIGN KEY (a, b) REFERENCES parent (x, y) ON DELETE CASCADE)" + ) as CreateTableStatement + val fk = statement.foreignKeys.single() + assertEquals(listOf("a", "b"), fk.columns) + assertEquals("parent", fk.referencedTable) + assertEquals(listOf("x", "y"), fk.referencedColumns) + } + + @Test + fun mismatchedForeignKeyColumnCountIsRejectedAtParseTime() { + assertFailsWith { + parseOne("CREATE TABLE t (a INTEGER, b INTEGER, FOREIGN KEY (a, b) REFERENCES parent (x))") + } + } + + // ---- CREATE TABLE validation ---- + + @Test + fun declaringBothInlineAndTableLevelPrimaryKeyIsRejected() { + assertFailsWith { + run("CREATE TABLE t (a INTEGER PRIMARY KEY, b INTEGER, PRIMARY KEY (a, b))") + } + } + + @Test + fun duplicateColumnWithinCompositePrimaryKeyIsRejected() { + assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER, PRIMARY KEY (a, a))") + } + } + + @Test + fun compositePrimaryKeyReferringToUnknownColumnIsRejected() { + assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER, PRIMARY KEY (a, missing))") + } + } + + @Test + fun compositePrimaryKeyImpliesNotNullOnEveryListedColumn() { + run("CREATE TABLE t (a INTEGER, b INTEGER, PRIMARY KEY (a, b))") + assertFailsWith { run("INSERT INTO t VALUES (1, NULL)") } + assertFailsWith { run("INSERT INTO t VALUES (NULL, 1)") } + } + + @Test + fun foreignKeyMustReferenceAnExactEnforcedUniqueGroupNotAnArbitraryPair() { + run("CREATE TABLE parent (a INTEGER, b INTEGER, c INTEGER, PRIMARY KEY (a, b))") + assertFailsWith { + run("CREATE TABLE child (x INTEGER, y INTEGER, FOREIGN KEY (x, y) REFERENCES parent (b, c))") + } + } + + @Test + fun foreignKeyMayReferenceACompositePrimaryKeyInAnyColumnOrder() { + run("CREATE TABLE parent (a INTEGER, b INTEGER, PRIMARY KEY (a, b))") + run("CREATE TABLE child (x INTEGER, y INTEGER, FOREIGN KEY (x, y) REFERENCES parent (b, a))") + } + + // ---- composite PRIMARY KEY uniqueness ---- + + @Test + fun compositePrimaryKeyRejectsAnExactDuplicateCombination() { + run("CREATE TABLE line_items (order_id INTEGER, sku TEXT, qty INTEGER, PRIMARY KEY (order_id, sku))") + run("INSERT INTO line_items VALUES (1, 'A', 3)") + assertFailsWith { run("INSERT INTO line_items VALUES (1, 'A', 5)") } + } + + @Test + fun compositePrimaryKeyAllowsDifferentCombinationsSharingOneColumnValue() { + run("CREATE TABLE line_items (order_id INTEGER, sku TEXT, qty INTEGER, PRIMARY KEY (order_id, sku))") + run("INSERT INTO line_items VALUES (1, 'A', 3)") + val result = exec("INSERT INTO line_items VALUES (1, 'B', 3)") + assertEquals(InsertResult(1), result) + } + + @Test + fun updateThatWouldCollideWithAnotherRowsCompositePrimaryKeyIsRejected() { + run("CREATE TABLE line_items (order_id INTEGER, sku TEXT, qty INTEGER, PRIMARY KEY (order_id, sku))") + run("INSERT INTO line_items VALUES (1, 'A', 3), (1, 'B', 5)") + assertFailsWith { run("UPDATE line_items SET sku = 'A' WHERE sku = 'B'") } + } + + // ---- composite FOREIGN KEY: existence checks ---- + + private fun seedOrdersAndLineItems() { + run("CREATE TABLE orders (region TEXT, order_no INTEGER, PRIMARY KEY (region, order_no))") + run( + "CREATE TABLE line_items (" + + "region TEXT, order_no INTEGER, sku TEXT, qty INTEGER, " + + "FOREIGN KEY (region, order_no) REFERENCES orders (region, order_no) " + + "ON DELETE CASCADE ON UPDATE CASCADE)" + ) + run("INSERT INTO orders VALUES ('eu', 1), ('us', 1)") + } + + @Test + fun compositeForeignKeyRejectsAPartialMatch() { + seedOrdersAndLineItems() + // ('eu', 1) and ('us', 1) both exist individually but not as the pair ('us', 2). + assertFailsWith { + run("INSERT INTO line_items VALUES ('us', 2, 'SKU1', 3)") + } + } + + @Test + fun compositeForeignKeyAcceptsAFullMatch() { + seedOrdersAndLineItems() + val result = exec("INSERT INTO line_items VALUES ('eu', 1, 'SKU1', 3)") + assertEquals(InsertResult(1), result) + } + + @Test + fun compositeForeignKeyMatchSimpleSkipsCheckWhenAnyLocalColumnIsNull() { + seedOrdersAndLineItems() + val result = exec("INSERT INTO line_items VALUES ('eu', NULL, 'SKU1', 3)") + assertEquals(InsertResult(1), result) + } + + @Test + fun compositeForeignKeyOnDeleteCascadeRemovesMatchingChildren() { + seedOrdersAndLineItems() + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1', 3), ('us', 1, 'SKU2', 4)") + run("DELETE FROM orders WHERE region = 'eu'") + val remaining = select("SELECT sku FROM line_items") + assertEquals(listOf(listOf(Value.TextValue("SKU2"))), remaining.rows) + } + + @Test + fun compositeForeignKeyOnUpdateCascadePropagatesBothColumnsTogether() { + seedOrdersAndLineItems() + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1', 3)") + run("UPDATE orders SET order_no = 7 WHERE region = 'eu'") + val updated = select("SELECT region, order_no FROM line_items WHERE sku = 'SKU1'") + assertEquals(listOf(listOf(Value.TextValue("eu"), int(7))), updated.rows) + } + + @Test + fun compositeForeignKeyOnDeleteRestrictBlocksByDefaultWithoutCascade() { + run("CREATE TABLE orders (region TEXT, order_no INTEGER, PRIMARY KEY (region, order_no))") + run( + "CREATE TABLE line_items (region TEXT, order_no INTEGER, sku TEXT, " + + "FOREIGN KEY (region, order_no) REFERENCES orders (region, order_no))" + ) + run("INSERT INTO orders VALUES ('eu', 1)") + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1')") + assertFailsWith { run("DELETE FROM orders WHERE region = 'eu'") } + } + + @Test + fun compositeForeignKeyOnDeleteSetNullNullsEveryLocalColumn() { + run("CREATE TABLE orders (region TEXT, order_no INTEGER, PRIMARY KEY (region, order_no))") + run( + "CREATE TABLE line_items (region TEXT, order_no INTEGER, sku TEXT, " + + "FOREIGN KEY (region, order_no) REFERENCES orders (region, order_no) ON DELETE SET NULL)" + ) + run("INSERT INTO orders VALUES ('eu', 1)") + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1')") + run("DELETE FROM orders WHERE region = 'eu'") + val row = select("SELECT region, order_no FROM line_items WHERE sku = 'SKU1'").rows.single() + assertEquals(listOf(Value.NullValue, Value.NullValue), row) + } + + @Test + fun rollbackAfterCompositeCascadeDeleteRestoresBothTables() { + seedOrdersAndLineItems() + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1', 3)") + run("BEGIN") + run("DELETE FROM orders WHERE region = 'eu'") + run("ROLLBACK") + assertEquals(2, select("SELECT * FROM orders").rows.size) + assertEquals(1, select("SELECT * FROM line_items").rows.size) + } + + // ---- composite FOREIGN KEY referencing a composite UNIQUE (not the PRIMARY KEY) ---- + + @Test + fun compositeForeignKeyMayReferenceACompositeUniqueConstraintInsteadOfThePrimaryKey() { + run( + "CREATE TABLE products (id INTEGER PRIMARY KEY, brand TEXT, model TEXT, UNIQUE (brand, model))" + ) + run( + "CREATE TABLE reviews (id INTEGER PRIMARY KEY, brand TEXT, model TEXT, stars INTEGER, " + + "FOREIGN KEY (brand, model) REFERENCES products (brand, model))" + ) + run("INSERT INTO products VALUES (1, 'acme', 'x100')") + val result = exec("INSERT INTO reviews VALUES (1, 'acme', 'x100', 5)") + assertEquals(InsertResult(1), result) + assertFailsWith { + run("INSERT INTO reviews VALUES (2, 'acme', 'missing', 4)") + } + } + + // ---- self-referencing composite FOREIGN KEY ---- + + @Test + fun selfReferencingCompositeForeignKeySeedableInASingleMultiRowInsert() { + run( + "CREATE TABLE edges (" + + "tenant TEXT, node INTEGER, parent_tenant TEXT, parent_node INTEGER, " + + "PRIMARY KEY (tenant, node), " + + "FOREIGN KEY (parent_tenant, parent_node) REFERENCES edges (tenant, node))" + ) + // The root's parent columns are NULL (no parent); child references the root, all seeded + // together in one INSERT so the child must see its not-yet-committed sibling. + run( + "INSERT INTO edges VALUES " + + "('t1', 1, NULL, NULL), ('t1', 2, 't1', 1), ('t1', 3, 't1', 2)" + ) + assertEquals(3, select("SELECT * FROM edges").rows.size) + } + + // ---- SqlWriter round-trip ---- + + @Test + fun createTableWithCompositePrimaryKeyRoundTripsThroughSqlWriter() { + val statement = parseOne( + "CREATE TABLE line_items (order_id INTEGER, sku TEXT, qty INTEGER, PRIMARY KEY (order_id, sku))" + ) + run(SqlWriter.write(statement)) + run("INSERT INTO line_items VALUES (1, 'A', 3)") + assertFailsWith { run("INSERT INTO line_items VALUES (1, 'A', 9)") } + } + + @Test + fun createTableWithCompositeForeignKeyRoundTripsThroughSqlWriter() { + run(SqlWriter.write(parseOne("CREATE TABLE orders (region TEXT, order_no INTEGER, PRIMARY KEY (region, order_no))"))) + run( + SqlWriter.write( + parseOne( + "CREATE TABLE line_items (region TEXT, order_no INTEGER, sku TEXT, " + + "FOREIGN KEY (region, order_no) REFERENCES orders (region, order_no) ON DELETE CASCADE)" + ) + ) + ) + run("INSERT INTO orders VALUES ('eu', 1)") + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1')") + run("DELETE FROM orders WHERE region = 'eu'") + assertTrue(select("SELECT * FROM line_items").rows.isEmpty()) + } + + @Test + fun dumpAndReplayPreservesCompositePrimaryKeyAndForeignKey() { + seedOrdersAndLineItems() + run("INSERT INTO line_items VALUES ('eu', 1, 'SKU1', 3)") + val dump = SqlWriter.dump(catalog) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + assertFailsWith { + Parser(Lexer("INSERT INTO orders VALUES ('eu', 1)").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + } + Parser(Lexer("DELETE FROM orders WHERE region = 'eu'").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + val remaining = freshExecutor.execute( + Parser(Lexer("SELECT * FROM line_items").tokenize()).parseProgram().single() + ) as SelectResult + assertTrue(remaining.rows.isEmpty()) + } +} diff --git a/src/test/kotlin/kestrel/CompoundQueryTest.kt b/src/test/kotlin/kestrel/CompoundQueryTest.kt new file mode 100644 index 0000000..8bc6333 --- /dev/null +++ b/src/test/kotlin/kestrel/CompoundQueryTest.kt @@ -0,0 +1,254 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class CompoundQueryTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seed() { + run("CREATE TABLE dual (x INTEGER)") + run("INSERT INTO dual VALUES (1)") + run("CREATE TABLE t1 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE t2 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("INSERT INTO t1 VALUES (1, 'ada'), (2, 'linus')") + run("INSERT INTO t2 VALUES (2, 'linus'), (3, 'grace')") + } + + @Test + fun unionRemovesDuplicateRows() { + seed() + val result = select("SELECT name FROM t1 UNION SELECT name FROM t2 ORDER BY name") + assertEquals(listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus"))), result.rows) + } + + @Test + fun unionAllKeepsDuplicateRows() { + seed() + val result = select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name") + assertEquals( + listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus")), listOf(text("linus"))), + result.rows + ) + } + + @Test + fun intersectReturnsRowsPresentInBothQueries() { + seed() + val result = select("SELECT name FROM t1 INTERSECT SELECT name FROM t2") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun exceptReturnsRowsOnlyInTheLeftQuery() { + seed() + val result = select("SELECT name FROM t1 EXCEPT SELECT name FROM t2") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun intersectAllKeepsDuplicatesUpToTheMinimumCountInEitherSide() { + run("CREATE TABLE bag_a (v INTEGER)") + run("CREATE TABLE bag_b (v INTEGER)") + run("INSERT INTO bag_a VALUES (1), (1), (1), (2)") + run("INSERT INTO bag_b VALUES (1), (1), (2), (2), (3)") + val result = select("SELECT v FROM bag_a INTERSECT ALL SELECT v FROM bag_b ORDER BY v") + assertEquals(listOf(listOf(int(1)), listOf(int(1)), listOf(int(2))), result.rows) + } + + @Test + fun exceptAllKeepsLeftDuplicatesNotConsumedByTheRightSide() { + run("CREATE TABLE bag_a (v INTEGER)") + run("CREATE TABLE bag_b (v INTEGER)") + run("INSERT INTO bag_a VALUES (1), (1), (1), (2)") + run("INSERT INTO bag_b VALUES (1), (1), (2), (2), (3)") + val result = select("SELECT v FROM bag_a EXCEPT ALL SELECT v FROM bag_b ORDER BY v") + assertEquals(listOf(listOf(int(1))), result.rows) + } + + @Test + fun mismatchedColumnCountsFail() { + seed() + assertFailsWith { + select("SELECT id, name FROM t1 UNION SELECT id FROM t2") + } + } + + @Test + fun compoundOperatorsChainLeftAssociatively() { + seed() + run("CREATE TABLE t3 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("INSERT INTO t3 VALUES (2, 'linus')") + val result = select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 INTERSECT SELECT name FROM t3 ORDER BY name") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun orderByAndLimitApplyToTheWholeCompoundResult() { + seed() + val result = select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name DESC LIMIT 2") + assertEquals(listOf(listOf(text("linus")), listOf(text("linus"))), result.rows) + } + + @Test + fun offsetSkipsRowsOfTheWholeCompoundResultBeforeLimitApplies() { + seed() + val result = select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name DESC LIMIT 2 OFFSET 1") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + } + + @Test + fun compoundQueryWorksAsDerivedTableInFrom() { + seed() + val result = select("SELECT count(*) FROM (SELECT name FROM t1 UNION SELECT name FROM t2) AS u") + assertEquals(listOf(listOf(int(3))), result.rows) + } + + @Test + fun compoundQueryWorksAsViewBody() { + seed() + run("CREATE VIEW combined AS SELECT name FROM t1 UNION SELECT name FROM t2") + val result = select("SELECT name FROM combined ORDER BY name") + assertEquals(listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus"))), result.rows) + run("INSERT INTO t1 VALUES (4, 'margaret')") + val refreshed = select("SELECT name FROM combined ORDER BY name") + assertEquals(4, refreshed.rows.size) + } + + @Test + fun explainDescribesCompoundQuery() { + seed() + val plan = (run("EXPLAIN SELECT name FROM t1 UNION SELECT name FROM t2").single() as ExplainResult).plan + assertTrue(plan.isNotEmpty()) + assertTrue(plan.any { it.contains("Union") }) + } + + @Test + fun withClauseMakesACteAvailableInFrom() { + seed() + run("WITH recent AS (SELECT id, name FROM t1) SELECT name FROM recent WHERE id = 1") + val result = select("WITH recent AS (SELECT id, name FROM t1) SELECT name FROM recent WHERE id = 1") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun laterCteCanReferenceAnEarlierCteInTheSameWithClause() { + seed() + val result = select( + "WITH first_cte AS (SELECT id, name FROM t1), second_cte AS (SELECT name FROM first_cte WHERE id = 2) " + + "SELECT name FROM second_cte" + ) + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun cteWithExplicitColumnListRenamesOutputColumns() { + seed() + val result = select("WITH renamed(label) AS (SELECT name FROM t1) SELECT label FROM renamed ORDER BY label") + assertEquals(listOf(listOf(text("ada")), listOf(text("linus"))), result.rows) + } + + @Test + fun cteColumnListSizeMustMatchQueryColumnCount() { + seed() + assertFailsWith { + select("WITH bad(a, b, c) AS (SELECT id FROM t1) SELECT * FROM bad") + } + } + + @Test + fun recursiveCteGeneratesABoundedSequence() { + seed() + val result = select( + "WITH RECURSIVE counter(n) AS (" + + "SELECT 1 FROM dual " + + "UNION ALL " + + "SELECT n + 1 FROM counter WHERE n < 5" + + ") SELECT n FROM counter ORDER BY n" + ) + assertEquals((1..5).map { listOf(int(it.toLong())) }, result.rows) + } + + @Test + fun recursiveCteAccumulatesARunningTotal() { + seed() + val result = select( + "WITH RECURSIVE sums(n, total) AS (" + + "SELECT 1, 1 FROM dual " + + "UNION ALL " + + "SELECT n + 1, total + n + 1 FROM sums WHERE n < 5" + + ") SELECT total FROM sums WHERE n = 5" + ) + assertEquals(listOf(listOf(int(15))), result.rows) + } + + @Test + fun recursiveCteWithUnionDeduplicatesAndTerminatesOnACycle() { + seed() + run("CREATE TABLE edges (from_node INTEGER, to_node INTEGER)") + run("INSERT INTO edges VALUES (1, 2), (2, 3), (3, 1), (3, 4)") + val result = select( + "WITH RECURSIVE reachable(node) AS (" + + "SELECT 1 FROM dual " + + "UNION " + + "SELECT edges.to_node FROM edges JOIN reachable ON edges.from_node = reachable.node" + + ") SELECT node FROM reachable ORDER BY node" + ) + assertEquals(listOf(1L, 2L, 3L, 4L).map { listOf(int(it)) }, result.rows) + } + + @Test + fun recursiveCteWithoutSelfReferenceBehavesAsAPlainCompoundQuery() { + seed() + val result = select( + "WITH RECURSIVE notActuallyRecursive AS (SELECT 1 AS n FROM dual UNION ALL SELECT 2 FROM dual) " + + "SELECT n FROM notActuallyRecursive ORDER BY n" + ) + assertEquals(listOf(listOf(int(1)), listOf(int(2))), result.rows) + } + + @Test + fun recursiveCteReferencingItselfOnBothSidesFails() { + seed() + assertFailsWith { + select( + "WITH RECURSIVE bad(n) AS (" + + "SELECT n FROM bad UNION ALL SELECT n FROM bad" + + ") SELECT n FROM bad" + ) + } + } + + @Test + fun nonTerminatingRecursiveCteFailsInsteadOfLoopingForever() { + seed() + assertFailsWith { + select( + "WITH RECURSIVE counter(n) AS (" + + "SELECT 1 FROM dual " + + "UNION ALL " + + "SELECT n + 1 FROM counter" + + ") SELECT n FROM counter" + ) + } + } +} diff --git a/src/test/kotlin/kestrel/DerivedTableTest.kt b/src/test/kotlin/kestrel/DerivedTableTest.kt new file mode 100644 index 0000000..c1164a9 --- /dev/null +++ b/src/test/kotlin/kestrel/DerivedTableTest.kt @@ -0,0 +1,114 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class DerivedTableTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seedUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER, dept TEXT)") + run( + "INSERT INTO users VALUES " + + "(1, 'ada', 36, 'eng'), (2, 'linus', 54, 'eng'), (3, 'grace', 85, 'navy'), (4, 'alan', 41, NULL)" + ) + } + + @Test + fun selectsFromASimpleDerivedTable() { + seedUsers() + val result = select("SELECT s.name FROM (SELECT id, name FROM users WHERE age >= 40) AS s ORDER BY s.id ASC") + assertEquals(listOf(text("linus"), text("grace"), text("alan")), result.rows.map { it.single() }) + } + + @Test + fun derivedTableOverAnAggregateQueryWorks() { + seedUsers() + val result = select( + "SELECT d.dept, d.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d " + + "WHERE d.n > 1 ORDER BY d.dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } + + @Test + fun derivedTableCanBeJoinedWithARealTable() { + seedUsers() + run("CREATE TABLE bonuses (dept TEXT, amount INTEGER)") + run("INSERT INTO bonuses VALUES ('eng', 1000), ('navy', 500)") + val result = select( + "SELECT d.dept, b.amount FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d " + + "JOIN bonuses AS b ON d.dept = b.dept ORDER BY d.dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(1000)), listOf(text("navy"), int(500))), result.rows) + } + + @Test + fun twoDerivedTablesCanBeJoinedTogether() { + seedUsers() + val result = select( + "SELECT a.dept, a.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS a " + + "JOIN (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS b ON a.dept = b.dept " + + "WHERE a.n = b.n ORDER BY a.dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(2)), listOf(text("navy"), int(1))), result.rows) + } + + @Test + fun derivedTableColumnsAreScopedToItsAliasNotTheUnderlyingTableName() { + seedUsers() + assertFailsWith { run("SELECT users.name FROM (SELECT id, name FROM users) AS s") } + } + + @Test + fun derivedTableIsReEvaluatedEachQueryReflectingLiveData() { + seedUsers() + val before = select("SELECT COUNT(*) AS n FROM (SELECT id FROM users WHERE dept = 'eng') AS e") + assertEquals(listOf(listOf(int(2))), before.rows) + run("INSERT INTO users VALUES (5, 'margaret', 61, 'eng')") + val after = select("SELECT COUNT(*) AS n FROM (SELECT id FROM users WHERE dept = 'eng') AS e") + assertEquals(listOf(listOf(int(3))), after.rows) + } + + @Test + fun derivedTableCanContainACorrelatedSubqueryOfItsOwn() { + seedUsers() + val result = select( + "SELECT s.name FROM (SELECT id, name, dept FROM users u WHERE age > " + + "(SELECT AVG(age) FROM users u2 WHERE u2.dept = u.dept)) AS s ORDER BY s.id ASC" + ) + assertEquals(listOf(text("linus")), result.rows.map { it.single() }) + } + + @Test + fun explainDescribesADerivedTableScan() { + seedUsers() + val explain = run("EXPLAIN SELECT s.name FROM (SELECT id, name FROM users) AS s").single() as kestrel.engine.ExplainResult + assertEquals(true, explain.plan.any { it.contains("Seq Scan on s") }) + } + + @Test + fun creatingAViewFromADerivedTableWorks() { + seedUsers() + run("CREATE VIEW dept_counts AS SELECT d.dept, d.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d") + val result = select("SELECT dept, n FROM dept_counts WHERE n > 1") + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } +} diff --git a/src/test/kotlin/kestrel/DistinctTest.kt b/src/test/kotlin/kestrel/DistinctTest.kt new file mode 100644 index 0000000..5119128 --- /dev/null +++ b/src/test/kotlin/kestrel/DistinctTest.kt @@ -0,0 +1,182 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.ExplainResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class DistinctTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seed() { + run("CREATE TABLE sales (dept TEXT, region TEXT, amount INTEGER)") + run( + "INSERT INTO sales VALUES " + + "('eng', 'eu', 100), ('eng', 'us', 200), ('eng', 'eu', 100), " + + "('ops', 'eu', 50), ('ops', 'eu', 70), ('ops', 'eu', 50)" + ) + } + + @Test + fun distinctRemovesDuplicateOutputRows() { + seed() + val result = select("SELECT DISTINCT dept FROM sales ORDER BY dept") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops"))), result.rows) + } + + @Test + fun distinctConsidersTheWholeProjectedRow() { + seed() + val result = select("SELECT DISTINCT dept, region FROM sales ORDER BY dept, region") + assertEquals( + listOf(listOf(text("eng"), text("eu")), listOf(text("eng"), text("us")), listOf(text("ops"), text("eu"))), + result.rows + ) + } + + @Test + fun selectAllIsTheDefaultAndKeepsDuplicates() { + seed() + assertEquals(6, select("SELECT ALL dept FROM sales").rows.size) + assertEquals(6, select("SELECT dept FROM sales").rows.size) + } + + @Test + fun distinctStarDedupsOnEveryColumn() { + seed() + val result = select("SELECT DISTINCT * FROM sales ORDER BY dept, region, amount") + assertEquals(4, result.rows.size) + } + + @Test + fun distinctWithLimitCountsDistinctRowsNotSourceRows() { + seed() + val result = select("SELECT DISTINCT dept FROM sales ORDER BY dept LIMIT 1") + assertEquals(listOf(listOf(text("eng"))), result.rows) + } + + @Test + fun distinctWithOffsetSkipsDistinctRowsNotSourceRows() { + seed() + val result = select("SELECT DISTINCT dept FROM sales ORDER BY dept LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun distinctCombinesWithGroupBy() { + run("CREATE TABLE t (a INTEGER, b INTEGER)") + run("INSERT INTO t VALUES (1, 10), (1, 20), (2, 10)") + // GROUP BY a, b produces three distinct groups; DISTINCT on the projected `a` alone + // collapses the two groups sharing a=1 down to one output row. + val result = select("SELECT DISTINCT a FROM t GROUP BY a, b ORDER BY a") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), result.rows) + } + + @Test + fun distinctOnKeepsFirstRowPerGroupInOrderByOrder() { + seed() + val result = select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, amount DESC") + assertEquals(listOf(listOf(text("eng"), int(200)), listOf(text("ops"), int(70))), result.rows) + } + + @Test + fun distinctOnWithoutOrderByUsesScanOrder() { + run("CREATE TABLE t (k TEXT, v INTEGER)") + run("INSERT INTO t VALUES ('a', 1), ('a', 2), ('b', 3)") + val result = select("SELECT DISTINCT ON (k) k, v FROM t") + assertEquals(listOf(listOf(text("a"), int(1)), listOf(text("b"), int(3))), result.rows) + } + + @Test + fun distinctOnStarProjectsEveryColumnOfTheKeptRow() { + seed() + val result = select("SELECT DISTINCT ON (dept) * FROM sales ORDER BY dept, amount DESC") + assertEquals(listOf("dept", "region", "amount"), result.columns) + assertEquals(listOf(text("eng"), text("us"), int(200)), result.rows[0]) + assertEquals(listOf(text("ops"), text("eu"), int(70)), result.rows[1]) + } + + @Test + fun distinctOnRequiresOrderByToStartWithItsOwnExpressions() { + seed() + assertFailsWith { + select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY amount") + } + } + + @Test + fun distinctOnPermitsAnOrderByThatOnlyMatchesItsLeadingExpressions() { + seed() + val result = select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, region, amount DESC") + assertEquals(2, result.rows.size) + } + + @Test + fun distinctOnWithNoOrderByAtAllIsAllowed() { + seed() + val result = select("SELECT DISTINCT ON (dept) dept FROM sales") + assertEquals(2, result.rows.size) + } + + @Test + fun distinctOnRejectsGroupBy() { + seed() + assertFailsWith { + select("SELECT DISTINCT ON (dept) dept, SUM(amount) FROM sales GROUP BY dept") + } + } + + @Test + fun distinctOnRejectsAnUnknownColumn() { + seed() + assertFailsWith { + select("SELECT DISTINCT ON (nope) dept FROM sales") + } + } + + @Test + fun distinctOnRejectsAWindowFunction() { + seed() + assertFailsWith { + select("SELECT DISTINCT ON (ROW_NUMBER() OVER (ORDER BY amount)) dept FROM sales") + } + } + + @Test + fun explainShowsUniqueForDistinct() { + seed() + val plan = (run("EXPLAIN SELECT DISTINCT dept FROM sales").single() as ExplainResult).plan + assertEquals("Unique", plan.first()) + } + + @Test + fun explainShowsUniqueDistinctOnForDistinctOn() { + seed() + val plan = (run("EXPLAIN SELECT DISTINCT ON (dept) dept FROM sales ORDER BY dept").single() as ExplainResult).plan + assertEquals("Unique (DISTINCT ON)", plan.first()) + } + + @Test + fun explainShowsLimitOutermostAndOffsetJustInside() { + seed() + val plan = (run("EXPLAIN SELECT dept FROM sales ORDER BY dept LIMIT 5 OFFSET 2").single() as ExplainResult).plan + assertEquals("Limit 5", plan[0]) + assertEquals(" Offset 2", plan[1]) + } +} diff --git a/src/test/kotlin/kestrel/EvaluatorTest.kt b/src/test/kotlin/kestrel/EvaluatorTest.kt index e7962ae..0702abc 100644 --- a/src/test/kotlin/kestrel/EvaluatorTest.kt +++ b/src/test/kotlin/kestrel/EvaluatorTest.kt @@ -1,12 +1,22 @@ package kestrel +import java.math.BigDecimal +import java.time.LocalDate import kestrel.catalog.Column import kestrel.catalog.TableSchema import kestrel.engine.Evaluator +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseBranch +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression import kestrel.sql.ast.ColumnExpression import kestrel.sql.ast.Expression +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UnaryOperator @@ -106,4 +116,448 @@ class EvaluatorTest { fun logicalOperatorOnNumberFails() { assertFailsWith { evalConstant(bin(BinaryOperator.And, intLit(1), trueLit)) } } + + @Test + fun stringFunctionsTransformText() { + assertEquals(Value.TextValue("ADA"), evalConstant(CallExpression("UPPER", listOf(textLit("ada"))))) + assertEquals(Value.TextValue("ada"), evalConstant(CallExpression("LOWER", listOf(textLit("ADA"))))) + assertEquals(Value.IntegerValue(3), evalConstant(CallExpression("LENGTH", listOf(textLit("ada"))))) + assertEquals(Value.TextValue("ada"), evalConstant(CallExpression("TRIM", listOf(textLit(" ada "))))) + } + + @Test + fun substrHandlesTwoAndThreeArgumentForms() { + assertEquals( + Value.TextValue("ell"), + evalConstant(CallExpression("SUBSTR", listOf(textLit("hello"), intLit(2), intLit(3)))) + ) + assertEquals(Value.TextValue("llo"), evalConstant(CallExpression("SUBSTR", listOf(textLit("hello"), intLit(3))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("SUBSTR", listOf(textLit("hello"), intLit(99))))) + } + + @Test + fun replaceAndConcat() { + assertEquals( + Value.TextValue("hexxo"), + evalConstant(CallExpression("REPLACE", listOf(textLit("hello"), textLit("l"), textLit("x")))) + ) + assertEquals( + Value.TextValue("ab-cd"), + evalConstant(CallExpression("CONCAT", listOf(textLit("ab"), textLit("-"), textLit("cd")))) + ) + } + + @Test + fun numericFunctionsComputeCorrectly() { + assertEquals(Value.IntegerValue(4), evalConstant(CallExpression("ABS", listOf(intLit(-4))))) + assertEquals(Value.RealValue(4.0), evalConstant(CallExpression("ABS", listOf(realLit(-4.0))))) + assertEquals(Value.IntegerValue(3), evalConstant(CallExpression("CEIL", listOf(realLit(2.1))))) + assertEquals(Value.IntegerValue(2), evalConstant(CallExpression("FLOOR", listOf(realLit(2.9))))) + assertEquals(Value.RealValue(2.35), evalConstant(CallExpression("ROUND", listOf(realLit(2.346), intLit(2))))) + assertEquals(Value.IntegerValue(5), evalConstant(CallExpression("ROUND", listOf(intLit(5))))) + } + + @Test + fun nullHandlingFunctionsSeeRawNulls() { + assertEquals(Value.IntegerValue(7), evalConstant(CallExpression("COALESCE", listOf(nullLit, nullLit, intLit(7))))) + assertEquals(Value.NullValue, evalConstant(CallExpression("COALESCE", listOf(nullLit, nullLit)))) + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("IFNULL", listOf(intLit(1), intLit(2))))) + assertEquals(Value.IntegerValue(2), evalConstant(CallExpression("IFNULL", listOf(nullLit, intLit(2))))) + assertEquals(Value.NullValue, evalConstant(CallExpression("NULLIF", listOf(intLit(5), intLit(5))))) + assertEquals(Value.IntegerValue(5), evalConstant(CallExpression("NULLIF", listOf(intLit(5), intLit(6))))) + assertEquals(Value.TextValue("NULL"), evalConstant(CallExpression("TYPEOF", listOf(nullLit)))) + assertEquals(Value.TextValue("INTEGER"), evalConstant(CallExpression("TYPEOF", listOf(intLit(1))))) + } + + @Test + fun mostFunctionsPropagateNullArguments() { + assertEquals(Value.NullValue, evalConstant(CallExpression("UPPER", listOf(nullLit)))) + assertEquals(Value.NullValue, evalConstant(CallExpression("ABS", listOf(nullLit)))) + } + + @Test + fun functionArityIsEnforced() { + assertFailsWith { evalConstant(CallExpression("UPPER", listOf(textLit("a"), textLit("b")))) } + assertFailsWith { evalConstant(CallExpression("SUBSTR", listOf(textLit("a")))) } + } + + @Test + fun unknownFunctionFails() { + assertFailsWith { evalConstant(CallExpression("NOPE", listOf(intLit(1)))) } + } + + @Test + fun currentDateReturnsTodaysWallClockDate() { + assertEquals(Value.DateValue(LocalDate.now()), evalConstant(CallExpression("CURRENT_DATE", emptyList()))) + } + + @Test + fun dateComponentFunctionsExtractYearMonthDay() { + val date = dateLit(LocalDate.of(2024, 3, 15)) + assertEquals(Value.IntegerValue(2024), evalConstant(CallExpression("YEAR", listOf(date)))) + assertEquals(Value.IntegerValue(3), evalConstant(CallExpression("MONTH", listOf(date)))) + assertEquals(Value.IntegerValue(15), evalConstant(CallExpression("DAY", listOf(date)))) + assertEquals(Value.IntegerValue(5), evalConstant(CallExpression("DAYOFWEEK", listOf(date)))) + } + + @Test + fun dateComponentFunctionsPropagateNullAndRejectNonDateArguments() { + assertEquals(Value.NullValue, evalConstant(CallExpression("YEAR", listOf(nullLit)))) + assertFailsWith { evalConstant(CallExpression("MONTH", listOf(intLit(1)))) } + } + + @Test + fun dateAddAndDateSubShiftByWholeDaysInEitherDirection() { + val date = dateLit(LocalDate.of(2024, 3, 15)) + assertEquals( + Value.DateValue(LocalDate.of(2024, 3, 25)), + evalConstant(CallExpression("DATE_ADD", listOf(date, intLit(10)))) + ) + assertEquals( + Value.DateValue(LocalDate.of(2024, 3, 5)), + evalConstant(CallExpression("DATE_SUB", listOf(date, intLit(10)))) + ) + // Crossing a month boundary exercises java.time's own carry logic rather than hand-rolled arithmetic. + assertEquals( + Value.DateValue(LocalDate.of(2024, 4, 2)), + evalConstant(CallExpression("DATE_ADD", listOf(date, intLit(18)))) + ) + assertEquals( + Value.DateValue(LocalDate.of(2024, 3, 9)), + evalConstant(CallExpression("DATE_ADD", listOf(date, intLit(-6)))) + ) + } + + @Test + fun datediffReturnsTheSignedDayCountBetweenTwoDates() { + val later = dateLit(LocalDate.of(2024, 3, 20)) + val earlier = dateLit(LocalDate.of(2024, 3, 15)) + assertEquals(Value.IntegerValue(5), evalConstant(CallExpression("DATEDIFF", listOf(later, earlier)))) + assertEquals(Value.IntegerValue(-5), evalConstant(CallExpression("DATEDIFF", listOf(earlier, later)))) + assertEquals(Value.IntegerValue(0), evalConstant(CallExpression("DATEDIFF", listOf(later, later)))) + } + + @Test + fun repeatBuildsTheStringNTimesAndTreatsNonPositiveCountsAsEmpty() { + assertEquals(Value.TextValue("ababab"), evalConstant(CallExpression("REPEAT", listOf(textLit("ab"), intLit(3))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("REPEAT", listOf(textLit("ab"), intLit(0))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("REPEAT", listOf(textLit("ab"), intLit(-1))))) + } + + @Test + fun lpadAndRpadPadToTheTargetLengthOrTruncateWhenAlreadyLongEnough() { + assertEquals(Value.TextValue("--ab"), evalConstant(CallExpression("LPAD", listOf(textLit("ab"), intLit(4), textLit("-"))))) + assertEquals(Value.TextValue("ab--"), evalConstant(CallExpression("RPAD", listOf(textLit("ab"), intLit(4), textLit("-"))))) + assertEquals(Value.TextValue(" ab"), evalConstant(CallExpression("LPAD", listOf(textLit("ab"), intLit(4))))) + assertEquals(Value.TextValue("he"), evalConstant(CallExpression("LPAD", listOf(textLit("hello"), intLit(2), textLit("-"))))) + assertEquals(Value.TextValue("he"), evalConstant(CallExpression("RPAD", listOf(textLit("hello"), intLit(2), textLit("-"))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("LPAD", listOf(textLit("hello"), intLit(0), textLit("-"))))) + } + + @Test + fun lpadCyclesAMultiCharacterPadStringAndTrimsItToExactlyFitTheTargetLength() { + assertEquals(Value.TextValue("xyxab"), evalConstant(CallExpression("LPAD", listOf(textLit("ab"), intLit(5), textLit("xy"))))) + } + + @Test + fun padFunctionsRejectAnEmptyPadStringWhenPaddingIsActuallyNeeded() { + assertFailsWith { + evalConstant(CallExpression("LPAD", listOf(textLit("ab"), intLit(4), textLit("")))) + } + } + + @Test + fun instrReturnsAOneBasedPositionOrZeroWhenNotFound() { + assertEquals(Value.IntegerValue(3), evalConstant(CallExpression("INSTR", listOf(textLit("hello"), textLit("llo"))))) + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("INSTR", listOf(textLit("hello"), textLit("h"))))) + assertEquals(Value.IntegerValue(0), evalConstant(CallExpression("INSTR", listOf(textLit("hello"), textLit("z"))))) + } + + @Test + fun powerStaysExactForTwoIntegersButFallsBackToRealOtherwise() { + assertEquals(Value.IntegerValue(1024), evalConstant(CallExpression("POWER", listOf(intLit(2), intLit(10))))) + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("POWER", listOf(intLit(5), intLit(0))))) + assertEquals(Value.RealValue(0.25), evalConstant(CallExpression("POWER", listOf(intLit(2), intLit(-2))))) + assertEquals(Value.RealValue(6.25), evalConstant(CallExpression("POWER", listOf(realLit(2.5), intLit(2))))) + } + + @Test + fun modComputesRemainderAndRejectsDivisionByZero() { + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("MOD", listOf(intLit(7), intLit(3))))) + assertEquals(Value.IntegerValue(-1), evalConstant(CallExpression("MOD", listOf(intLit(-7), intLit(3))))) + assertEquals(Value.RealValue(0.5), evalConstant(CallExpression("MOD", listOf(realLit(2.5), intLit(1))))) + assertFailsWith { evalConstant(CallExpression("MOD", listOf(intLit(1), intLit(0)))) } + } + + @Test + fun reverseFlipsTheCharactersOfAString() { + assertEquals(Value.TextValue("cba"), evalConstant(CallExpression("REVERSE", listOf(textLit("abc"))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("REVERSE", listOf(textLit(""))))) + assertFailsWith { evalConstant(CallExpression("REVERSE", listOf(intLit(1)))) } + } + + @Test + fun leftAndRightTakeCharactersFromEachEndAndClampToTheStringsLength() { + assertEquals(Value.TextValue("hel"), evalConstant(CallExpression("LEFT", listOf(textLit("hello"), intLit(3))))) + assertEquals(Value.TextValue("llo"), evalConstant(CallExpression("RIGHT", listOf(textLit("hello"), intLit(3))))) + assertEquals(Value.TextValue("hello"), evalConstant(CallExpression("LEFT", listOf(textLit("hello"), intLit(100))))) + assertEquals(Value.TextValue("hello"), evalConstant(CallExpression("RIGHT", listOf(textLit("hello"), intLit(100))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("LEFT", listOf(textLit("hello"), intLit(0))))) + assertEquals(Value.TextValue(""), evalConstant(CallExpression("RIGHT", listOf(textLit("hello"), intLit(-1))))) + } + + @Test + fun signReturnsMinusOneZeroOrOneAcrossEveryNumericType() { + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("SIGN", listOf(intLit(5))))) + assertEquals(Value.IntegerValue(-1), evalConstant(CallExpression("SIGN", listOf(intLit(-5))))) + assertEquals(Value.IntegerValue(0), evalConstant(CallExpression("SIGN", listOf(intLit(0))))) + assertEquals(Value.IntegerValue(1), evalConstant(CallExpression("SIGN", listOf(realLit(0.1))))) + assertEquals(Value.IntegerValue(-1), evalConstant(CallExpression("SIGN", listOf(realLit(-0.1))))) + val negativeDecimal = LiteralExpression(Value.DecimalValue(BigDecimal("-3.50"))) + assertEquals(Value.IntegerValue(-1), evalConstant(CallExpression("SIGN", listOf(negativeDecimal)))) + assertFailsWith { evalConstant(CallExpression("SIGN", listOf(textLit("x")))) } + } + + @Test + fun truncCutsDigitsTowardZeroWithoutRoundingUnlikeRound() { + assertEquals(Value.IntegerValue(4), evalConstant(CallExpression("TRUNC", listOf(intLit(4))))) + assertEquals(Value.RealValue(1.9), evalConstant(CallExpression("TRUNC", listOf(realLit(1.99), intLit(1))))) + assertEquals(Value.RealValue(-1.9), evalConstant(CallExpression("TRUNC", listOf(realLit(-1.99), intLit(1))))) + val decimal = LiteralExpression(Value.DecimalValue(BigDecimal("1.999"))) + assertEquals( + Value.DecimalValue(BigDecimal("1.99")), + evalConstant(CallExpression("TRUNC", listOf(decimal, intLit(2)))) + ) + // TRUNC never rounds up, exactly the case where it must differ from ROUND's HALF_UP default. + assertEquals(Value.RealValue(1.0), evalConstant(CallExpression("TRUNC", listOf(realLit(1.9))))) + assertEquals(Value.RealValue(2.0), evalConstant(CallExpression("ROUND", listOf(realLit(1.9))))) + } + + @Test + fun greatestAndLeastPickTheOrderingExtremeAcrossMixedNumericTypes() { + assertEquals( + Value.IntegerValue(7), + evalConstant(CallExpression("GREATEST", listOf(intLit(3), intLit(7), intLit(1)))) + ) + assertEquals( + Value.IntegerValue(1), + evalConstant(CallExpression("LEAST", listOf(intLit(3), intLit(7), intLit(1)))) + ) + assertEquals( + Value.RealValue(7.5), + evalConstant(CallExpression("GREATEST", listOf(intLit(3), realLit(7.5), intLit(1)))) + ) + assertEquals(Value.TextValue("banana"), evalConstant(CallExpression("GREATEST", listOf(textLit("apple"), textLit("banana"))))) + } + + @Test + fun greatestAndLeastPropagateNullAndRequireAtLeastTwoArguments() { + assertEquals(Value.NullValue, evalConstant(CallExpression("GREATEST", listOf(intLit(1), nullLit)))) + assertEquals(Value.NullValue, evalConstant(CallExpression("LEAST", listOf(nullLit, intLit(1))))) + assertFailsWith { evalConstant(CallExpression("GREATEST", listOf(intLit(1)))) } + } + + @Test + fun functionRequiringTextRejectsWrongType() { + assertFailsWith { evalConstant(CallExpression("UPPER", listOf(intLit(1)))) } + } + + @Test + fun isNullDetectsNullness() { + assertEquals(Value.BooleanValue(true), evalConstant(IsNullExpression(nullLit, negated = false))) + assertEquals(Value.BooleanValue(false), evalConstant(IsNullExpression(intLit(1), negated = false))) + assertEquals(Value.BooleanValue(true), evalConstant(IsNullExpression(intLit(1), negated = true))) + assertEquals(Value.BooleanValue(false), evalConstant(IsNullExpression(nullLit, negated = true))) + } + + @Test + fun inMatchesAnyListMember() { + assertEquals( + Value.BooleanValue(true), + evalConstant(InExpression(intLit(2), listOf(intLit(1), intLit(2), intLit(3)), negated = false)) + ) + assertEquals( + Value.BooleanValue(false), + evalConstant(InExpression(intLit(9), listOf(intLit(1), intLit(2)), negated = false)) + ) + assertEquals( + Value.BooleanValue(false), + evalConstant(InExpression(intLit(2), listOf(intLit(1), intLit(2)), negated = true)) + ) + } + + @Test + fun inWithNullOperandIsNull() { + assertEquals(Value.NullValue, evalConstant(InExpression(nullLit, listOf(intLit(1)), negated = false))) + } + + @Test + fun inWithUnmatchedNullInListIsNullNotFalse() { + assertEquals( + Value.NullValue, + evalConstant(InExpression(intLit(9), listOf(intLit(1), nullLit), negated = false)) + ) + } + + @Test + fun betweenIsInclusiveOnBothEnds() { + assertEquals(Value.BooleanValue(true), evalConstant(BetweenExpression(intLit(18), intLit(18), intLit(65), negated = false))) + assertEquals(Value.BooleanValue(true), evalConstant(BetweenExpression(intLit(65), intLit(18), intLit(65), negated = false))) + assertEquals(Value.BooleanValue(false), evalConstant(BetweenExpression(intLit(17), intLit(18), intLit(65), negated = false))) + assertEquals(Value.BooleanValue(true), evalConstant(BetweenExpression(intLit(17), intLit(18), intLit(65), negated = true))) + } + + @Test + fun likeSupportsPercentAndUnderscoreWildcards() { + assertEquals(Value.BooleanValue(true), evalConstant(LikeExpression(textLit("hello"), textLit("h%"), negated = false))) + assertEquals(Value.BooleanValue(true), evalConstant(LikeExpression(textLit("hello"), textLit("h_llo"), negated = false))) + assertEquals(Value.BooleanValue(false), evalConstant(LikeExpression(textLit("hello"), textLit("world"), negated = false))) + assertEquals(Value.BooleanValue(false), evalConstant(LikeExpression(textLit("hello"), textLit("h%"), negated = true))) + } + + @Test + fun likeEscapesRegexMetacharactersInPattern() { + assertEquals(Value.BooleanValue(true), evalConstant(LikeExpression(textLit("a.b"), textLit("a.b"), negated = false))) + assertEquals(Value.BooleanValue(false), evalConstant(LikeExpression(textLit("axb"), textLit("a.b"), negated = false))) + } + + @Test + fun searchedCaseReturnsFirstMatchingBranch() { + val expression = CaseExpression( + operand = null, + branches = listOf( + CaseBranch(bin(BinaryOperator.Less, intLit(10), intLit(5)), textLit("small")), + CaseBranch(bin(BinaryOperator.GreaterEqual, intLit(10), intLit(5)), textLit("big")) + ), + elseExpression = textLit("none") + ) + assertEquals(Value.TextValue("big"), evalConstant(expression)) + } + + @Test + fun simpleCaseComparesOperandAgainstBranches() { + val expression = CaseExpression( + operand = intLit(2), + branches = listOf(CaseBranch(intLit(1), textLit("one")), CaseBranch(intLit(2), textLit("two"))), + elseExpression = textLit("other") + ) + assertEquals(Value.TextValue("two"), evalConstant(expression)) + } + + @Test + fun caseFallsThroughToElseWhenNoBranchMatches() { + val expression = CaseExpression( + operand = null, + branches = listOf(CaseBranch(falseLit, textLit("nope"))), + elseExpression = textLit("fallback") + ) + assertEquals(Value.TextValue("fallback"), evalConstant(expression)) + } + + @Test + fun caseWithoutElseYieldsNullWhenNoBranchMatches() { + val expression = CaseExpression(operand = null, branches = listOf(CaseBranch(falseLit, textLit("nope"))), elseExpression = null) + assertEquals(Value.NullValue, evalConstant(expression)) + } + + private fun cast(operand: Expression, type: ColumnType, precision: Int? = null, scale: Int? = null): Expression = + CastExpression(operand, type, precision, scale) + + @Test + fun castTextToIntegerRealAndBoolean() { + assertEquals(Value.IntegerValue(42), evalConstant(cast(textLit("42"), ColumnType.Integer))) + assertEquals(Value.RealValue(3.5), evalConstant(cast(textLit("3.5"), ColumnType.Real))) + assertEquals(Value.BooleanValue(true), evalConstant(cast(textLit("true"), ColumnType.Boolean))) + assertEquals(Value.BooleanValue(false), evalConstant(cast(textLit("FALSE"), ColumnType.Boolean))) + } + + @Test + fun castTextToDecimalAppliesScale() { + val result = evalConstant(cast(textLit("19.999"), ColumnType.Decimal, 10, 2)) as Value.DecimalValue + assertEquals("20.00", result.value.toPlainString()) + } + + @Test + fun castTextToDateParsesIsoFormat() { + val result = evalConstant(cast(textLit("2024-01-15"), ColumnType.Date)) + assertEquals(Value.DateValue(LocalDate.of(2024, 1, 15)), result) + } + + @Test + fun castNumbersAndDatesToText() { + assertEquals(Value.TextValue("42"), evalConstant(cast(intLit(42), ColumnType.Text))) + assertEquals( + Value.TextValue("2024-01-15"), + evalConstant(cast(LiteralExpression(Value.DateValue(LocalDate.of(2024, 1, 15))), ColumnType.Text)) + ) + } + + @Test + fun castOfNullIsAlwaysNull() { + assertEquals(Value.NullValue, evalConstant(cast(nullLit, ColumnType.Integer))) + assertEquals(Value.NullValue, evalConstant(cast(nullLit, ColumnType.Decimal))) + } + + @Test + fun castRejectsUnparseableText() { + assertFailsWith { evalConstant(cast(textLit("not a number"), ColumnType.Integer)) } + assertFailsWith { evalConstant(cast(textLit("not a date"), ColumnType.Date)) } + assertFailsWith { evalConstant(cast(textLit("nope"), ColumnType.Boolean)) } + } + + @Test + fun decimalOverflowBeyondDeclaredPrecisionFails() { + assertFailsWith { evalConstant(cast(textLit("123456"), ColumnType.Decimal, 5, 0)) } + } + + @Test + fun decimalArithmeticIsExactUnlikeDoubles() { + val a = cast(textLit("0.1"), ColumnType.Decimal, 10, 2) + val b = cast(textLit("0.2"), ColumnType.Decimal, 10, 2) + val sum = evalConstant(bin(BinaryOperator.Add, a, b)) as Value.DecimalValue + assertEquals("0.30", sum.value.toPlainString()) + } + + @Test + fun decimalEqualityIgnoresTrailingZeroScale() { + val fifteenTenths = LiteralExpression(Value.DecimalValue(BigDecimal("1.50"))) + val fifteen = LiteralExpression(Value.DecimalValue(BigDecimal("1.5"))) + assertEquals(Value.BooleanValue(true), evalConstant(bin(BinaryOperator.Equal, fifteenTenths, fifteen))) + } + + @Test + fun decimalDivisionByZeroFails() { + val a = cast(textLit("1"), ColumnType.Decimal, 10, 2) + val zero = cast(textLit("0"), ColumnType.Decimal, 10, 2) + assertFailsWith { evalConstant(bin(BinaryOperator.Divide, a, zero)) } + } + + @Test + fun decimalComparesCorrectlyAgainstIntegerAndReal() { + val decimal = LiteralExpression(Value.DecimalValue(BigDecimal("19.99"))) + assertEquals(Value.BooleanValue(true), evalConstant(bin(BinaryOperator.Less, decimal, intLit(20)))) + assertEquals(Value.BooleanValue(true), evalConstant(bin(BinaryOperator.Greater, decimal, realLit(19.98)))) + } + + private fun dateLit(value: LocalDate): Expression = LiteralExpression(Value.DateValue(value)) + + @Test + fun addingAndSubtractingIntegerDaysFromADate() { + val start = dateLit(LocalDate.of(2024, 1, 1)) + assertEquals(Value.DateValue(LocalDate.of(2024, 1, 6)), evalConstant(bin(BinaryOperator.Add, start, intLit(5)))) + assertEquals(Value.DateValue(LocalDate.of(2023, 12, 27)), evalConstant(bin(BinaryOperator.Subtract, start, intLit(5)))) + } + + @Test + fun subtractingTwoDatesYieldsIntegerDayDifference() { + val later = dateLit(LocalDate.of(2024, 1, 10)) + val earlier = dateLit(LocalDate.of(2024, 1, 1)) + assertEquals(Value.IntegerValue(9), evalConstant(bin(BinaryOperator.Subtract, later, earlier))) + } + + @Test + fun multiplyingADateFails() { + val start = dateLit(LocalDate.of(2024, 1, 1)) + assertFailsWith { evalConstant(bin(BinaryOperator.Multiply, start, intLit(2))) } + } } diff --git a/src/test/kotlin/kestrel/ExecutorTest.kt b/src/test/kotlin/kestrel/ExecutorTest.kt index a0713d5..17fa8ba 100644 --- a/src/test/kotlin/kestrel/ExecutorTest.kt +++ b/src/test/kotlin/kestrel/ExecutorTest.kt @@ -77,6 +77,72 @@ class ExecutorTest { assertEquals(listOf(text("grace")), result.rows[0]) } + @Test + fun offsetSkipsLeadingRowsWithoutALimit() { + seedUsers() + val result = select("SELECT name FROM users ORDER BY age ASC OFFSET 1") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + } + + @Test + fun limitAndOffsetTogetherPageThroughTheOrderedResult() { + seedUsers() + val result = select("SELECT name FROM users ORDER BY age ASC LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun offsetPastTheEndOfTheResultReturnsNoRows() { + seedUsers() + val result = select("SELECT name FROM users ORDER BY age ASC OFFSET 100") + assertEquals(emptyList>(), result.rows) + } + + @Test + fun offsetWrittenBeforeLimitStillAppliesLimitAfterTheSkip() { + seedUsers() + val result = select("SELECT name FROM users ORDER BY age ASC OFFSET 1 LIMIT 1") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun orderByMultipleColumnsBreaksTiesWithTheSecondKey() { + run("CREATE TABLE staff (id INTEGER PRIMARY KEY, dept TEXT NOT NULL, name TEXT NOT NULL)") + run( + "INSERT INTO staff VALUES " + + "(1, 'eng', 'guido'), (2, 'eng', 'ada'), (3, 'ops', 'grace'), (4, 'eng', 'linus')" + ) + val result = select("SELECT dept, name FROM staff ORDER BY dept ASC, name ASC") + assertEquals( + listOf( + listOf(text("eng"), text("ada")), + listOf(text("eng"), text("guido")), + listOf(text("eng"), text("linus")), + listOf(text("ops"), text("grace")) + ), + result.rows + ) + } + + @Test + fun orderByMultipleColumnsCanMixAscendingAndDescending() { + run("CREATE TABLE staff (id INTEGER PRIMARY KEY, dept TEXT NOT NULL, name TEXT NOT NULL)") + run( + "INSERT INTO staff VALUES " + + "(1, 'eng', 'guido'), (2, 'eng', 'ada'), (3, 'ops', 'grace'), (4, 'eng', 'linus')" + ) + val result = select("SELECT dept, name FROM staff ORDER BY dept ASC, name DESC") + assertEquals( + listOf( + listOf(text("eng"), text("linus")), + listOf(text("eng"), text("guido")), + listOf(text("eng"), text("ada")), + listOf(text("ops"), text("grace")) + ), + result.rows + ) + } + @Test fun projectionSupportsExpressionsAndAliases() { seedUsers() @@ -197,4 +263,22 @@ class ExecutorTest { val result = select("SELECT name FROM users WHERE age > 40 AND age < 80") assertEquals(listOf(listOf(text("linus"))), result.rows) } + + @Test + fun temporaryTableSupportsFullCrudJustLikeARegularTable() { + run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, note TEXT NOT NULL)") + run("INSERT INTO scratch VALUES (1, 'first'), (2, 'second')") + run("UPDATE scratch SET note = 'updated' WHERE id = 1") + run("DELETE FROM scratch WHERE id = 2") + val result = select("SELECT id, note FROM scratch") + assertEquals(listOf(listOf(int(1), text("updated"))), result.rows) + } + + @Test + fun temporaryTableEnforcesItsOwnPrimaryKeyAndCanBeIndexed() { + run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, tag TEXT)") + run("CREATE INDEX scratchTag ON scratch (tag)") + run("INSERT INTO scratch VALUES (1, 'a')") + assertFailsWith { run("INSERT INTO scratch VALUES (1, 'b')") } + } } diff --git a/src/test/kotlin/kestrel/ExpressionLanguageTest.kt b/src/test/kotlin/kestrel/ExpressionLanguageTest.kt new file mode 100644 index 0000000..bbc960c --- /dev/null +++ b/src/test/kotlin/kestrel/ExpressionLanguageTest.kt @@ -0,0 +1,325 @@ +package kestrel + +import java.math.BigDecimal +import java.time.LocalDate +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.ExplainResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class ExpressionLanguageTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seedUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER, bio TEXT)") + run( + "INSERT INTO users VALUES " + + "(1, 'ada', 36, 'mathematician'), (2, 'linus', 54, NULL), " + + "(3, 'grace', 85, 'rear admiral')" + ) + } + + @Test + fun scalarFunctionInProjection() { + seedUsers() + val result = select("SELECT UPPER(name) FROM users ORDER BY id ASC") + assertEquals(listOf("upper"), result.columns) + assertEquals(listOf(text("ADA"), text("LINUS"), text("GRACE")), result.rows.map { it[0] }) + } + + @Test + fun scalarFunctionInWhereClause() { + seedUsers() + val result = select("SELECT name FROM users WHERE LENGTH(name) > 4 ORDER BY name ASC") + assertEquals(listOf(listOf(text("grace")), listOf(text("linus"))), result.rows) + } + + @Test + fun scalarFunctionUsableInUpdateAssignment() { + seedUsers() + run("UPDATE users SET name = UPPER(name) WHERE id = 1") + val result = select("SELECT name FROM users WHERE id = 1") + assertEquals(listOf(listOf(text("ADA"))), result.rows) + } + + @Test + fun nestedFunctionCalls() { + seedUsers() + val result = select("SELECT SUBSTR(UPPER(name), 1, 1) FROM users WHERE id = 1") + assertEquals(listOf(listOf(text("A"))), result.rows) + } + + @Test + fun isNullFiltersRowsWithMissingValues() { + seedUsers() + val result = select("SELECT name FROM users WHERE bio IS NULL") + assertEquals(listOf(listOf(text("linus"))), result.rows) + val negated = select("SELECT COUNT(*) FROM users WHERE bio IS NOT NULL") + assertEquals(listOf(listOf(int(2))), negated.rows) + } + + @Test + fun inClauseMatchesAgainstLiteralSet() { + seedUsers() + val result = select("SELECT name FROM users WHERE age IN (36, 85) ORDER BY age ASC") + assertEquals(listOf(listOf(text("ada")), listOf(text("grace"))), result.rows) + } + + @Test + fun notInExcludesMatchingRows() { + seedUsers() + val result = select("SELECT name FROM users WHERE age NOT IN (36, 85)") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun betweenSelectsInclusiveRange() { + seedUsers() + val result = select("SELECT name FROM users WHERE age BETWEEN 40 AND 90 ORDER BY age ASC") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + } + + @Test + fun likeMatchesWildcardPattern() { + seedUsers() + val result = select("SELECT name FROM users WHERE bio LIKE '%admiral' ") + assertEquals(listOf(listOf(text("grace"))), result.rows) + } + + @Test + fun caseExpressionInProjection() { + seedUsers() + val result = select( + "SELECT name, CASE WHEN age < 50 THEN 'young' WHEN age < 80 THEN 'mid' ELSE 'senior' END " + + "FROM users ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(text("ada"), text("young")), + listOf(text("linus"), text("mid")), + listOf(text("grace"), text("senior")) + ), + result.rows + ) + } + + @Test + fun caseExpressionUsableInWhereClause() { + seedUsers() + val result = select( + "SELECT name FROM users WHERE CASE WHEN age > 40 THEN TRUE ELSE FALSE END ORDER BY name ASC" + ) + assertEquals(listOf(listOf(text("grace")), listOf(text("linus"))), result.rows) + } + + @Test + fun combiningPredicatesWithLogicalOperators() { + seedUsers() + val result = select( + "SELECT name FROM users WHERE age BETWEEN 30 AND 60 AND name LIKE 'a%'" + ) + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun aggregateOfScalarFunctionResult() { + seedUsers() + val result = select("SELECT SUM(LENGTH(name)) FROM users") + assertEquals(listOf(listOf(int(3 + 5 + 5))), result.rows) + } + + @Test + fun groupByWithFunctionOnGroupedColumn() { + run("CREATE TABLE t (category TEXT, amount INTEGER)") + run("INSERT INTO t VALUES ('Eng', 10), ('eng', 20), ('Ops', 5)") + val result = select("SELECT LOWER(category), SUM(amount) FROM t GROUP BY LOWER(category) ORDER BY LOWER(category) ASC") + assertEquals( + listOf(listOf(text("eng"), int(30)), listOf(text("ops"), int(5))), + result.rows + ) + } + + @Test + fun scalarFunctionNotAllowedAsAggregateGroupingViolation() { + seedUsers() + assertFailsWith { + select("SELECT UPPER(name), COUNT(*) FROM users GROUP BY age") + } + } + + @Test + fun newPredicateFormsCoexistWithIndexPushdownOnOtherConjuncts() { + seedUsers() + run("CREATE INDEX idxId ON users (id)") + val plan = (run("EXPLAIN SELECT name FROM users WHERE id = 1 AND age BETWEEN 1 AND 100").single() as ExplainResult).plan + assertTrue(plan.any { it.contains("Index Scan") }, "expected an index scan but was: $plan") + val result = select("SELECT name FROM users WHERE id = 1 AND age BETWEEN 1 AND 100") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun unknownFunctionNameReportsClearError() { + seedUsers() + assertFailsWith { select("SELECT NOSUCHFUNC(name) FROM users") } + } + + private fun decimal(value: String) = Value.DecimalValue(BigDecimal(value)) + private fun date(value: String) = Value.DateValue(LocalDate.parse(value)) + + @Test + fun decimalColumnCoercesIntegerAndRealButRejectsBareText() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount DECIMAL(10, 2))") + run("INSERT INTO orders VALUES (1, 5), (2, 5.5)") + val result = select("SELECT amount FROM orders ORDER BY id ASC") + assertEquals(listOf(listOf(decimal("5.00")), listOf(decimal("5.50"))), result.rows) + assertFailsWith { run("INSERT INTO orders VALUES (3, '5.00')") } + } + + @Test + fun dateColumnRequiresExplicitCastFromText() { + run("CREATE TABLE events (id INTEGER PRIMARY KEY, happened DATE)") + assertFailsWith { run("INSERT INTO events VALUES (1, '2024-01-01')") } + run("INSERT INTO events VALUES (1, CAST('2024-01-01' AS DATE))") + val result = select("SELECT happened FROM events") + assertEquals(listOf(listOf(date("2024-01-01"))), result.rows) + } + + @Test + fun decimalSumAndAverageStayExactAcrossGroupBy() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, dept TEXT, amount DECIMAL(10, 2))") + run( + "INSERT INTO orders VALUES " + + "(1, 'eng', 10.10), (2, 'eng', 10.20), (3, 'eng', 10.30), (4, 'ops', 1.11)" + ) + val result = select("SELECT dept, SUM(amount), AVG(amount) FROM orders GROUP BY dept ORDER BY dept ASC") + assertEquals( + listOf( + listOf(text("eng"), decimal("30.60"), decimal("10.20")), + listOf(text("ops"), decimal("1.11"), decimal("1.11")) + ), + result.rows + ) + } + + @Test + fun dateArithmeticEndToEndThroughSelectAndWhere() { + run("CREATE TABLE events (id INTEGER PRIMARY KEY, happened DATE)") + run( + "INSERT INTO events VALUES " + + "(1, CAST('2024-01-01' AS DATE)), (2, CAST('2024-06-01' AS DATE))" + ) + val projected = select("SELECT happened + 7 FROM events ORDER BY id ASC") + assertEquals(listOf(listOf(date("2024-01-08")), listOf(date("2024-06-08"))), projected.rows) + + val filtered = select( + "SELECT id FROM events WHERE happened > CAST('2024-03-01' AS DATE)" + ) + assertEquals(listOf(listOf(int(2))), filtered.rows) + } + + @Test + fun castExpressionWorksInProjectionAndWhereAgainstOrdinaryColumns() { + seedUsers() + val projected = select("SELECT CAST(age AS TEXT) FROM users WHERE id = 1") + assertEquals(listOf(listOf(text("36"))), projected.rows) + + val filtered = select("SELECT name FROM users WHERE CAST(age AS REAL) > 40.0 ORDER BY name ASC") + assertEquals(listOf(listOf(text("grace")), listOf(text("linus"))), filtered.rows) + } + + @Test + fun dateComponentFunctionsEndToEndThroughSelectAndWhere() { + run("CREATE TABLE events (id INTEGER PRIMARY KEY, happened DATE)") + run( + "INSERT INTO events VALUES " + + "(1, CAST('2024-03-15' AS DATE)), (2, CAST('2023-11-02' AS DATE))" + ) + val projected = select("SELECT YEAR(happened), MONTH(happened), DAY(happened) FROM events ORDER BY id ASC") + assertEquals( + listOf(listOf(int(2024), int(3), int(15)), listOf(int(2023), int(11), int(2))), + projected.rows + ) + + val filtered = select("SELECT id FROM events WHERE YEAR(happened) = 2023") + assertEquals(listOf(listOf(int(2))), filtered.rows) + + val today = select("SELECT CURRENT_DATE() FROM events WHERE id = 1") + assertEquals(listOf(listOf(date(LocalDate.now().toString()))), today.rows) + } + + @Test + fun dateAddDateSubAndDatediffEndToEndThroughSelectAndWhere() { + run("CREATE TABLE events (id INTEGER PRIMARY KEY, happened DATE)") + run( + "INSERT INTO events VALUES " + + "(1, CAST('2024-03-15' AS DATE)), (2, CAST('2024-03-20' AS DATE))" + ) + val shifted = select( + "SELECT DATE_ADD(happened, 5), DATE_SUB(happened, 5) FROM events ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(date("2024-03-20"), date("2024-03-10")), + listOf(date("2024-03-25"), date("2024-03-15")) + ), + shifted.rows + ) + + val diff = select( + "SELECT DATEDIFF((SELECT happened FROM events WHERE id = 2), happened) FROM events WHERE id = 1" + ) + assertEquals(listOf(listOf(int(5))), diff.rows) + + val filtered = select("SELECT id FROM events WHERE DATE_ADD(happened, 5) = CAST('2024-03-25' AS DATE)") + assertEquals(listOf(listOf(int(2))), filtered.rows) + } + + @Test + fun repeatLpadAndRpadEndToEndThroughSelectAndUpdate() { + run("CREATE TABLE codes (id INTEGER PRIMARY KEY, code TEXT)") + run("INSERT INTO codes VALUES (1, '7'), (2, '42')") + val padded = select("SELECT id, LPAD(code, 4, '0') FROM codes ORDER BY id ASC") + assertEquals(listOf(listOf(int(1), text("0007")), listOf(int(2), text("0042"))), padded.rows) + + run("UPDATE codes SET code = RPAD(code, 5, '-') WHERE id = 1") + assertEquals(listOf(listOf(text("7----"))), select("SELECT code FROM codes WHERE id = 1").rows) + + val separator = select("SELECT REPEAT('-', 3) FROM codes WHERE id = 2") + assertEquals(listOf(listOf(text("---"))), separator.rows) + } + + @Test + fun signTruncGreatestAndLeastEndToEndThroughSelectWhereAndUpdate() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER, limitAmount INTEGER, ratio REAL)") + run("INSERT INTO accounts VALUES (1, -50, 100, 0.0), (2, 75, 40, 0.0), (3, 0, 0, 0.0)") + + val signs = select("SELECT id, SIGN(balance) FROM accounts ORDER BY id ASC") + assertEquals(listOf(listOf(int(1), int(-1)), listOf(int(2), int(1)), listOf(int(3), int(0))), signs.rows) + + val capped = select("SELECT id, LEAST(balance, limitAmount) FROM accounts ORDER BY id ASC") + assertEquals(listOf(listOf(int(1), int(-50)), listOf(int(2), int(40)), listOf(int(3), int(0))), capped.rows) + + val overLimit = select("SELECT id FROM accounts WHERE GREATEST(balance, 0) > limitAmount") + assertEquals(listOf(listOf(int(2))), overLimit.rows) + + run("UPDATE accounts SET ratio = TRUNC(balance / 3.0) WHERE id = 2") + assertEquals(listOf(listOf(Value.RealValue(25.0))), select("SELECT ratio FROM accounts WHERE id = 2").rows) + } +} diff --git a/src/test/kotlin/kestrel/ForeignKeyTest.kt b/src/test/kotlin/kestrel/ForeignKeyTest.kt new file mode 100644 index 0000000..c46e3d6 --- /dev/null +++ b/src/test/kotlin/kestrel/ForeignKeyTest.kt @@ -0,0 +1,228 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.DeleteResult +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class ForeignKeyTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seedDepartmentsAndEmployees() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run( + "CREATE TABLE employees (" + + "id INTEGER PRIMARY KEY, name TEXT NOT NULL, dept_id INTEGER REFERENCES departments(id))" + ) + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'ops')") + } + + @Test + fun insertRejectsForeignKeyViolation() { + seedDepartmentsAndEmployees() + val error = assertFailsWith { + run("INSERT INTO employees VALUES (1, 'ada', 99)") + } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun insertAllowsNullForeignKeyValue() { + seedDepartmentsAndEmployees() + val result = exec("INSERT INTO employees VALUES (1, 'ada', NULL)") + assertEquals(InsertResult(1), result) + } + + @Test + fun insertSucceedsWhenReferencedRowExists() { + seedDepartmentsAndEmployees() + val result = exec("INSERT INTO employees VALUES (1, 'ada', 1)") + assertEquals(InsertResult(1), result) + } + + @Test + fun updateRejectsForeignKeyViolationOnChildColumn() { + seedDepartmentsAndEmployees() + run("INSERT INTO employees VALUES (1, 'ada', 1)") + assertFailsWith { + run("UPDATE employees SET dept_id = 99 WHERE id = 1") + } + } + + @Test + fun foreignKeyRequiresReferencedTableToExist() { + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER REFERENCES missing(id))") + } + } + + @Test + fun foreignKeyRequiresReferencedColumnToBeAPrimaryKey() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x TEXT REFERENCES departments(name))") + } + } + + @Test + fun tableLevelForeignKeySyntaxIsEquivalentToColumnLevel() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, " + + "FOREIGN KEY (dept_id) REFERENCES departments(id))" + ) + run("INSERT INTO departments VALUES (1, 'eng')") + assertEquals(InsertResult(1), exec("INSERT INTO employees VALUES (1, 1)")) + assertFailsWith { run("INSERT INTO employees VALUES (2, 99)") } + } + + @Test + fun deleteIsRestrictedByDefaultWhenChildrenReferenceTheRow() { + seedDepartmentsAndEmployees() + run("INSERT INTO employees VALUES (1, 'ada', 1)") + val error = assertFailsWith { run("DELETE FROM departments WHERE id = 1") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + assertEquals(1, select("SELECT * FROM departments WHERE id = 1").rows.size) + } + + @Test + fun deleteWithoutReferencingChildrenSucceeds() { + seedDepartmentsAndEmployees() + assertEquals(DeleteResult(1), exec("DELETE FROM departments WHERE id = 2")) + } + + private fun seedCascadeDepartments() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE ON UPDATE CASCADE)" + ) + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'ops')") + run("INSERT INTO employees VALUES (1, 'ada', 1), (2, 'linus', 1), (3, 'grace', 2)") + } + + @Test + fun onDeleteCascadeRemovesReferencingChildren() { + seedCascadeDepartments() + assertEquals(DeleteResult(1), exec("DELETE FROM departments WHERE id = 1")) + val remaining = select("SELECT name FROM employees ORDER BY id") + assertEquals(listOf(listOf(text("grace"))), remaining.rows) + } + + @Test + fun onUpdateCascadeChangesChildForeignKeyWhenParentPrimaryKeyChanges() { + seedCascadeDepartments() + run("UPDATE departments SET id = 10 WHERE id = 1") + val depts = select("SELECT dept_id FROM employees ORDER BY id") + assertEquals(listOf(listOf(int(10)), listOf(int(10)), listOf(int(2))), depts.rows) + } + + @Test + fun onUpdateRestrictByDefaultBlocksPrimaryKeyChangeWhenChildrenReference() { + seedDepartmentsAndEmployees() + run("INSERT INTO employees VALUES (1, 'ada', 1)") + assertFailsWith { run("UPDATE departments SET id = 99 WHERE id = 1") } + } + + private fun seedSetNullDepartments(childNotNull: Boolean) { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + val childColumn = if (childNotNull) "dept_id INTEGER NOT NULL" else "dept_id INTEGER" + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "$childColumn REFERENCES departments(id) ON DELETE SET NULL)" + ) + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 'ada', 1)") + } + + @Test + fun onDeleteSetNullNullsOutReferencingChildren() { + seedSetNullDepartments(childNotNull = false) + exec("DELETE FROM departments WHERE id = 1") + val row = select("SELECT dept_id FROM employees WHERE id = 1") + assertEquals(listOf(listOf(Value.NullValue)), row.rows) + } + + @Test + fun onDeleteSetNullFailsWhenChildColumnIsNotNull() { + seedSetNullDepartments(childNotNull = true) + val error = assertFailsWith { run("DELETE FROM departments WHERE id = 1") } + assertTrue(error.message!!.contains("NOT NULL")) + } + + @Test + fun selfReferencingForeignKeyIsAllowedAndCascadesThroughMultipleLevels() { + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "manager_id INTEGER REFERENCES employees(id) ON DELETE CASCADE)" + ) + run( + "INSERT INTO employees VALUES " + + "(1, 'ceo', NULL), (2, 'vp', 1), (3, 'manager', 2), (4, 'ic', 3)" + ) + assertEquals(DeleteResult(1), exec("DELETE FROM employees WHERE id = 1")) + val remaining = select("SELECT id FROM employees") + assertEquals(emptyList(), remaining.rows) + } + + @Test + fun dropTableIsRestrictedByDefaultWhenAForeignKeyReferencesIt() { + seedDepartmentsAndEmployees() + val error = assertFailsWith { run("DROP TABLE departments") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropTableCascadeDropsTheConstraintButKeepsTheChildTableAndItsRows() { + seedDepartmentsAndEmployees() + run("INSERT INTO employees VALUES (1, 'ada', 1)") + exec("DROP TABLE departments CASCADE") + assertEquals(listOf("employees"), catalog.tableNames) + assertEquals(1, select("SELECT * FROM employees").rows.size) + // the constraint is gone, so a now-dangling value is accepted + assertEquals(InsertResult(1), exec("INSERT INTO employees VALUES (2, 'linus', 99)")) + } + + @Test + fun rollbackAfterCascadeDeleteRestoresChildRows() { + seedCascadeDepartments() + run("BEGIN") + run("DELETE FROM departments WHERE id = 1") + assertEquals(1, select("SELECT * FROM employees").rows.size) + run("ROLLBACK") + assertEquals(3, select("SELECT * FROM employees").rows.size) + assertEquals(2, select("SELECT * FROM departments").rows.size) + } + + @Test + fun rollbackAfterCascadeDropRestoresTheForeignKeyConstraint() { + seedDepartmentsAndEmployees() + run("BEGIN") + run("DROP TABLE departments CASCADE") + run("ROLLBACK") + assertFailsWith { + run("INSERT INTO employees VALUES (1, 'ada', 99)") + } + } +} diff --git a/src/test/kotlin/kestrel/GeneratedColumnTest.kt b/src/test/kotlin/kestrel/GeneratedColumnTest.kt new file mode 100644 index 0000000..01f7743 --- /dev/null +++ b/src/test/kotlin/kestrel/GeneratedColumnTest.kt @@ -0,0 +1,316 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.MergeResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `GENERATED ALWAYS AS (expr) STORED` on the in-memory engine - a column whose value is always + * derived from the rest of its own row, never assigned directly, and recomputed on every + * INSERT/UPDATE. See [kestrel.store.sql.DiskExecutorGeneratedColumnTest] for the disk engine. + */ +class GeneratedColumnTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + + // ---- computed on INSERT ---- + + @Test + fun generatedColumnIsComputedWhenOmittedFromInsert() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (5)") + assertEquals(listOf(listOf(int(5), int(10))), select("SELECT a, b FROM t").rows) + } + + @Test + fun generatedColumnIsComputedWhenOmittedFromPositionalInsertColumnList() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a, b) VALUES (5, DEFAULT)") + assertEquals(listOf(listOf(int(5), int(10))), select("SELECT a, b FROM t").rows) + } + + @Test + fun insertRejectsAnExplicitValueForAGeneratedColumn() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { run("INSERT INTO t (a, b) VALUES (5, 99)") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun generatedColumnRecomputesFreshForEveryRowOfAMultiRowInsert() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (1), (2), (3)") + assertEquals( + listOf(listOf(int(1), int(2)), listOf(int(2), int(4)), listOf(int(3), int(6))), + select("SELECT a, b FROM t ORDER BY a").rows + ) + } + + @Test + fun generatedColumnCanReferenceMultipleOtherColumns() { + run("CREATE TABLE t (price INTEGER, qty INTEGER, total INTEGER GENERATED ALWAYS AS (price * qty) STORED)") + run("INSERT INTO t (price, qty) VALUES (7, 3)") + assertEquals(listOf(listOf(int(21))), select("SELECT total FROM t").rows) + } + + // ---- computed on UPDATE ---- + + @Test + fun updatingADependencyColumnRecomputesTheGeneratedColumn() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (5)") + run("UPDATE t SET a = 9") + assertEquals(listOf(listOf(int(18))), select("SELECT b FROM t").rows) + } + + @Test + fun updateRejectsAnExplicitValueForAGeneratedColumn() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (5)") + val error = assertFailsWith { run("UPDATE t SET b = 1") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun updateSetToDefaultOnAGeneratedColumnRecomputesIt() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (5)") + run("UPDATE t SET a = 9, b = DEFAULT") + assertEquals(listOf(listOf(int(18))), select("SELECT b FROM t").rows) + } + + // ---- ON CONFLICT DO UPDATE ---- + + @Test + fun onConflictDoUpdateRecomputesTheGeneratedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (id, a) VALUES (1, 5)") + run("INSERT INTO t (id, a) VALUES (1, 20) ON CONFLICT (id) DO UPDATE SET a = EXCLUDED.a") + assertEquals(listOf(listOf(int(40))), select("SELECT b FROM t").rows) + } + + @Test + fun onConflictDoUpdateRejectsAnExplicitValueForAGeneratedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (id, a) VALUES (1, 5)") + assertFailsWith { + run("INSERT INTO t (id, a) VALUES (1, 20) ON CONFLICT (id) DO UPDATE SET b = 1") + } + } + + // ---- MERGE ---- + + @Test + fun mergeInsertComputesTheGeneratedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("CREATE TABLE s (id INTEGER, a INTEGER)") + run("INSERT INTO s VALUES (1, 5)") + val result = exec( + "MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, a) VALUES (s.id, s.a)" + ) as MergeResult + assertEquals(MergeResult(1, 0, 0), result) + assertEquals(listOf(listOf(int(10))), select("SELECT b FROM t").rows) + } + + @Test + fun mergeUpdateRecomputesTheGeneratedColumn() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("CREATE TABLE s (id INTEGER, a INTEGER)") + run("INSERT INTO t (id, a) VALUES (1, 5)") + run("INSERT INTO s VALUES (1, 9)") + exec("MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET a = s.a") + assertEquals(listOf(listOf(int(18))), select("SELECT b FROM t").rows) + } + + // ---- ordering with CHECK / NOT NULL ---- + + @Test + fun checkConstraintSeesTheComputedGeneratedValueNotItsOldOne() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED CHECK (b > 0))") + assertFailsWith { run("INSERT INTO t (a) VALUES (-1)") } + run("INSERT INTO t (a) VALUES (3)") + assertEquals(listOf(listOf(int(6))), select("SELECT b FROM t").rows) + } + + @Test + fun notNullOnAGeneratedColumnRejectsARowWhoseComputedValueIsNull() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a) STORED NOT NULL)") + assertFailsWith { run("INSERT INTO t (a) VALUES (NULL)") } + } + + // ---- validation at CREATE TABLE ---- + + @Test + fun rejectsGeneratedExpressionReferencingANonexistentColumn() { + val error = assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (missing) STORED)") + } + assertTrue(error.message!!.contains("no such column")) + } + + @Test + fun rejectsGeneratedExpressionContainingAnAggregate() { + assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (COUNT(a)) STORED)") + } + } + + @Test + fun rejectsGeneratedExpressionContainingASubquery() { + run("CREATE TABLE other (x INTEGER)") + assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS ((SELECT x FROM other)) STORED)") + } + } + + @Test + fun rejectsGeneratedExpressionContainingNextval() { + run("CREATE SEQUENCE seq") + assertFailsWith { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (NEXTVAL('seq')) STORED)") + } + } + + @Test + fun rejectsAGeneratedColumnReferencingAnotherGeneratedColumn() { + val error = assertFailsWith { + run( + "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, " + + "c INTEGER GENERATED ALWAYS AS (b + 1) STORED)" + ) + } + assertTrue(error.message!!.contains("GENERATED")) + } + + // ---- RENAME COLUMN / DROP COLUMN interaction ---- + + @Test + fun renameColumnPropagatesIntoADependentGeneratedExpression() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("ALTER TABLE t RENAME COLUMN a TO a2") + run("INSERT INTO t (a2) VALUES (5)") + assertEquals(listOf(listOf(int(10))), select("SELECT b FROM t").rows) + } + + @Test + fun dropColumnRestrictsWhenAGeneratedColumnDependsOnIt() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { run("ALTER TABLE t DROP COLUMN a") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun dropColumnCascadeRemovesTheDependentGeneratedColumnToo() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, c TEXT)") + run("INSERT INTO t (a, c) VALUES (5, 'x')") + run("ALTER TABLE t DROP COLUMN a CASCADE") + val result = select("SELECT * FROM t") + assertEquals(listOf("c"), result.columns) + assertEquals(listOf(listOf(text("x"))), result.rows) + } + + // ---- ALTER TABLE ADD COLUMN GENERATED ALWAYS AS ---- + + @Test + fun alterTableAddColumnGeneratedBackfillsEachExistingRowFromItsOwnOtherColumns() { + run("CREATE TABLE t (a INTEGER)") + run("INSERT INTO t (a) VALUES (1), (2), (3)") + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + assertEquals( + listOf(listOf(int(1), int(2)), listOf(int(2), int(4)), listOf(int(3), int(6))), + select("SELECT a, b FROM t ORDER BY a ASC").rows + ) + } + + @Test + fun alterTableAddColumnGeneratedWorksOnAnEmptyTable() { + run("CREATE TABLE t (a INTEGER)") + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + run("INSERT INTO t (a) VALUES (5)") + assertEquals(listOf(listOf(int(10))), select("SELECT b FROM t").rows) + } + + @Test + fun afterAlterTableAddColumnGeneratedAnInsertStillRejectsAnExplicitValue() { + run("CREATE TABLE t (a INTEGER)") + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + val error = assertFailsWith { run("INSERT INTO t (a, b) VALUES (5, 99)") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun afterAlterTableAddColumnGeneratedUpdatingTheDependencyColumnRecomputesIt() { + run("CREATE TABLE t (a INTEGER)") + run("INSERT INTO t (a) VALUES (5)") + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + run("UPDATE t SET a = 9") + assertEquals(listOf(listOf(int(18))), select("SELECT b FROM t").rows) + } + + @Test + fun alterTableAddColumnGeneratedNotNullSkipsTheDefaultRequirementOnANonEmptyTable() { + run("CREATE TABLE t (a INTEGER)") + run("INSERT INTO t (a) VALUES (1), (2)") + // A plain NOT NULL column with no DEFAULT would be rejected here (see the DEFAULT-less + // ADD COLUMN rule tested elsewhere) - GENERATED has its own per-row value to backfill with + // instead, so it needs no DEFAULT at all even though the table already has rows. + run("ALTER TABLE t ADD COLUMN b INTEGER NOT NULL GENERATED ALWAYS AS (a * 2) STORED") + assertEquals(listOf(listOf(int(2)), listOf(int(4))), select("SELECT b FROM t ORDER BY a ASC").rows) + } + + @Test + fun alterTableAddColumnGeneratedNotNullRejectsARowWhoseBackfilledValueIsNull() { + run("CREATE TABLE t (a INTEGER)") + run("INSERT INTO t (a) VALUES (NULL)") + val error = assertFailsWith { + run("ALTER TABLE t ADD COLUMN b INTEGER NOT NULL GENERATED ALWAYS AS (a) STORED") + } + assertTrue(error.message!!.contains("NOT NULL")) + } + + @Test + fun alterTableAddColumnGeneratedRejectsReferencingANonexistentColumn() { + run("CREATE TABLE t (a INTEGER)") + val error = assertFailsWith { + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (missing) STORED") + } + assertTrue(error.message!!.contains("no such column")) + } + + @Test + fun alterTableAddColumnGeneratedRejectsReferencingAnAlreadyExistingGeneratedColumn() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { + run("ALTER TABLE t ADD COLUMN c INTEGER GENERATED ALWAYS AS (b + 1) STORED") + } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun alterTableAddColumnGeneratedRejectsAnAggregate() { + run("CREATE TABLE t (a INTEGER)") + assertFailsWith { + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (COUNT(a)) STORED") + } + } + + private fun text(value: String) = Value.TextValue(value) +} diff --git a/src/test/kotlin/kestrel/GroupingSetsTest.kt b/src/test/kotlin/kestrel/GroupingSetsTest.kt new file mode 100644 index 0000000..fca5ef5 --- /dev/null +++ b/src/test/kotlin/kestrel/GroupingSetsTest.kt @@ -0,0 +1,213 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `GROUP BY ROLLUP (...)` / `CUBE (...)` / `GROUPING SETS (...)` and the `GROUPING(...)` function - + * a single query reporting subtotals (and a grand total) at several levels at once instead of a + * separate `UNION ALL` of differently-grouped queries. See [kestrel.sql.ast.GroupingSpec] and + * [kestrel.engine.GroupingSetExpansion] for how the sets are computed and + * [kestrel.engine.GroupAggregation] for how a collapsed column reads as NULL outside an aggregate. + */ +class GroupingSetsTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private val NULL = Value.NullValue + + private fun seed() { + run("CREATE TABLE sales (dept TEXT, region TEXT, amount INTEGER)") + run( + "INSERT INTO sales VALUES " + + "('eng', 'eu', 100), ('eng', 'us', 200), ('ops', 'eu', 50), ('ops', 'eu', 70)" + ) + } + + @Test + fun rollupProducesDetailSubtotalsAndAGrandTotal() { + seed() + val result = select( + "SELECT dept, region, SUM(amount) AS total, GROUPING(dept) AS gd, GROUPING(region) AS gr " + + "FROM sales GROUP BY ROLLUP (dept, region) " + + "ORDER BY GROUPING(dept), dept, GROUPING(region), region" + ) + assertEquals(listOf("dept", "region", "total", "gd", "gr"), result.columns) + assertEquals( + listOf( + listOf(text("eng"), text("eu"), int(100), int(0), int(0)), + listOf(text("eng"), text("us"), int(200), int(0), int(0)), + listOf(text("eng"), NULL, int(300), int(0), int(1)), + listOf(text("ops"), text("eu"), int(120), int(0), int(0)), + listOf(text("ops"), NULL, int(120), int(0), int(1)), + listOf(NULL, NULL, int(420), int(1), int(1)) + ), + result.rows + ) + } + + @Test + fun cubeProducesEverySubsetOfGroupingColumns() { + seed() + val result = select( + "SELECT dept, region, SUM(amount) AS total " + + "FROM sales GROUP BY CUBE (dept, region) " + + "ORDER BY GROUPING(dept), GROUPING(region), dept, region" + ) + assertEquals( + listOf( + listOf(text("eng"), text("eu"), int(100)), + listOf(text("eng"), text("us"), int(200)), + listOf(text("ops"), text("eu"), int(120)), + listOf(text("eng"), NULL, int(300)), + listOf(text("ops"), NULL, int(120)), + listOf(NULL, text("eu"), int(220)), + listOf(NULL, text("us"), int(200)), + listOf(NULL, NULL, int(420)) + ), + result.rows + ) + } + + @Test + fun explicitGroupingSetsSkipsTheFullDetailSetWhenNotListed() { + seed() + val result = select( + "SELECT dept, region, SUM(amount) AS total " + + "FROM sales GROUP BY GROUPING SETS ((dept), (region), ()) " + + "ORDER BY GROUPING(dept), GROUPING(region), dept, region" + ) + assertEquals( + listOf( + listOf(text("eng"), NULL, int(300)), + listOf(text("ops"), NULL, int(120)), + listOf(NULL, text("eu"), int(220)), + listOf(NULL, text("us"), int(200)), + listOf(NULL, NULL, int(420)) + ), + result.rows + ) + } + + @Test + fun havingFiltersAcrossEverySetAtOnce() { + seed() + val result = select( + "SELECT dept, SUM(amount) AS total FROM sales GROUP BY ROLLUP (dept) " + + "HAVING SUM(amount) > 150 ORDER BY GROUPING(dept), dept" + ) + assertEquals(listOf(listOf(text("eng"), int(300)), listOf(NULL, int(420))), result.rows) + } + + @Test + fun plainGroupByAlwaysReportsGroupingAsZero() { + seed() + val result = select("SELECT dept, GROUPING(dept) FROM sales GROUP BY dept ORDER BY dept") + assertEquals(listOf(listOf(text("eng"), int(0)), listOf(text("ops"), int(0))), result.rows) + } + + @Test + fun aCollapsedColumnInsideCaseEvaluatesAsNull() { + seed() + val result = select( + "SELECT CASE WHEN dept IS NULL THEN 'ALL DEPARTMENTS' ELSE dept END AS label, SUM(amount) AS total " + + "FROM sales GROUP BY ROLLUP (dept) ORDER BY GROUPING(dept), dept" + ) + assertEquals( + listOf( + listOf(text("eng"), int(300)), + listOf(text("ops"), int(120)), + listOf(text("ALL DEPARTMENTS"), int(420)) + ), + result.rows + ) + } + + @Test + fun aggregatesInsideTheSameQuerySeeRealPerRowValuesRegardlessOfCollapsedColumns() { + seed() + // COUNT(*) on the grand-total row must count every row (4), not be tripped up by dept/region + // reading as NULL there - the nulling only ever applies outside an aggregate's own argument. + val result = select( + "SELECT dept, COUNT(*) AS n FROM sales GROUP BY ROLLUP (dept) ORDER BY GROUPING(dept), dept" + ) + assertEquals( + listOf(listOf(text("eng"), int(2)), listOf(text("ops"), int(2)), listOf(NULL, int(4))), + result.rows + ) + } + + @Test + fun rollupOverAnEmptyTableStillProducesOneGrandTotalRow() { + run("CREATE TABLE empty_sales (dept TEXT, amount INTEGER)") + val result = select("SELECT dept, COUNT(*), SUM(amount) FROM empty_sales GROUP BY ROLLUP (dept)") + assertEquals(listOf(listOf(NULL, int(0), NULL)), result.rows) + } + + @Test + fun groupingFunctionOutsideAGroupedQueryIsRejected() { + seed() + val failure = assertFailsWith { select("SELECT GROUPING(dept) FROM sales") } + assertTrue(failure.message!!.contains("GROUP BY"), failure.message) + } + + @Test + fun groupingFunctionArgumentMustItselfBeAGroupingColumn() { + seed() + val failure = assertFailsWith { + select("SELECT GROUPING(amount) FROM sales GROUP BY ROLLUP (dept)") + } + assertTrue(failure.message!!.contains("must appear in GROUP BY"), failure.message) + } + + @Test + fun groupingFunctionRequiresAtLeastOneArgument() { + seed() + val failure = assertFailsWith { + select("SELECT GROUPING() FROM sales GROUP BY ROLLUP (dept)") + } + assertTrue(failure.message!!.contains("at least one argument"), failure.message) + } + + @Test + fun cubeRejectsTooManyColumns() { + val columns = (1..17).joinToString(", ") { "c$it" } + run("CREATE TABLE wide (${(1..17).joinToString(", ") { "c$it INTEGER" }})") + val failure = assertFailsWith { + select("SELECT COUNT(*) FROM wide GROUP BY CUBE ($columns)") + } + assertTrue(failure.message!!.contains("CUBE supports at most"), failure.message) + } + + @Test + fun explainDescribesTheNumberOfGroupingSets() { + seed() + val plan = (run("EXPLAIN SELECT dept, region, SUM(amount) FROM sales GROUP BY CUBE (dept, region)").single() + as kestrel.engine.ExplainResult).plan + assertTrue(plan.any { it.contains("GroupAggregate") && it.contains("4 grouping sets") }, plan.toString()) + } + + @Test + fun explainOnAPlainGroupByDoesNotMentionGroupingSets() { + seed() + val plan = (run("EXPLAIN SELECT dept, SUM(amount) FROM sales GROUP BY dept").single() + as kestrel.engine.ExplainResult).plan + assertTrue(plan.any { it == "GroupAggregate keys: dept" }, plan.toString()) + } +} diff --git a/src/test/kotlin/kestrel/IndexTest.kt b/src/test/kotlin/kestrel/IndexTest.kt index 56dcf6e..1dab8ad 100644 --- a/src/test/kotlin/kestrel/IndexTest.kt +++ b/src/test/kotlin/kestrel/IndexTest.kt @@ -112,4 +112,54 @@ class IndexTest { fun droppingUnknownIndexIsRejected() { assertFailsWith { run("DROP INDEX ghost") } } + + @Test + fun createUniqueIndexRejectsADuplicateOnInsert() { + seed() + run("CREATE UNIQUE INDEX idxName ON users (name)") + assertFailsWith { run("INSERT INTO users VALUES (6, 'ada', 21)") } + } + + @Test + fun createUniqueIndexRejectsADuplicateOnUpdate() { + seed() + run("CREATE UNIQUE INDEX idxName ON users (name)") + assertFailsWith { run("UPDATE users SET name = 'ada' WHERE name = 'linus'") } + } + + @Test + fun createUniqueIndexAllowsMultipleNulls() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT)") + run("CREATE UNIQUE INDEX idxTag ON t (tag)") + run("INSERT INTO t VALUES (1, NULL), (2, NULL)") + assertEquals(2, select("SELECT id FROM t").rows.size) + } + + @Test + fun creatingAUniqueIndexOverExistingDuplicatesFails() { + seed() + run("INSERT INTO users VALUES (6, 'ada', 21)") + assertFailsWith { run("CREATE UNIQUE INDEX idxName ON users (name)") } + // A failed creation must not leave a half-built index registered under the name. + run("CREATE INDEX idxName ON users (name)") + } + + @Test + fun uniqueIndexStillAcceleratesEqualityLookups() { + seed() + run("CREATE UNIQUE INDEX idxName ON users (name)") + val plan = explain("EXPLAIN SELECT id FROM users WHERE name = 'ada'") + assertTrue(plan.any { it.contains("Index Scan using idxName") }, "expected index scan, got $plan") + } + + @Test + fun rollbackAfterAFailedInsertLeavesTheUniqueIndexUsable() { + seed() + run("CREATE UNIQUE INDEX idxName ON users (name)") + run("BEGIN") + assertFailsWith { run("INSERT INTO users VALUES (6, 'ada', 21)") } + run("ROLLBACK") + run("INSERT INTO users VALUES (6, 'zoe', 21)") + assertEquals(6, select("SELECT id FROM users").rows.size) + } } diff --git a/src/test/kotlin/kestrel/JoinTest.kt b/src/test/kotlin/kestrel/JoinTest.kt index 3733522..d8a0ade 100644 --- a/src/test/kotlin/kestrel/JoinTest.kt +++ b/src/test/kotlin/kestrel/JoinTest.kt @@ -59,6 +59,110 @@ class JoinTest { assertEquals(listOf(text("ada"), int(100)), result.rows[3]) } + private fun seedWithOrphanOrder() { + seed() + // References a userId that does not exist, giving RIGHT/FULL JOIN an unmatched right row. + run("INSERT INTO orders VALUES (13, 99, 999)") + } + + @Test + fun rightJoinKeepsUnmatchedRightRowsWithNulls() { + seedWithOrphanOrder() + val result = select( + "SELECT u.name, o.total FROM users u RIGHT JOIN orders o ON u.id = o.userId ORDER BY o.id ASC" + ) + assertEquals( + listOf( + listOf(text("ada"), int(100)), + listOf(text("ada"), int(40)), + listOf(text("linus"), int(70)), + listOf(Value.NullValue, int(999)) + ), + result.rows + ) + } + + @Test + fun rightJoinDropsUnmatchedLeftRows() { + seedWithOrphanOrder() + val result = select("SELECT u.name FROM users u RIGHT JOIN orders o ON u.id = o.userId") + assertTrue(result.rows.none { it == listOf(text("grace")) }, "got ${result.rows}") + } + + @Test + fun fullJoinKeepsBothUnmatchedSides() { + seedWithOrphanOrder() + val result = select("SELECT u.name, o.total FROM users u FULL JOIN orders o ON u.id = o.userId") + assertEquals( + setOf( + listOf(text("ada"), int(100)), + listOf(text("ada"), int(40)), + listOf(text("linus"), int(70)), + listOf(text("grace"), Value.NullValue), + listOf(Value.NullValue, int(999)) + ), + result.rows.toSet() + ) + assertEquals(5, result.rows.size) + } + + @Test + fun fullJoinHashAndNestedLoopProduceSameResult() { + seedWithOrphanOrder() + val hashed = select( + "SELECT u.name, o.total FROM users u FULL JOIN orders o ON u.id = o.userId" + ).rows.toSet() + val nested = select( + "SELECT u.name, o.total FROM users u FULL JOIN orders o ON u.id >= o.userId AND u.id <= o.userId" + ).rows.toSet() + assertEquals(hashed, nested) + } + + @Test + fun rightJoinPadsWithFullWidthOfAllPrecedingTablesInAChain() { + seed() + run("CREATE TABLE items (orderId INTEGER, label TEXT)") + // orderId 99 matches no `orders` row, so it can only survive via the RIGHT JOIN's own + // unmatched-right padding - proving that padding spans BOTH preceding tables (users AND + // orders), not just orders (the immediate left side of this particular JOIN). + run("INSERT INTO items VALUES (10, 'book'), (99, 'orphanItem')") + val result = select( + "SELECT u.name, o.total, i.label FROM users u " + + "JOIN orders o ON u.id = o.userId " + + "RIGHT JOIN items i ON o.id = i.orderId" + ) + assertEquals( + setOf( + listOf(text("ada"), int(100), text("book")), + listOf(Value.NullValue, Value.NullValue, text("orphanItem")) + ), + result.rows.toSet() + ) + assertEquals(2, result.rows.size) + } + + @Test + fun rightAndFullJoinAcceptOptionalOuterKeyword() { + seed() + assertEquals( + select("SELECT u.name FROM users u RIGHT JOIN orders o ON u.id = o.userId").rows.toSet(), + select("SELECT u.name FROM users u RIGHT OUTER JOIN orders o ON u.id = o.userId").rows.toSet() + ) + assertEquals( + select("SELECT u.name FROM users u FULL JOIN orders o ON u.id = o.userId").rows.toSet(), + select("SELECT u.name FROM users u FULL OUTER JOIN orders o ON u.id = o.userId").rows.toSet() + ) + } + + @Test + fun plannerDescribesRightAndFullJoinLabels() { + seed() + val rightNested = explain("EXPLAIN SELECT u.name FROM users u RIGHT JOIN orders o ON u.id < o.userId") + assertTrue(rightNested.any { it.contains("Nested Loop Right") }, "got $rightNested") + val fullHash = explain("EXPLAIN SELECT u.name FROM users u FULL JOIN orders o ON u.id = o.userId") + assertTrue(fullHash.any { it.contains("Hash Join") }, "got $fullHash") + } + @Test fun hashJoinAndNestedLoopProduceSameResult() { seed() @@ -115,4 +219,41 @@ class JoinTest { select("SELECT id FROM users u JOIN orders o ON u.id = o.userId") } } + + @Test + fun crossJoinProducesTheFullCartesianProduct() { + run("CREATE TABLE colors (name TEXT)") + run("CREATE TABLE sizes (label TEXT)") + run("INSERT INTO colors VALUES ('red'), ('blue')") + run("INSERT INTO sizes VALUES ('s'), ('m'), ('l')") + val result = select("SELECT colors.name, sizes.label FROM colors CROSS JOIN sizes ORDER BY colors.name, sizes.label") + assertEquals( + listOf( + listOf(text("blue"), text("l")), listOf(text("blue"), text("m")), listOf(text("blue"), text("s")), + listOf(text("red"), text("l")), listOf(text("red"), text("m")), listOf(text("red"), text("s")) + ), + result.rows + ) + } + + @Test + fun crossJoinNeverUsesHashJoinAndIsDescribedAsCrossInExplain() { + run("CREATE TABLE colors (name TEXT)") + run("CREATE TABLE sizes (label TEXT)") + val plan = explain("EXPLAIN SELECT colors.name FROM colors CROSS JOIN sizes") + assertTrue(plan.any { it.contains("Nested Loop Cross") }, "got $plan") + assertTrue(plan.none { it.contains("Hash Join") }, "got $plan") + } + + @Test + fun crossJoinChainedWithAFilterActsLikeAnExplicitCartesianJoin() { + seed() + val cross = select( + "SELECT u.name, o.total FROM users u CROSS JOIN orders o WHERE u.id = o.userId ORDER BY o.total ASC" + ).rows + val inner = select( + "SELECT u.name, o.total FROM users u JOIN orders o ON u.id = o.userId ORDER BY o.total ASC" + ).rows + assertEquals(inner, cross) + } } diff --git a/src/test/kotlin/kestrel/LateralJoinTest.kt b/src/test/kotlin/kestrel/LateralJoinTest.kt new file mode 100644 index 0000000..4b56e87 --- /dev/null +++ b/src/test/kotlin/kestrel/LateralJoinTest.kt @@ -0,0 +1,201 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class LateralJoinTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + + private fun text(value: String) = Value.TextValue(value) + private fun int(value: Long) = Value.IntegerValue(value) + + private fun seed() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'sales')") + run( + "INSERT INTO employees VALUES " + + "(1, 1, 'ada', 300), (2, 1, 'linus', 250), (3, 1, 'grace', 400), " + + "(4, 2, 'sam', 150), (5, 2, 'kay', 200)" + ) + } + + @Test + fun lateralJoinComputesTopOnePerGroupWhichAPlainJoinCannotExpress() { + seed() + val result = select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "JOIN LATERAL (SELECT e.name, e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf( + listOf(text("eng"), text("grace"), int(400)), + listOf(text("sales"), text("kay"), int(200)) + ), + result.rows + ) + } + + @Test + fun lateralJoinWithLimitGreaterThanOneReturnsMultipleRowsPerLeftRow() { + seed() + val result = select( + "SELECT d.name, top.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 2) AS top ON TRUE " + + "ORDER BY d.name ASC, top.name ASC" + ) + assertEquals( + listOf( + listOf(text("eng"), text("ada")), + listOf(text("eng"), text("grace")), + listOf(text("sales"), text("kay")), + listOf(text("sales"), text("sam")) + ), + result.rows + ) + } + + @Test + fun innerJoinLateralDropsGroupsWithNoMatchingRows() { + seed() + run("INSERT INTO departments VALUES (3, 'empty')") + val result = select( + "SELECT d.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id LIMIT 1) AS top ON TRUE" + ) + assertTrue(result.rows.none { it == listOf(text("empty")) }) + assertEquals(2, result.rows.size) + } + + @Test + fun leftJoinLateralKeepsGroupsWithNoMatchingRowsAsNull() { + seed() + run("INSERT INTO departments VALUES (3, 'empty')") + val result = select( + "SELECT d.name, top.name FROM departments d " + + "LEFT JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf( + listOf(text("empty"), Value.NullValue), + listOf(text("eng"), text("grace")), + listOf(text("sales"), text("kay")) + ), + result.rows + ) + } + + @Test + fun leftJoinLateralOverAnEmptyBaseTableProducesNoRowsWithoutRunningTheSubquery() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT)") + val result = select( + "SELECT d.name, top.name FROM departments d " + + "LEFT JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id) AS top ON TRUE" + ) + assertEquals(emptyList(), result.rows) + } + + @Test + fun chainedLateralJoinsSeeEarlierLateralAliases() { + seed() + run("CREATE TABLE dual (x INTEGER)") + run("INSERT INTO dual VALUES (1)") + val result = select( + "SELECT d.name, bonus.amount FROM departments d " + + "JOIN LATERAL (SELECT e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "JOIN LATERAL (SELECT top.salary / 10 AS amount FROM dual) AS bonus ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(40)), listOf(text("sales"), int(20))), result.rows) + } + + @Test + fun lateralSubqueryCannotReferenceATableJoinedLater() { + seed() + assertFailsWith { + select( + "SELECT * FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.salary = later.salary) AS top ON TRUE " + + "JOIN employees later ON later.deptId = d.id" + ) + } + } + + @Test + fun lateralKeywordBeforeAPlainTableIsRejected() { + seed() + assertFailsWith { + select("SELECT * FROM departments d JOIN LATERAL employees ON TRUE") + } + } + + @Test + fun lateralSubqueryWithStarProjectionResolvesColumnsStructurallyFromItsBaseTable() { + seed() + // The lateral subquery itself uses `SELECT *`, so its output column names come from + // [kestrel.engine.SelectRunner.structuralColumnNames] walking the base table's schema in + // the catalog, not from an executed row - exercised here before any left row exists yet. + val result = select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "JOIN LATERAL (SELECT * FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf(listOf(text("eng"), text("grace"), int(400)), listOf(text("sales"), text("kay"), int(200))), + result.rows + ) + } + + @Test + fun explainAnnotatesTheJoinAsLateralAndNeverUsesHashJoin() { + seed() + val plan = explain( + "EXPLAIN SELECT d.name, top.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id) AS top ON top.name = d.name" + ) + assertTrue(plan.any { it.contains("Lateral") }, "got $plan") + assertTrue(plan.none { it.contains("Hash Join") }, "got $plan") + } + + @Test + fun crossJoinLateralIsEquivalentToPlainJoinLateralOnTrue() { + seed() + val cross = select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "CROSS JOIN LATERAL (SELECT e.name, e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ORDER BY d.name ASC" + ) + val onTrue = select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "JOIN LATERAL (SELECT e.name, e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE ORDER BY d.name ASC" + ) + assertEquals(onTrue.rows, cross.rows) + } +} diff --git a/src/test/kotlin/kestrel/LexerTest.kt b/src/test/kotlin/kestrel/LexerTest.kt index 702395e..c333de9 100644 --- a/src/test/kotlin/kestrel/LexerTest.kt +++ b/src/test/kotlin/kestrel/LexerTest.kt @@ -105,4 +105,140 @@ class LexerTest { fun unexpectedCharacterFails() { assertFailsWith { lex("SELECT @") } } + + @Test + fun predicateKeywordsAreRecognized() { + assertEquals( + listOf( + TokenType.Is, TokenType.In, TokenType.Between, TokenType.Like, + TokenType.Case, TokenType.When, TokenType.Then, TokenType.Else, TokenType.End, + TokenType.EndOfInput + ), + types("IS in Between LIKE case WHEN then ELSE end") + ) + } + + @Test + fun existsKeywordIsRecognizedAndCaseInsensitive() { + assertEquals(listOf(TokenType.Exists, TokenType.Exists, TokenType.EndOfInput), types("EXISTS exists")) + } + + @Test + fun viewAndReplaceKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf(TokenType.View, TokenType.Replace, TokenType.View, TokenType.Replace, TokenType.EndOfInput), + types("VIEW REPLACE view replace") + ) + } + + @Test + fun compoundQueryKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf( + TokenType.Union, TokenType.All, TokenType.Intersect, TokenType.Except, + TokenType.Union, TokenType.All, TokenType.Intersect, TokenType.Except, + TokenType.EndOfInput + ), + types("UNION ALL INTERSECT EXCEPT union all intersect except") + ) + } + + @Test + fun withAndRecursiveKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf(TokenType.With, TokenType.Recursive, TokenType.With, TokenType.Recursive, TokenType.EndOfInput), + types("WITH RECURSIVE with recursive") + ) + } + + @Test + fun castCascadeRestrictDecimalAndDateKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf( + TokenType.Cast, TokenType.Cascade, TokenType.Restrict, TokenType.Decimal, TokenType.Date, + TokenType.Cast, TokenType.Cascade, TokenType.Restrict, TokenType.Decimal, TokenType.Date, + TokenType.EndOfInput + ), + types("CAST CASCADE RESTRICT DECIMAL DATE cast cascade restrict decimal date") + ) + } + + @Test + fun foreignKeyKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf( + TokenType.Foreign, TokenType.Key, TokenType.References, TokenType.Check, + TokenType.No, TokenType.Action, + TokenType.Foreign, TokenType.Key, TokenType.References, TokenType.Check, + TokenType.No, TokenType.Action, + TokenType.EndOfInput + ), + types("FOREIGN KEY REFERENCES CHECK NO ACTION foreign key references check no action") + ) + } + + @Test + fun uniqueKeywordIsRecognizedAndCaseInsensitive() { + assertEquals( + listOf(TokenType.Unique, TokenType.Unique, TokenType.EndOfInput), + types("UNIQUE unique") + ) + } + + @Test + fun prepareExecuteAndDeallocateKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf( + TokenType.Prepare, TokenType.Execute, TokenType.Deallocate, + TokenType.Prepare, TokenType.Execute, TokenType.Deallocate, + TokenType.EndOfInput + ), + types("PREPARE EXECUTE DEALLOCATE prepare execute deallocate") + ) + } + + @Test + fun tempAndTemporaryKeywordsAreRecognizedAndCaseInsensitive() { + assertEquals( + listOf(TokenType.Temp, TokenType.Temporary, TokenType.Temp, TokenType.Temporary, TokenType.EndOfInput), + types("TEMP TEMPORARY temp temporary") + ) + } + + @Test + fun collateKeywordIsRecognizedAndCaseInsensitive() { + assertEquals( + listOf(TokenType.Collate, TokenType.Collate, TokenType.EndOfInput), + types("COLLATE collate") + ) + } + + @Test + fun parameterTokenCarriesItsOneBasedIndex() { + val token = lex("\$1").first() + assertEquals(TokenType.Parameter, token.type) + assertEquals(1L, token.intValue) + assertEquals("\$1", token.lexeme) + } + + @Test + fun parameterTokenSupportsMultipleDigits() { + val token = lex("\$42").first() + assertEquals(TokenType.Parameter, token.type) + assertEquals(42L, token.intValue) + } + + @Test + fun consecutiveParametersLexAsSeparateTokens() { + assertEquals( + listOf(TokenType.Parameter, TokenType.Comma, TokenType.Parameter, TokenType.EndOfInput), + types("\$1,\$2") + ) + } + + @Test + fun dollarSignWithNoFollowingDigitIsALexError() { + assertFailsWith { lex("\$") } + assertFailsWith { lex("\$abc") } + } } diff --git a/src/test/kotlin/kestrel/MergeTest.kt b/src/test/kotlin/kestrel/MergeTest.kt new file mode 100644 index 0000000..2112d87 --- /dev/null +++ b/src/test/kotlin/kestrel/MergeTest.kt @@ -0,0 +1,459 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.MergeResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class MergeTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + + private fun text(value: String) = Value.TextValue(value) + + private fun setUpAccountsAndStaging() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER NOT NULL)") + run("CREATE TABLE staging (id INTEGER, amount INTEGER)") + } + + @Test + fun matchedRowIsUpdated() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 50)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount" + ) as MergeResult + assertEquals(MergeResult(0, 1, 0), result) + assertEquals(listOf(listOf(int(150))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun unmatchedRowIsInserted() { + setUpAccountsAndStaging() + run("INSERT INTO staging VALUES (7, 30)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(1, 0, 0), result) + assertEquals(listOf(listOf(int(7), int(30))), select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun matchedRowIsDeleted() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 0)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id WHEN MATCHED THEN DELETE" + ) as MergeResult + assertEquals(MergeResult(0, 0, 1), result) + assertEquals(emptyList(), select("SELECT * FROM accounts").rows) + } + + @Test + fun aMultiRowMergeInsertsUpdatesAndDeletesInOneStatement() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100), (2, 200)") + run("INSERT INTO staging VALUES (1, 10), (2, -999), (3, 5)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND accounts.balance + staging.amount < 0 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(1, 1, 1), result) + assertEquals( + listOf(listOf(int(1), int(110)), listOf(int(3), int(5))), + select("SELECT id, balance FROM accounts ORDER BY id").rows + ) + } + + @Test + fun firstSatisfiedWhenMatchedClauseWinsInWrittenOrder() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 5)") + exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 100 THEN DELETE " + + "WHEN MATCHED AND staging.amount > 0 THEN UPDATE SET balance = 1 " + + "WHEN MATCHED THEN UPDATE SET balance = 2" + ) + assertEquals(listOf(listOf(int(1))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun firstSatisfiedWhenNotMatchedClauseWinsInWrittenOrder() { + setUpAccountsAndStaging() + run("INSERT INTO staging VALUES (1, 500)") + exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND staging.amount > 1000 THEN INSERT (id, balance) VALUES (staging.id, 1) " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, 2)" + ) + assertEquals(listOf(listOf(int(1), int(2))), select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun noSatisfiedWhenClauseLeavesTheRowUntouched() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 5)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 1000 THEN UPDATE SET balance = 0" + ) as MergeResult + assertEquals(MergeResult(0, 0, 0), result) + assertEquals(listOf(listOf(int(100))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun anUnmatchedSourceRowWithNoSatisfiedNotMatchedClauseInsertsNothing() { + setUpAccountsAndStaging() + run("INSERT INTO staging VALUES (1, 5)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND staging.amount > 1000 THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(0, 0, 0), result) + assertEquals(emptyList(), select("SELECT * FROM accounts").rows) + } + + @Test + fun aTargetRowMatchedByTwoSourceRowsCannotBeAffectedTwice() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 5), (1, 10)") + val error = assertFailsWith { + run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount" + ) + } + assertTrue(error.message!!.contains("a second time"), error.message) + } + + @Test + fun aTargetRowMatchedByTwoSourceRowsIsFineWhenOnlyOneClauseActuallyFires() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 5), (1, 999)") + exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 100 THEN UPDATE SET balance = accounts.balance + staging.amount" + ) + assertEquals(listOf(listOf(int(1099))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun anUnqualifiedColumnPresentOnBothSidesIsAmbiguousInOnCondition() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER)") + run("CREATE TABLE s (id INTEGER, v INTEGER)") + val error = assertFailsWith { + run("MERGE INTO t USING s ON id = s.id WHEN MATCHED THEN UPDATE SET v = 1") + } + assertTrue(error.message!!.contains("ambiguous"), error.message) + } + + @Test + fun aQualifierMatchingNeitherSideFails() { + setUpAccountsAndStaging() + val error = assertFailsWith { + run("MERGE INTO accounts USING staging ON bogus.id = staging.id WHEN MATCHED THEN DELETE") + } + assertTrue(error.message!!.contains("no such table or alias"), error.message) + } + + @Test + fun aWhenNotMatchedClauseCannotReferenceTheTargetTable() { + setUpAccountsAndStaging() + val error = assertFailsWith { + run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND accounts.balance > 0 THEN INSERT (id, balance) VALUES (staging.id, 1)" + ) + } + assertTrue(error.message!!.contains("WHEN NOT MATCHED"), error.message) + } + + @Test + fun columnValidationRunsUpFrontEvenWhenNoSourceRowEverReachesTheBadClause() { + setUpAccountsAndStaging() + val error = assertFailsWith { + run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED THEN INSERT (id, nope) VALUES (staging.id, 1)" + ) + } + assertTrue(error.message!!.contains("no such column"), error.message) + } + + @Test + fun theTargetAndSourceCannotShareTheSameNameOrAlias() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE other (id INTEGER)") + val error = assertFailsWith { + run("MERGE INTO t USING other AS t ON t.id = t.id WHEN MATCHED THEN DELETE") + } + assertTrue(error.message!!.contains("target and source"), error.message) + } + + @Test + fun sourceCanBeADerivedTableSubquery() { + setUpAccountsAndStaging() + run("INSERT INTO staging VALUES (1, 10), (1, 20)") + exec( + "MERGE INTO accounts USING (SELECT id, SUM(amount) AS total FROM staging GROUP BY id) AS agg " + + "ON accounts.id = agg.id " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (agg.id, agg.total)" + ) + assertEquals(listOf(listOf(int(1), int(30))), select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun aDefaultKeywordInTheInsertValuesUsesTheColumnDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT DEFAULT 'unlabeled')") + run("CREATE TABLE s (id INTEGER)") + run("INSERT INTO s VALUES (1)") + exec("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, DEFAULT)") + assertEquals(listOf(listOf(int(1), text("unlabeled"))), select("SELECT id, label FROM t").rows) + } + + @Test + fun anOmittedColumnListInsertsIntoEveryColumnInSchemaOrder() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + run("CREATE TABLE s (id INTEGER, label TEXT)") + run("INSERT INTO s VALUES (1, 'a')") + exec("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT VALUES (s.id, s.label)") + assertEquals(listOf(listOf(int(1), text("a"))), select("SELECT id, label FROM t").rows) + } + + @Test + fun aNextvalDefaultStillAdvancesThroughAMergeInsert() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + run("CREATE TABLE s (label TEXT)") + run("INSERT INTO s VALUES ('a'), ('b')") + exec("MERGE INTO t USING s ON t.label = s.label WHEN NOT MATCHED THEN INSERT (label) VALUES (s.label)") + assertEquals( + listOf(listOf(int(1), text("a")), listOf(int(2), text("b"))), + select("SELECT id, label FROM t ORDER BY id").rows + ) + } + + @Test + fun notNullViolationOnAMergeInsertFailsAndInsertsNothing() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT NOT NULL)") + run("CREATE TABLE s (id INTEGER)") + run("INSERT INTO s VALUES (1)") + assertFailsWith { + run("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id)") + } + assertEquals(emptyList(), select("SELECT * FROM t").rows) + } + + @Test + fun aForeignKeyViolationOnAMergeUpdateIsRejected() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("CREATE TABLE moves (id INTEGER, new_dept INTEGER)") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + run("INSERT INTO moves VALUES (1, 999)") + val error = assertFailsWith { + run("MERGE INTO employees USING moves ON employees.id = moves.id WHEN MATCHED THEN UPDATE SET dept_id = moves.new_dept") + } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message) + assertEquals(listOf(listOf(int(1))), select("SELECT dept_id FROM employees").rows) + } + + @Test + fun rowTriggersFireForEachMergeDrivenAction() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + run("CREATE TABLE s (id INTEGER, label TEXT)") + run("CREATE TABLE log (event TEXT)") + run("INSERT INTO t VALUES (1, 'old'), (2, 'gone')") + run("INSERT INTO s VALUES (1, 'new'), (2, 'new'), (3, 'fresh')") + run("CREATE TRIGGER log_insert AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('insert:', NEW.label)); END") + run("CREATE TRIGGER log_update AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('update:', NEW.label)); END") + run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('delete:', OLD.label)); END") + exec( + "MERGE INTO t USING s ON t.id = s.id " + + "WHEN MATCHED AND s.id = 2 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET label = s.label " + + "WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, s.label)" + ) + assertEquals( + setOf(text("update:new"), text("delete:gone"), text("insert:fresh")), + select("SELECT event FROM log").rows.map { it.single() }.toSet() + ) + } + + @Test + fun rollingBackATransactionUndoesEveryEffectOfAMerge() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 50), (2, 20)") + run("BEGIN") + exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) + run("ROLLBACK") + assertEquals(listOf(listOf(int(1), int(100))), select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun anAliasedTargetAndSourceCanBeReferencedByTheirAlias() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 25)") + exec( + "MERGE INTO accounts AS acc USING staging AS s ON acc.id = s.id " + + "WHEN MATCHED THEN UPDATE SET balance = acc.balance + s.amount" + ) + assertEquals(listOf(listOf(int(125))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun mergeRequiresAtLeastOneWhenClause() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE s (id INTEGER)") + assertFailsWith { run("MERGE INTO t USING s ON t.id = s.id") } + } + + @Test + fun mergingIntoAnUnknownTableFails() { + run("CREATE TABLE s (id INTEGER)") + assertFailsWith { + run("MERGE INTO nope USING s ON nope.id = s.id WHEN MATCHED THEN DELETE") + } + } + + @Test + fun returningStarProjectsEveryColumnOfEveryActionedRow() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 10), (2, 5)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount) " + + "RETURNING *" + ) as MergeResult + assertEquals(listOf("id", "balance"), result.returning!!.columns) + assertEquals( + listOf(listOf(int(1), int(110)), listOf(int(2), int(5))), + result.returning!!.rows + ) + } + + @Test + fun returningEvaluatesExpressionsAgainstTheAffectedRow() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 10)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "RETURNING balance * 2 AS doubled" + ) as MergeResult + assertEquals(listOf("doubled"), result.returning!!.columns) + assertEquals(listOf(listOf(int(220))), result.returning!!.rows) + } + + @Test + fun returningIncludesThePreDeletionImageOfADeletedRow() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 0)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id WHEN MATCHED THEN DELETE RETURNING id, balance" + ) as MergeResult + assertEquals(listOf(listOf(int(1), int(100))), result.returning!!.rows) + assertEquals(emptyList(), select("SELECT * FROM accounts").rows) + } + + @Test + fun mergeActionLabelsEachReturnedRowByWhichClauseFired() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + run("CREATE TABLE s (id INTEGER, label TEXT)") + run("INSERT INTO t VALUES (1, 'old'), (2, 'gone')") + run("INSERT INTO s VALUES (1, 'new'), (2, 'new'), (3, 'fresh')") + val result = exec( + "MERGE INTO t USING s ON t.id = s.id " + + "WHEN MATCHED AND s.id = 2 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET label = s.label " + + "WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, s.label) " + + "RETURNING id, MERGE_ACTION() AS kind" + ) as MergeResult + assertEquals( + setOf(listOf(int(1), text("UPDATE")), listOf(int(2), text("DELETE")), listOf(int(3), text("INSERT"))), + result.returning!!.rows.toSet() + ) + } + + @Test + fun noRowMatchingAnyClauseReturnsAnEmptyReturningResultNotANullOne() { + setUpAccountsAndStaging() + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 5)") + val result = exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 1000 THEN UPDATE SET balance = 0 " + + "RETURNING *" + ) as MergeResult + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun mergeActionIsRejectedOutsideOfAMergesOwnReturningClause() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + val error = assertFailsWith { select("SELECT MERGE_ACTION() FROM t") } + assertTrue(error.message!!.contains("MERGE_ACTION"), error.message) + } + + @Test + fun mergeActionInAPlainInsertReturningIsRejectedOnceARowActuallyReturns() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { + run("INSERT INTO t VALUES (1) RETURNING MERGE_ACTION()") + } + assertTrue(error.message!!.contains("MERGE_ACTION"), error.message) + } + + @Test + fun returningColumnValidationRunsUpFrontEvenWhenNoSourceRowEverReachesAMatchingClause() { + setUpAccountsAndStaging() + val error = assertFailsWith { + run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN DELETE RETURNING nope" + ) + } + assertTrue(error.message!!.contains("no such column"), error.message) + } +} diff --git a/src/test/kotlin/kestrel/OrderByNullsTest.kt b/src/test/kotlin/kestrel/OrderByNullsTest.kt new file mode 100644 index 0000000..e66f6dd --- /dev/null +++ b/src/test/kotlin/kestrel/OrderByNullsTest.kt @@ -0,0 +1,117 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * `ORDER BY expr [ASC|DESC] [NULLS FIRST|NULLS LAST]` on the in-memory engine, covering every + * comparator that shares [kestrel.engine.compareOrderKeyValues]: plain top-level ORDER BY, a + * GROUP BY's own ORDER BY (over already-aggregated rows), a window function's `OVER (... ORDER BY + * ...)`, and STRING_AGG's internal ORDER BY. [kestrel.store.sql.DiskExecutorOrderByNullsTest] mirrors + * the plain-ORDER-BY and window cases against the disk engine. + */ +class OrderByNullsTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + /** ids 2 and 4 are NULL; the rest ascend 10/20/30 out of id order so ASC/DESC actually reorder them. */ + private fun seedScored() { + run("CREATE TABLE scored (id INTEGER PRIMARY KEY, score INTEGER)") + run("INSERT INTO scored VALUES (1, 10), (2, NULL), (3, 30), (4, NULL), (5, 20)") + } + + @Test + fun ascendingOrderByDefaultsToNullsFirstButNullsLastOverridesIt() { + seedScored() + assertEquals( + listOf(2, 4, 1, 5, 3).map { listOf(int(it.toLong())) }, + select("SELECT id FROM scored ORDER BY score").rows + ) + assertEquals( + listOf(1, 5, 3, 2, 4).map { listOf(int(it.toLong())) }, + select("SELECT id FROM scored ORDER BY score NULLS LAST").rows + ) + } + + @Test + fun descendingOrderByDefaultsToNullsLastButNullsFirstOverridesIt() { + seedScored() + assertEquals( + listOf(3, 5, 1, 2, 4).map { listOf(int(it.toLong())) }, + select("SELECT id FROM scored ORDER BY score DESC").rows + ) + assertEquals( + listOf(2, 4, 3, 5, 1).map { listOf(int(it.toLong())) }, + select("SELECT id FROM scored ORDER BY score DESC NULLS FIRST").rows + ) + } + + @Test + fun nullsLastOnASecondaryKeyOnlyAffectsThatKeyInMultiColumnOrderBy() { + run("CREATE TABLE grouped2 (id INTEGER PRIMARY KEY, grp TEXT, score INTEGER)") + run("INSERT INTO grouped2 VALUES (1, 'a', NULL), (2, 'a', 5), (3, 'b', NULL), (4, 'b', 1)") + val result = select("SELECT id FROM grouped2 ORDER BY grp, score NULLS LAST") + assertEquals(listOf(2, 1, 4, 3).map { listOf(int(it.toLong())) }, result.rows) + } + + @Test + fun explicitNullsLastControlsWhereAnAllNullAggregateGroupSortsRelativeToOthers() { + run("CREATE TABLE g (id INTEGER PRIMARY KEY, dept TEXT, bonus INTEGER)") + run("INSERT INTO g VALUES (1, 'x', 100), (2, 'x', NULL), (3, 'y', NULL)") + assertEquals( + listOf(listOf(text("y"), Value.NullValue), listOf(text("x"), int(100))), + select("SELECT dept, MAX(bonus) FROM g GROUP BY dept ORDER BY MAX(bonus)").rows + ) + assertEquals( + listOf(listOf(text("x"), int(100)), listOf(text("y"), Value.NullValue)), + select("SELECT dept, MAX(bonus) FROM g GROUP BY dept ORDER BY MAX(bonus) NULLS LAST").rows + ) + } + + @Test + fun windowFunctionOrderByHonorsAnExplicitNullsLastClause() { + seedScored() + val result = select( + "SELECT id, ROW_NUMBER() OVER (ORDER BY score NULLS LAST) AS rn FROM scored ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(4)), listOf(int(3), int(3)), + listOf(int(4), int(5)), listOf(int(5), int(2)) + ), + result.rows + ) + } + + @Test + fun stringAggInternalOrderByHonorsAnExplicitNullsLastClause() { + run("CREATE TABLE namedScored (id INTEGER PRIMARY KEY, name TEXT, score INTEGER)") + run( + "INSERT INTO namedScored VALUES " + + "(1, 'a', 10), (2, 'b', NULL), (3, 'c', 30), (4, 'd', NULL), (5, 'e', 20)" + ) + assertEquals( + listOf(listOf(text("b,d,a,e,c"))), + select("SELECT STRING_AGG(name, ',' ORDER BY score) FROM namedScored").rows + ) + assertEquals( + listOf(listOf(text("a,e,c,b,d"))), + select("SELECT STRING_AGG(name, ',' ORDER BY score NULLS LAST) FROM namedScored").rows + ) + } +} diff --git a/src/test/kotlin/kestrel/ParserTest.kt b/src/test/kotlin/kestrel/ParserTest.kt index 7771760..6891e29 100644 --- a/src/test/kotlin/kestrel/ParserTest.kt +++ b/src/test/kotlin/kestrel/ParserTest.kt @@ -1,29 +1,91 @@ package kestrel +import kestrel.catalog.ForeignKeyAction +import kestrel.sql.ast.AddColumn +import kestrel.sql.ast.AlterSequenceStatement +import kestrel.sql.ast.AlterTableStatement import kestrel.sql.ast.Assignment +import kestrel.sql.ast.BetweenExpression import kestrel.sql.ast.BinaryExpression import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.CallExpression +import kestrel.sql.ast.CaseExpression +import kestrel.sql.ast.CastExpression +import kestrel.sql.ast.ChangeColumnType +import kestrel.sql.ast.CollateExpression import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.CreateIndexStatement +import kestrel.sql.ast.CreateSequenceStatement import kestrel.sql.ast.CreateTableStatement +import kestrel.sql.ast.CreateTriggerStatement +import kestrel.sql.ast.CreateViewStatement +import kestrel.sql.ast.CubeSpec +import kestrel.sql.ast.CurrentValueExpression +import kestrel.sql.ast.DeallocateStatement import kestrel.sql.ast.DeleteStatement +import kestrel.sql.ast.DropColumn +import kestrel.sql.ast.DropMode +import kestrel.sql.ast.DropSequenceStatement import kestrel.sql.ast.DropTableStatement +import kestrel.sql.ast.DropTriggerStatement +import kestrel.sql.ast.DropViewStatement +import kestrel.sql.ast.ExecuteStatement +import kestrel.sql.ast.ExistsExpression +import kestrel.sql.ast.ExplainStatement import kestrel.sql.ast.Expression import kestrel.sql.ast.ExpressionProjection +import kestrel.sql.ast.FrameBoundType +import kestrel.sql.ast.FrameExclude +import kestrel.sql.ast.FrameUnit +import kestrel.sql.ast.FunctionExpression +import kestrel.sql.ast.GroupingSetsSpec +import kestrel.sql.ast.InExpression +import kestrel.sql.ast.InSubqueryExpression import kestrel.sql.ast.InsertStatement +import kestrel.sql.ast.IsNullExpression +import kestrel.sql.ast.JoinType +import kestrel.sql.ast.LikeExpression import kestrel.sql.ast.LiteralExpression +import kestrel.sql.ast.MergeActionExpression +import kestrel.sql.ast.MergeMatchedDelete +import kestrel.sql.ast.MergeMatchedUpdate +import kestrel.sql.ast.MergeNotMatchedInsert +import kestrel.sql.ast.MergeStatement +import kestrel.sql.ast.NextValueExpression +import kestrel.sql.ast.OrderKey +import kestrel.sql.ast.ParameterExpression +import kestrel.sql.ast.PrepareStatement +import kestrel.sql.ast.QueryStatement +import kestrel.sql.ast.RenameColumn +import kestrel.sql.ast.RenameTable +import kestrel.sql.ast.RollupSpec +import kestrel.sql.ast.ScalarSubqueryExpression import kestrel.sql.ast.SelectStatement +import kestrel.sql.ast.SetOperationStatement +import kestrel.sql.ast.SetOperator import kestrel.sql.ast.StarProjection import kestrel.sql.ast.Statement +import kestrel.sql.ast.SubquerySource +import kestrel.sql.ast.TableSource +import kestrel.sql.ast.TriggerEvent +import kestrel.sql.ast.TriggerTiming +import kestrel.sql.ast.TruncateTableStatement import kestrel.sql.ast.UnaryExpression import kestrel.sql.ast.UnaryOperator import kestrel.sql.ast.UpdateStatement +import kestrel.sql.ast.WindowExpression +import kestrel.sql.ast.WindowFunctionKind +import kestrel.sql.ast.WithQuery import kestrel.sql.lexer.Lexer import kestrel.sql.parser.Parser +import kestrel.storage.Collation import kestrel.storage.ColumnType import kestrel.storage.Value import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue @@ -36,6 +98,8 @@ class ParserTest { private fun whereOf(sql: String): Expression = (parseOne(sql) as SelectStatement).where!! + private fun tableName(select: QueryStatement): String = ((select as SelectStatement).from as TableSource).name + private fun firstProjection(sql: String): Expression = ((parseOne(sql) as SelectStatement).projection as ExpressionProjection).items.first().expression @@ -55,6 +119,38 @@ class ParserTest { assertTrue(!statement.columns[2].notNull) } + @Test + fun parsesCreateTableWithoutTempIsNotTemporary() { + val statement = parseOne("CREATE TABLE users (id INTEGER PRIMARY KEY)") as CreateTableStatement + assertTrue(!statement.temporary) + } + + @Test + fun parsesCreateTempTable() { + val statement = parseOne("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, note TEXT)") as CreateTableStatement + assertTrue(statement.temporary) + assertEquals("scratch", statement.tableName) + assertEquals(2, statement.columns.size) + } + + @Test + fun parsesCreateTemporaryTable() { + val statement = parseOne("CREATE TEMPORARY TABLE scratch (id INTEGER PRIMARY KEY)") as CreateTableStatement + assertTrue(statement.temporary) + } + + @Test + fun tempKeywordIsCaseInsensitive() { + val statement = parseOne("create temp table scratch (id integer primary key)") as CreateTableStatement + assertTrue(statement.temporary) + } + + @Test + fun tempOnAnythingOtherThanTableIsAParseError() { + assertFailsWith { parse("CREATE TEMP VIEW v AS SELECT 1") } + assertFailsWith { parse("CREATE TEMP INDEX idx ON t (a)") } + } + @Test fun parsesInsertWithoutColumnList() { val statement = parseOne("INSERT INTO users VALUES (1, 'ada', 36)") @@ -71,11 +167,20 @@ class ParserTest { assertEquals(3, statement.rows.size) } + @Test + fun parsesInsertValuesWithLiteralDefaultKeyword() { + val statement = parseOne("INSERT INTO t (a, b, c) VALUES (1, DEFAULT, 3)") as InsertStatement + assertEquals(3, statement.rows[0].size) + assertNotNull(statement.rows[0][0]) + assertNull(statement.rows[0][1]) + assertNotNull(statement.rows[0][2]) + } + @Test fun parsesSelectStar() { val statement = parseOne("SELECT * FROM users") as SelectStatement assertEquals(StarProjection, statement.projection) - assertEquals("users", statement.tableName) + assertEquals("users", tableName(statement)) assertNull(statement.where) assertNull(statement.orderBy) assertNull(statement.limit) @@ -88,10 +193,90 @@ class ParserTest { ) as SelectStatement val projection = statement.projection as ExpressionProjection assertEquals("who", projection.items[0].alias) - assertEquals(true, statement.orderBy!!.descending) + assertEquals(true, statement.orderBy!!.keys.single().descending) assertEquals(5, statement.limit) } + @Test + fun parsesLimitFollowedByOffset() { + val statement = parseOne("SELECT id FROM users LIMIT 10 OFFSET 20") as SelectStatement + assertEquals(10, statement.limit) + assertEquals(20, statement.offset) + } + + @Test + fun parsesOffsetFollowedByLimitInEitherOrder() { + val statement = parseOne("SELECT id FROM users OFFSET 20 LIMIT 10") as SelectStatement + assertEquals(10, statement.limit) + assertEquals(20, statement.offset) + } + + @Test + fun parsesOffsetAloneWithoutLimit() { + val statement = parseOne("SELECT id FROM users OFFSET 5") as SelectStatement + assertNull(statement.limit) + assertEquals(5, statement.offset) + } + + @Test + fun offsetDefaultsToNullWhenAbsent() { + val statement = parseOne("SELECT id FROM users LIMIT 5") as SelectStatement + assertNull(statement.offset) + } + + @Test + fun rejectsARepeatedLimitOrOffsetClause() { + assertFailsWith { parse("SELECT id FROM users LIMIT 5 LIMIT 10") } + assertFailsWith { parse("SELECT id FROM users OFFSET 5 OFFSET 10") } + } + + @Test + fun rejectsANonIntegerAfterOffset() { + assertFailsWith { parse("SELECT id FROM users OFFSET 'five'") } + } + + @Test + fun parsesOrderByWithMultipleKeysAndIndependentDirections() { + val statement = parseOne("SELECT name FROM users ORDER BY dept ASC, age DESC, id") as SelectStatement + val keys = statement.orderBy!!.keys + assertEquals(3, keys.size) + assertEquals("dept", (keys[0].expression as ColumnExpression).name) + assertEquals(false, keys[0].descending) + assertEquals("age", (keys[1].expression as ColumnExpression).name) + assertEquals(true, keys[1].descending) + assertEquals("id", (keys[2].expression as ColumnExpression).name) + assertEquals(false, keys[2].descending) + } + + @Test + fun parsesOrderByWithExplicitNullsFirstAndNullsLastPerKey() { + val statement = parseOne( + "SELECT name FROM users ORDER BY dept NULLS FIRST, age DESC NULLS LAST, id" + ) as SelectStatement + val keys = statement.orderBy!!.keys + assertEquals(3, keys.size) + assertEquals(false, keys[0].descending) + assertEquals(true, keys[0].nullsFirst) + assertEquals(true, keys[1].descending) + assertEquals(false, keys[1].nullsFirst) + assertEquals(false, keys[2].descending) + assertNull(keys[2].nullsFirst) + } + + @Test + fun nullsAndFirstAndLastRemainUsableAsOrdinaryColumnNamesOutsideTheNullsClause() { + val statement = parseOne("SELECT id FROM users ORDER BY nulls DESC") as SelectStatement + val key = statement.orderBy!!.keys.single() + assertEquals(ColumnExpression("nulls"), key.expression) + assertEquals(true, key.descending) + assertNull(key.nullsFirst) + } + + @Test + fun nullsClauseWithoutFirstOrLastFailsToParse() { + assertFailsWith { parseOne("SELECT id FROM users ORDER BY age NULLS") } + } + @Test fun parsesDeleteAndDropAndUpdate() { assertTrue(parseOne("DELETE FROM t WHERE id = 1") is DeleteStatement) @@ -101,6 +286,73 @@ class ParserTest { assertEquals(Assignment("a", intLit(1)), update.assignments[0]) } + @Test + fun updateSetToDefaultParsesTheAssignmentValueAsNull() { + val update = parseOne("UPDATE t SET a = DEFAULT, b = 2 WHERE id = 3") as UpdateStatement + assertEquals(Assignment("a", null), update.assignments[0]) + assertEquals(Assignment("b", intLit(2)), update.assignments[1]) + } + + @Test + fun insertWithoutReturningLeavesItNull() { + val statement = parseOne("INSERT INTO t VALUES (1)") as InsertStatement + assertNull(statement.returning) + } + + @Test + fun insertReturningStarParsesAsStarProjection() { + val statement = parseOne("INSERT INTO t VALUES (1) RETURNING *") as InsertStatement + assertEquals(StarProjection, statement.returning) + } + + @Test + fun insertReturningExpressionListWithAliasParsesLikeASelectProjection() { + val statement = parseOne("INSERT INTO t (a, b) VALUES (1, 2) RETURNING a, b + 1 AS bplus") as InsertStatement + val projection = statement.returning as ExpressionProjection + assertEquals(2, projection.items.size) + assertEquals("a", (projection.items[0].expression as ColumnExpression).name) + assertNull(projection.items[0].alias) + assertEquals("bplus", projection.items[1].alias) + } + + @Test + fun insertOnConflictReturningParsesReturningAfterTheConflictClause() { + val statement = parseOne( + "INSERT INTO t (id) VALUES (1) ON CONFLICT (id) DO NOTHING RETURNING id" + ) as InsertStatement + assertNotNull(statement.onConflict) + val projection = statement.returning as ExpressionProjection + assertEquals("id", (projection.items.single().expression as ColumnExpression).name) + } + + @Test + fun updateReturningParsesAfterWhere() { + val statement = parseOne("UPDATE t SET a = 1 WHERE id = 3 RETURNING id, a") as UpdateStatement + val projection = statement.returning as ExpressionProjection + assertEquals(2, projection.items.size) + } + + @Test + fun updateReturningWithoutWhereStillParses() { + val statement = parseOne("UPDATE t SET a = 1 RETURNING *") as UpdateStatement + assertNull(statement.where) + assertEquals(StarProjection, statement.returning) + } + + @Test + fun deleteReturningParsesAfterWhere() { + val statement = parseOne("DELETE FROM t WHERE id = 3 RETURNING id") as DeleteStatement + val projection = statement.returning as ExpressionProjection + assertEquals("id", (projection.items.single().expression as ColumnExpression).name) + } + + @Test + fun deleteReturningWithoutWhereStillParses() { + val statement = parseOne("DELETE FROM t RETURNING *") as DeleteStatement + assertNull(statement.where) + assertEquals(StarProjection, statement.returning) + } + @Test fun multipleStatementsAreSeparatedBySemicolons() { val statements = parse("CREATE TABLE t (a INTEGER); DROP TABLE t;") @@ -167,4 +419,1746 @@ class ParserTest { fun reportsUnknownStatement() { assertFailsWith { parse("EXPLAIN users") } } + + @Test + fun parsesScalarFunctionCallWithMultipleArguments() { + val expression = firstProjection("SELECT SUBSTR(name, 1, 3) FROM t") + assertEquals( + CallExpression("SUBSTR", listOf(ColumnExpression("name"), intLit(1), intLit(3))), + expression + ) + } + + @Test + fun parsesScalarFunctionCallWithNoArguments() { + assertEquals(CallExpression("NOW", emptyList()), firstProjection("SELECT NOW() FROM t")) + } + + @Test + fun aggregateFunctionWithWrongArityFails() { + assertFailsWith { parse("SELECT SUM(a, b) FROM t") } + assertFailsWith { parse("SELECT COUNT() FROM t") } + } + + @Test + fun stringAggParsesTheValueAndSeparatorWithNoOrderByLeavingItEmpty() { + val expression = firstProjection("SELECT STRING_AGG(name, ', ') FROM t") + assertEquals( + FunctionExpression( + "STRING_AGG", + ColumnExpression("name"), + star = false, + separator = LiteralExpression(Value.TextValue(", ")), + orderBy = emptyList() + ), + expression + ) + } + + @Test + fun stringAggParsesAnInternalOrderByWithMultipleKeysAndExplicitDirections() { + val expression = firstProjection("SELECT STRING_AGG(name, ',' ORDER BY dept ASC, name DESC) FROM t") + assertEquals( + FunctionExpression( + "STRING_AGG", + ColumnExpression("name"), + star = false, + separator = LiteralExpression(Value.TextValue(",")), + orderBy = listOf( + OrderKey(ColumnExpression("dept"), descending = false), + OrderKey(ColumnExpression("name"), descending = true) + ) + ), + expression + ) + } + + @Test + fun stringAggParsesAnExplicitNullsLastInsideItsInternalOrderBy() { + val expression = firstProjection("SELECT STRING_AGG(name, ',' ORDER BY dept NULLS LAST) FROM t") + assertEquals( + FunctionExpression( + "STRING_AGG", + ColumnExpression("name"), + star = false, + separator = LiteralExpression(Value.TextValue(",")), + orderBy = listOf(OrderKey(ColumnExpression("dept"), descending = false, nullsFirst = false)) + ), + expression + ) + } + + @Test + fun stringAggRequiresBothAValueAndASeparatorArgument() { + assertFailsWith { parse("SELECT STRING_AGG(name) FROM t") } + assertFailsWith { parse("SELECT STRING_AGG() FROM t") } + } + + @Test + fun stringAggCannotBeUsedAsAWindowFunction() { + assertFailsWith { parse("SELECT STRING_AGG(name, ',') OVER () FROM t") } + } + + @Test + fun countDistinctSetsTheDistinctFlagWithTheGivenArgumentAsTheSoleArgument() { + val expression = firstProjection("SELECT COUNT(DISTINCT dept) FROM t") + assertEquals( + FunctionExpression("COUNT", ColumnExpression("dept"), star = false, distinct = true), + expression + ) + } + + @Test + fun sumAvgMinMaxAcceptDistinctTooNotJustCount() { + for (name in listOf("SUM", "AVG", "MIN", "MAX")) { + val expression = firstProjection("SELECT $name(DISTINCT amount) FROM t") + assertEquals( + FunctionExpression(name, ColumnExpression("amount"), star = false, distinct = true), + expression + ) + } + } + + @Test + fun plainAggregateCallsDefaultDistinctToFalse() { + val expression = firstProjection("SELECT COUNT(dept) FROM t") as FunctionExpression + assertEquals(false, expression.distinct) + } + + @Test + fun countDistinctStarIsRejected() { + assertFailsWith { parse("SELECT COUNT(DISTINCT *) FROM t") } + } + + @Test + fun countDistinctWithMultipleArgumentsIsRejected() { + assertFailsWith { parse("SELECT COUNT(DISTINCT dept, region) FROM t") } + } + + @Test + fun distinctAggregateCannotBeUsedAsAWindowFunction() { + assertFailsWith { parse("SELECT COUNT(DISTINCT dept) OVER () FROM t") } + assertFailsWith { parse("SELECT SUM(DISTINCT amount) OVER () FROM t") } + } + + @Test + fun aggregateFilterSetsTheFilterFieldToTheWhereCondition() { + val expression = firstProjection("SELECT COUNT(dept) FILTER (WHERE amount > 100) FROM t") + assertEquals( + FunctionExpression( + "COUNT", + ColumnExpression("dept"), + star = false, + filter = BinaryExpression(BinaryOperator.Greater, ColumnExpression("amount"), LiteralExpression(Value.IntegerValue(100))) + ), + expression + ) + } + + @Test + fun countStarFilterKeepsStarTrueAlongsideTheFilter() { + val expression = firstProjection("SELECT COUNT(*) FILTER (WHERE amount > 0) FROM t") as FunctionExpression + assertEquals(true, expression.star) + assertEquals(true, expression.filter != null) + } + + @Test + fun plainAggregateCallsDefaultFilterToNull() { + val expression = firstProjection("SELECT COUNT(dept) FROM t") as FunctionExpression + assertEquals(null, expression.filter) + } + + @Test + fun filterCanBeCombinedWithDistinct() { + val expression = firstProjection("SELECT COUNT(DISTINCT dept) FILTER (WHERE amount > 0) FROM t") as FunctionExpression + assertEquals(true, expression.distinct) + assertEquals(true, expression.filter != null) + } + + @Test + fun filterRequiresAWhereClauseInsideItsParentheses() { + assertFailsWith { parse("SELECT COUNT(dept) FILTER (amount > 100) FROM t") } + } + + @Test + fun filterIsRejectedOnNonAggregateFunctions() { + assertFailsWith { parse("SELECT UPPER(dept) FILTER (WHERE amount > 0) FROM t") } + } + + @Test + fun filterCannotBeUsedWithAWindowFunction() { + assertFailsWith { parse("SELECT COUNT(dept) FILTER (WHERE amount > 0) OVER () FROM t") } + } + + @Test + fun filterColumnNamedFilterIsStillUsableAsAPlainIdentifier() { + val expression = firstProjection("SELECT filter FROM t") as ColumnExpression + assertEquals("filter", expression.name) + } + + // LEFT/RIGHT are otherwise reserved (LEFT/RIGHT JOIN), so LEFT(str, n)/RIGHT(str, n) as scalar + // functions each need their own disambiguation in the parser. + @Test + fun leftParsesAsAScalarFunctionCallDespiteAlsoBeingTheLeftJoinKeyword() { + val expression = firstProjection("SELECT LEFT(name, 3) FROM t") + assertEquals(CallExpression("LEFT", listOf(ColumnExpression("name"), intLit(3))), expression) + } + + @Test + fun rightParsesAsAScalarFunctionCallDespiteAlsoBeingTheRightJoinKeyword() { + val expression = firstProjection("SELECT RIGHT(name, 3) FROM t") + assertEquals(CallExpression("RIGHT", listOf(ColumnExpression("name"), intLit(3))), expression) + } + + @Test + fun leftJoinStillParsesAsAJoinNotAFunctionCall() { + val statement = parseOne("SELECT * FROM t LEFT JOIN u ON t.id = u.id") as SelectStatement + assertEquals(1, statement.joins.size) + assertEquals(JoinType.Left, statement.joins[0].type) + } + + @Test + fun bareLeftKeywordWithoutAFollowingParenIsAParseError() { + assertFailsWith { parse("SELECT LEFT FROM t") } + } + + @Test + fun bareRightKeywordWithoutAFollowingParenIsAParseError() { + assertFailsWith { parse("SELECT RIGHT FROM t") } + } + + @Test + fun rightJoinAndFullJoinParseWithOptionalOuter() { + val right = parseOne("SELECT * FROM t RIGHT JOIN u ON t.id = u.id") as SelectStatement + assertEquals(JoinType.Right, right.joins[0].type) + val rightOuter = parseOne("SELECT * FROM t RIGHT OUTER JOIN u ON t.id = u.id") as SelectStatement + assertEquals(JoinType.Right, rightOuter.joins[0].type) + val full = parseOne("SELECT * FROM t FULL JOIN u ON t.id = u.id") as SelectStatement + assertEquals(JoinType.Full, full.joins[0].type) + val fullOuter = parseOne("SELECT * FROM t FULL OUTER JOIN u ON t.id = u.id") as SelectStatement + assertEquals(JoinType.Full, fullOuter.joins[0].type) + } + + @Test + fun rightAndFullJoinLateralAreRejected() { + assertFailsWith { + parse("SELECT * FROM t RIGHT JOIN LATERAL (SELECT * FROM u WHERE u.id = t.id) x ON true") + } + assertFailsWith { + parse("SELECT * FROM t FULL JOIN LATERAL (SELECT * FROM u WHERE u.id = t.id) x ON true") + } + } + + @Test + fun crossJoinParsesWithoutAnOnClauseAndOnIsNull() { + val statement = parseOne("SELECT * FROM t CROSS JOIN u") as SelectStatement + assertEquals(JoinType.Cross, statement.joins[0].type) + assertEquals(null, statement.joins[0].on) + } + + @Test + fun crossJoinWithAnExplicitOnClauseIsAParseError() { + assertFailsWith { parse("SELECT * FROM t CROSS JOIN u ON t.id = u.id") } + } + + @Test + fun crossJoinLateralIsAccepted() { + val statement = parseOne( + "SELECT * FROM t CROSS JOIN LATERAL (SELECT * FROM u WHERE u.id = t.id) x" + ) as SelectStatement + assertEquals(JoinType.Cross, statement.joins[0].type) + assertEquals(null, statement.joins[0].on) + } + + @Test + fun parsesIsNullAndIsNotNull() { + assertEquals(IsNullExpression(ColumnExpression("age"), negated = false), whereOf("SELECT 1 FROM t WHERE age IS NULL")) + assertEquals( + IsNullExpression(ColumnExpression("age"), negated = true), + whereOf("SELECT 1 FROM t WHERE age IS NOT NULL") + ) + } + + @Test + fun parsesInAndNotIn() { + assertEquals( + InExpression(ColumnExpression("id"), listOf(intLit(1), intLit(2), intLit(3)), negated = false), + whereOf("SELECT 1 FROM t WHERE id IN (1, 2, 3)") + ) + assertEquals( + InExpression(ColumnExpression("id"), listOf(intLit(1)), negated = true), + whereOf("SELECT 1 FROM t WHERE id NOT IN (1)") + ) + } + + @Test + fun parsesBetweenAndNotBetween() { + assertEquals( + BetweenExpression(ColumnExpression("age"), intLit(18), intLit(65), negated = false), + whereOf("SELECT 1 FROM t WHERE age BETWEEN 18 AND 65") + ) + assertEquals( + BetweenExpression(ColumnExpression("age"), intLit(18), intLit(65), negated = true), + whereOf("SELECT 1 FROM t WHERE age NOT BETWEEN 18 AND 65") + ) + } + + @Test + fun parsesLikeAndNotLike() { + assertEquals( + LikeExpression(ColumnExpression("name"), LiteralExpression(Value.TextValue("a%")), negated = false), + whereOf("SELECT 1 FROM t WHERE name LIKE 'a%'") + ) + assertEquals( + LikeExpression(ColumnExpression("name"), LiteralExpression(Value.TextValue("a%")), negated = true), + whereOf("SELECT 1 FROM t WHERE name NOT LIKE 'a%'") + ) + } + + @Test + fun notInWithoutInOrBetweenOrLikeFails() { + assertFailsWith { parse("SELECT 1 FROM t WHERE id NOT = 1") } + } + + @Test + fun parsesSearchedCaseExpression() { + val expression = firstProjection( + "SELECT CASE WHEN age < 18 THEN 'minor' WHEN age < 65 THEN 'adult' ELSE 'senior' END FROM t" + ) as CaseExpression + assertNull(expression.operand) + assertEquals(2, expression.branches.size) + assertEquals(LiteralExpression(Value.TextValue("senior")), expression.elseExpression) + } + + @Test + fun parsesSimpleCaseExpressionWithoutElse() { + val expression = firstProjection("SELECT CASE grade WHEN 1 THEN 'a' END FROM t") as CaseExpression + assertEquals(ColumnExpression("grade"), expression.operand) + assertEquals(1, expression.branches.size) + assertNull(expression.elseExpression) + } + + @Test + fun caseWithoutWhenClauseFails() { + assertFailsWith { parse("SELECT CASE WHEN THEN 1 END FROM t") } + assertFailsWith { parse("SELECT CASE ELSE 1 END FROM t") } + } + + @Test + fun caseMissingEndFails() { + assertFailsWith { parse("SELECT CASE WHEN 1 THEN 2 FROM t") } + } + + @Test + fun parsesScalarSubqueryInWhereClause() { + val expression = whereOf("SELECT 1 FROM t WHERE age > (SELECT AVG(age) FROM t)") as BinaryExpression + val subquery = expression.right as ScalarSubqueryExpression + assertEquals("t", tableName(subquery.select)) + assertEquals(BinaryOperator.Greater, expression.operator) + } + + @Test + fun parsesInSubquery() { + val expression = whereOf("SELECT 1 FROM t WHERE id IN (SELECT id FROM other)") as InSubqueryExpression + assertEquals(ColumnExpression("id"), expression.operand) + assertEquals("other", tableName(expression.select)) + assertEquals(false, expression.negated) + } + + @Test + fun parsesNotInSubquery() { + val expression = whereOf("SELECT 1 FROM t WHERE id NOT IN (SELECT id FROM other)") as InSubqueryExpression + assertEquals(true, expression.negated) + } + + @Test + fun parsesExistsAndNotExists() { + val exists = whereOf("SELECT 1 FROM t WHERE EXISTS (SELECT 1 FROM other)") as ExistsExpression + assertEquals("other", tableName(exists.select)) + val notExists = whereOf("SELECT 1 FROM t WHERE NOT EXISTS (SELECT 1 FROM other)") as UnaryExpression + assertEquals(UnaryOperator.Not, notExists.operator) + assertEquals("other", tableName((notExists.operand as ExistsExpression).select)) + } + + @Test + fun parenthesizedNonSelectExpressionIsStillJustGrouping() { + assertEquals(intLit(1), firstProjection("SELECT (1) FROM t")) + } + + @Test + fun parsesCreateView() { + val statement = parseOne("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") as CreateViewStatement + assertEquals("adults", statement.viewName) + assertEquals(false, statement.orReplace) + assertEquals("users", tableName(statement.query)) + } + + @Test + fun parsesCreateOrReplaceView() { + val statement = parseOne("CREATE OR REPLACE VIEW adults AS SELECT id FROM users") as CreateViewStatement + assertEquals(true, statement.orReplace) + } + + @Test + fun orReplaceOnNonViewFails() { + assertFailsWith { parseOne("CREATE OR REPLACE TABLE t (id INTEGER)") } + } + + @Test + fun parsesDropView() { + val statement = parseOne("DROP VIEW adults") as DropViewStatement + assertEquals("adults", statement.viewName) + assertEquals(DropMode.Restrict, statement.mode) + } + + @Test + fun dropTableAndDropViewAcceptExplicitCascadeAndRestrict() { + assertEquals(DropMode.Cascade, (parseOne("DROP TABLE t CASCADE") as DropTableStatement).mode) + assertEquals(DropMode.Restrict, (parseOne("DROP TABLE t RESTRICT") as DropTableStatement).mode) + assertEquals(DropMode.Cascade, (parseOne("DROP VIEW v CASCADE") as DropViewStatement).mode) + assertEquals(DropMode.Restrict, (parseOne("DROP VIEW v RESTRICT") as DropViewStatement).mode) + } + + @Test + fun parsesTruncateWithAndWithoutTheTableKeyword() { + val withKeyword = parseOne("TRUNCATE TABLE t") as TruncateTableStatement + assertEquals(listOf("t"), withKeyword.tableNames) + assertEquals(false, withKeyword.restartIdentity) + assertEquals(DropMode.Restrict, withKeyword.mode) + + val withoutKeyword = parseOne("TRUNCATE t") as TruncateTableStatement + assertEquals(listOf("t"), withoutKeyword.tableNames) + } + + @Test + fun parsesTruncateWithMultipleTableNames() { + val statement = parseOne("TRUNCATE TABLE a, b, c") as TruncateTableStatement + assertEquals(listOf("a", "b", "c"), statement.tableNames) + } + + @Test + fun parsesTruncateRestartIdentityAndContinueIdentity() { + assertEquals(true, (parseOne("TRUNCATE TABLE t RESTART IDENTITY") as TruncateTableStatement).restartIdentity) + assertEquals(false, (parseOne("TRUNCATE TABLE t CONTINUE IDENTITY") as TruncateTableStatement).restartIdentity) + assertEquals(false, (parseOne("TRUNCATE TABLE t") as TruncateTableStatement).restartIdentity) + } + + @Test + fun parsesTruncateCascadeAndRestrictAfterAnIdentityClause() { + val statement = parseOne("TRUNCATE TABLE t RESTART IDENTITY CASCADE") as TruncateTableStatement + assertEquals(true, statement.restartIdentity) + assertEquals(DropMode.Cascade, statement.mode) + } + + @Test + fun parsesMergeWithATableSourceAndAnUpdateClause() { + val statement = parseOne( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = staging.balance" + ) as MergeStatement + assertEquals("accounts", statement.targetTable) + assertNull(statement.targetAlias) + val source = statement.source as TableSource + assertEquals("staging", source.name) + assertNull(source.alias) + assertTrue(statement.onCondition is BinaryExpression) + assertEquals(1, statement.whenClauses.size) + val clause = statement.whenClauses.single() as MergeMatchedUpdate + assertNull(clause.condition) + assertEquals(listOf(Assignment("balance", ColumnExpression("balance", "staging"))), clause.assignments) + } + + @Test + fun parsesMergeTargetAndSourceAliasesWithAndWithoutAs() { + val explicitAs = parseOne( + "MERGE INTO accounts AS acc USING staging AS s ON acc.id = s.id WHEN MATCHED THEN DELETE" + ) as MergeStatement + assertEquals("acc", explicitAs.targetAlias) + assertEquals("s", (explicitAs.source as TableSource).alias) + + val bareAlias = parseOne( + "MERGE INTO accounts acc USING staging s ON acc.id = s.id WHEN MATCHED THEN DELETE" + ) as MergeStatement + assertEquals("acc", bareAlias.targetAlias) + assertEquals("s", (bareAlias.source as TableSource).alias) + } + + @Test + fun parsesMergeUsingADerivedTableSubquerySource() { + val statement = parseOne( + "MERGE INTO accounts USING (SELECT id FROM staging) AS s ON accounts.id = s.id WHEN MATCHED THEN DELETE" + ) as MergeStatement + val source = statement.source as SubquerySource + assertEquals("s", source.alias) + assertTrue(source.select is SelectStatement) + } + + @Test + fun parsesMergeWhenMatchedDeleteWithAnAndCondition() { + val statement = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED AND s.flag THEN DELETE" + ) as MergeStatement + val clause = statement.whenClauses.single() as MergeMatchedDelete + assertEquals(ColumnExpression("flag", "s"), clause.condition) + } + + @Test + fun parsesMergeWhenNotMatchedInsertWithAndWithoutAColumnList() { + val withColumns = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, s.label)" + ) as MergeStatement + val withColumnsClause = withColumns.whenClauses.single() as MergeNotMatchedInsert + assertEquals(listOf("id", "label"), withColumnsClause.columns) + assertEquals(listOf(ColumnExpression("id", "s"), ColumnExpression("label", "s")), withColumnsClause.values) + + val withoutColumns = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT VALUES (s.id, DEFAULT)" + ) as MergeStatement + val withoutColumnsClause = withoutColumns.whenClauses.single() as MergeNotMatchedInsert + assertNull(withoutColumnsClause.columns) + assertEquals(listOf(ColumnExpression("id", "s"), null), withoutColumnsClause.values) + } + + @Test + fun parsesMergeWhenNotMatchedWithAnAndCondition() { + val statement = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED AND s.active THEN INSERT (id) VALUES (s.id)" + ) as MergeStatement + val clause = statement.whenClauses.single() as MergeNotMatchedInsert + assertEquals(ColumnExpression("active", "s"), clause.condition) + } + + @Test + fun parsesMergeWithMultipleWhenClausesInWrittenOrder() { + val statement = parseOne( + "MERGE INTO t USING s ON t.id = s.id " + + "WHEN MATCHED AND s.gone THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET v = s.v " + + "WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id)" + ) as MergeStatement + assertEquals(3, statement.whenClauses.size) + assertTrue(statement.whenClauses[0] is MergeMatchedDelete) + assertTrue(statement.whenClauses[1] is MergeMatchedUpdate) + assertTrue(statement.whenClauses[2] is MergeNotMatchedInsert) + } + + @Test + fun mergeWithNoWhenClauseFailsToParse() { + assertFailsWith { parseOne("MERGE INTO t USING s ON t.id = s.id") } + } + + @Test + fun mergeWhenMatchedRequiresUpdateOrDelete() { + assertFailsWith { parseOne("MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN INSERT (id) VALUES (s.id)") } + } + + @Test + fun mergeWhenNotMatchedOnlySupportsInsert() { + assertFailsWith { parseOne("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN DELETE") } + } + + @Test + fun mergeWithoutReturningLeavesItNull() { + val statement = parseOne("MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE") as MergeStatement + assertNull(statement.returning) + } + + @Test + fun mergeReturningStarParsesAsStarProjectionAfterTheLastWhenClause() { + val statement = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE RETURNING *" + ) as MergeStatement + assertEquals(StarProjection, statement.returning) + } + + @Test + fun mergeReturningExpressionListParsesLikeASelectProjection() { + val statement = parseOne( + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE RETURNING id, MERGE_ACTION() AS kind" + ) as MergeStatement + val projection = statement.returning as ExpressionProjection + assertEquals(2, projection.items.size) + assertEquals("id", (projection.items[0].expression as ColumnExpression).name) + assertNull(projection.items[0].alias) + assertEquals(MergeActionExpression, projection.items[1].expression) + assertEquals("kind", projection.items[1].alias) + } + + @Test + fun mergeActionParsesAsItsOwnExpressionNode() { + val select = parseOne("SELECT MERGE_ACTION() FROM t") as SelectStatement + val items = (select.projection as ExpressionProjection).items + assertEquals(MergeActionExpression, items.single().expression) + } + + @Test + fun mergeActionTakesNoArguments() { + assertFailsWith { parse("SELECT MERGE_ACTION(1)") } + assertFailsWith { parse("SELECT MERGE_ACTION('x')") } + } + + @Test + fun parsesDecimalColumnWithAndWithoutPrecisionAndScale() { + val statement = parseOne( + "CREATE TABLE t (bare DECIMAL, scaled DECIMAL(10), precise DECIMAL(10, 2))" + ) as CreateTableStatement + assertEquals(ColumnType.Decimal, statement.columns[0].type) + assertNull(statement.columns[0].precision) + assertNull(statement.columns[0].scale) + assertEquals(10, statement.columns[1].precision) + assertEquals(0, statement.columns[1].scale) + assertEquals(10, statement.columns[2].precision) + assertEquals(2, statement.columns[2].scale) + } + + @Test + fun parsesDateColumn() { + val statement = parseOne("CREATE TABLE t (born DATE)") as CreateTableStatement + assertEquals(ColumnType.Date, statement.columns[0].type) + } + + @Test + fun parsesCastExpressionWithAndWithoutPrecisionAndScale() { + val plain = firstProjection("SELECT CAST(a AS INTEGER) FROM t") as CastExpression + assertEquals(ColumnType.Integer, plain.targetType) + assertNull(plain.precision) + + val decimal = firstProjection("SELECT CAST(a AS DECIMAL(10, 2)) FROM t") as CastExpression + assertEquals(ColumnType.Decimal, decimal.targetType) + assertEquals(10, decimal.precision) + assertEquals(2, decimal.scale) + } + + @Test + fun parsesCollateExpressionWithEachBuiltInSequenceCaseInsensitively() { + val nocase = firstProjection("SELECT name COLLATE NOCASE FROM t") as CollateExpression + assertEquals(Collation.NoCase, nocase.collation) + assertEquals(ColumnExpression("name"), nocase.operand) + + val rtrim = firstProjection("SELECT name COLLATE rtrim FROM t") as CollateExpression + assertEquals(Collation.Rtrim, rtrim.collation) + + val binary = firstProjection("SELECT name COLLATE BINARY FROM t") as CollateExpression + assertEquals(Collation.Binary, binary.collation) + } + + @Test + fun collateWithAnUnknownSequenceNameFailsToParse() { + assertFailsWith { parse("SELECT name COLLATE BOGUS FROM t") } + } + + @Test + fun collateWithoutAFollowingNameFailsToParse() { + assertFailsWith { parse("SELECT name COLLATE FROM t") } + } + + @Test + fun collateBindsTighterThanComparisonSoItWrapsOnlyTheColumnItFollows() { + val where = whereOf("SELECT 1 FROM t WHERE name COLLATE NOCASE = 'bob'") + val comparison = where as BinaryExpression + assertEquals(BinaryOperator.Equal, comparison.operator) + val left = comparison.left as CollateExpression + assertEquals(Collation.NoCase, left.collation) + assertEquals(ColumnExpression("name"), left.operand) + assertEquals(LiteralExpression(Value.TextValue("bob")), comparison.right) + } + + @Test + fun collateBindsTighterThanMultiplicationSoItWrapsOnlyItsOwnOperand() { + val projection = firstProjection("SELECT a * b COLLATE NOCASE FROM t") as BinaryExpression + assertEquals(BinaryOperator.Multiply, projection.operator) + assertEquals(ColumnExpression("a"), projection.left) + val right = projection.right as CollateExpression + assertEquals(Collation.NoCase, right.collation) + assertEquals(ColumnExpression("b"), right.operand) + } + + @Test + fun collateBindsLooserThanUnaryMinusSoItWrapsTheWholeNegation() { + val projection = firstProjection("SELECT -a COLLATE NOCASE FROM t") as CollateExpression + assertEquals(Collation.NoCase, projection.collation) + val negated = projection.operand as UnaryExpression + assertEquals(UnaryOperator.Negate, negated.operator) + assertEquals(ColumnExpression("a"), negated.operand) + } + + @Test + fun chainedCollateWrapsEachPreviousCollateExpression() { + val projection = firstProjection("SELECT a COLLATE BINARY COLLATE NOCASE FROM t") as CollateExpression + assertEquals(Collation.NoCase, projection.collation) + val inner = projection.operand as CollateExpression + assertEquals(Collation.Binary, inner.collation) + assertEquals(ColumnExpression("a"), inner.operand) + } + + @Test + fun parsesColumnLevelCollateInCreateTable() { + val statement = parseOne("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, other TEXT)") as CreateTableStatement + assertEquals(Collation.NoCase, statement.columns[1].collation) + assertEquals(Collation.Binary, statement.columns[0].collation) + assertEquals(Collation.Binary, statement.columns[2].collation) + } + + /** + * COLLATE before DEFAULT is unambiguous: the column-definition loop consumes it as its own + * clause before ever calling [Parser.parseExpression] for DEFAULT's value. The other order + * (`DEFAULT 'x' COLLATE NOCASE`) is deliberately not covered here - `parseExpression` itself + * accepts a trailing `COLLATE` (see `parseCollate`), so it greedily becomes part of the DEFAULT + * expression instead (`DEFAULT ('x' COLLATE NOCASE)`), not a separate column-level clause; not a + * bug, just the same expression-grammar greediness this column-level COLLATE reuses. + */ + @Test + fun columnLevelCollateBeforeDefaultAttachesToTheColumnNotTheDefaultExpression() { + val statement = parseOne("CREATE TABLE t (name TEXT COLLATE NOCASE DEFAULT 'x')") as CreateTableStatement + assertEquals(Collation.NoCase, statement.columns[0].collation) + assertEquals(LiteralExpression(Value.TextValue("x")), statement.columns[0].default) + } + + @Test + fun columnLevelCollateWithAnUnknownSequenceNameFailsToParse() { + assertFailsWith { parse("CREATE TABLE t (name TEXT COLLATE BOGUS)") } + } + + @Test + fun columnLevelCollateWithoutAFollowingNameFailsToParse() { + assertFailsWith { parse("CREATE TABLE t (name TEXT COLLATE)") } + } + + @Test + fun parsesColumnLevelCollateInAlterTableAddColumn() { + val statement = parseOne("ALTER TABLE t ADD COLUMN name TEXT COLLATE NOCASE") as AlterTableStatement + val action = statement.action as AddColumn + assertEquals(Collation.NoCase, action.column.collation) + } + + @Test + fun parsesDerivedTableInFromWithRequiredAlias() { + val statement = parseOne("SELECT s.id FROM (SELECT id FROM users) AS s") as SelectStatement + val from = statement.from as SubquerySource + assertEquals("s", from.alias) + assertEquals("users", tableName(from.select)) + } + + @Test + fun parsesDerivedTableWithImplicitAlias() { + val statement = parseOne("SELECT s.id FROM (SELECT id FROM users) s") as SelectStatement + assertEquals("s", (statement.from as SubquerySource).alias) + } + + @Test + fun derivedTableWithoutAliasFails() { + assertFailsWith { parseOne("SELECT id FROM (SELECT id FROM users)") } + } + + @Test + fun parsesDerivedTableAsJoinTarget() { + val statement = parseOne( + "SELECT * FROM users JOIN (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d ON users.dept = d.dept" + ) as SelectStatement + val joinFrom = statement.joins.single().from as SubquerySource + assertEquals("d", joinFrom.alias) + } + + @Test + fun parsesLateralJoinMarksTheSubquerySource() { + val statement = parseOne( + "SELECT * FROM users u JOIN LATERAL (SELECT id FROM orders WHERE orders.userId = u.id) AS o ON TRUE" + ) as SelectStatement + val joinFrom = statement.joins.single().from as SubquerySource + assertEquals("o", joinFrom.alias) + assertTrue(joinFrom.lateral) + } + + @Test + fun nonLateralDerivedTableJoinIsNotMarkedLateral() { + val statement = parseOne("SELECT * FROM users JOIN (SELECT id FROM orders) AS o ON TRUE") as SelectStatement + assertFalse((statement.joins.single().from as SubquerySource).lateral) + } + + @Test + fun lateralBeforeAPlainTableFails() { + assertFailsWith { parseOne("SELECT * FROM users JOIN LATERAL orders ON TRUE") } + } + + @Test + fun parsesRankingWindowFunctionWithPartitionAndOrder() { + val statement = parseOne( + "SELECT ROW_NUMBER() OVER (PARTITION BY dept ORDER BY salary DESC) AS rn FROM users" + ) as SelectStatement + val item = (statement.projection as ExpressionProjection).items.single() + val window = item.expression as WindowExpression + assertEquals("ROW_NUMBER", window.name) + assertEquals(WindowFunctionKind.RowNumber, window.kind) + assertNull(window.argument) + assertFalse(window.star) + assertEquals(listOf(ColumnExpression("dept")), window.partitionBy) + assertEquals(1, window.orderBy.size) + assertEquals(ColumnExpression("salary"), window.orderBy.single().expression) + assertTrue(window.orderBy.single().descending) + assertEquals("rn", item.alias) + } + + @Test + fun parsesAggregateWindowFunctionWithStarAndNoPartition() { + val statement = parseOne("SELECT COUNT(*) OVER () FROM users") as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(WindowFunctionKind.Aggregate, window.kind) + assertTrue(window.star) + assertNull(window.argument) + assertTrue(window.partitionBy.isEmpty()) + assertTrue(window.orderBy.isEmpty()) + } + + @Test + fun aggregateWindowFunctionWithSingleArgumentAndOrderBy() { + val statement = parseOne("SELECT SUM(salary) OVER (ORDER BY id) FROM users") as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals("SUM", window.name) + assertEquals(ColumnExpression("salary"), window.argument) + assertFalse(window.star) + assertEquals(listOf(ColumnExpression("id")), window.orderBy.map { it.expression }) + } + + @Test + fun windowOverOrderByParsesAnExplicitNullsFirstClause() { + val statement = parseOne("SELECT SUM(salary) OVER (ORDER BY bonus NULLS FIRST) FROM users") as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val key = window.orderBy.single() + assertEquals(ColumnExpression("bonus"), key.expression) + assertEquals(false, key.descending) + assertEquals(true, key.nullsFirst) + } + + @Test + fun functionNameWithoutOverIsNotTreatedAsAWindowFunction() { + val statement = parseOne("SELECT ROW_NUMBER() FROM users") as SelectStatement + val expression = (statement.projection as ExpressionProjection).items.single().expression + assertTrue(expression is CallExpression) + assertEquals("ROW_NUMBER", (expression as CallExpression).name) + } + + @Test + fun rowNumberWithArgumentsFailsToParse() { + assertFailsWith { parseOne("SELECT ROW_NUMBER(id) OVER () FROM users") } + } + + @Test + fun rankWithAnArgumentFailsToParse() { + assertFailsWith { parseOne("SELECT RANK(id) OVER (ORDER BY id) FROM users") } + } + + @Test + fun nonWindowableFunctionBeforeOverFailsToParse() { + assertFailsWith { parseOne("SELECT UPPER(name) OVER () FROM users") } + } + + @Test + fun lagWithOffsetAndDefaultParsesIntoTheirOwnFields() { + val statement = parseOne( + "SELECT LAG(salary, 2, 0) OVER (PARTITION BY dept ORDER BY id) AS prev FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals("LAG", window.name) + assertEquals(WindowFunctionKind.Lag, window.kind) + assertEquals(ColumnExpression("salary"), window.argument) + assertEquals(LiteralExpression(Value.IntegerValue(2)), window.offset) + assertEquals(LiteralExpression(Value.IntegerValue(0)), window.defaultValue) + assertFalse(window.star) + } + + @Test + fun leadWithASingleArgumentLeavesOffsetAndDefaultNull() { + val statement = parseOne("SELECT LEAD(salary) OVER (ORDER BY id) FROM users") as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(WindowFunctionKind.Lead, window.kind) + assertEquals(ColumnExpression("salary"), window.argument) + assertNull(window.offset) + assertNull(window.defaultValue) + } + + @Test + fun parsesFirstValueLastValueAndNtile() { + val statement = parseOne( + "SELECT FIRST_VALUE(salary) OVER (ORDER BY id), LAST_VALUE(salary) OVER (ORDER BY id), " + + "NTILE(4) OVER (ORDER BY id) FROM users" + ) as SelectStatement + val windows = (statement.projection as ExpressionProjection).items.map { it.expression as WindowExpression } + assertEquals(WindowFunctionKind.FirstValue, windows[0].kind) + assertEquals(WindowFunctionKind.LastValue, windows[1].kind) + assertEquals(WindowFunctionKind.Ntile, windows[2].kind) + assertEquals(LiteralExpression(Value.IntegerValue(4)), windows[2].argument) + } + + @Test + fun lagWithNoArgumentsFailsToParse() { + assertFailsWith { parseOne("SELECT LAG() OVER (ORDER BY id) FROM users") } + } + + @Test + fun lagWithFourArgumentsFailsToParse() { + assertFailsWith { parseOne("SELECT LAG(salary, 1, 0, 9) OVER (ORDER BY id) FROM users") } + } + + @Test + fun firstValueWithTwoArgumentsFailsToParse() { + assertFailsWith { parseOne("SELECT FIRST_VALUE(salary, id) OVER (ORDER BY id) FROM users") } + } + + @Test + fun ntileWithNoArgumentsFailsToParse() { + assertFailsWith { parseOne("SELECT NTILE() OVER (ORDER BY id) FROM users") } + } + + @Test + fun parsesExplicitRowsBetweenFrameWithLiteralOffsets() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND 2 FOLLOWING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameBoundType.Preceding, frame.start.type) + assertEquals(LiteralExpression(Value.IntegerValue(1)), frame.start.offset) + assertEquals(FrameBoundType.Following, frame.end.type) + assertEquals(LiteralExpression(Value.IntegerValue(2)), frame.end.offset) + } + + @Test + fun parsesUnboundedAndCurrentRowFrameBounds() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameBoundType.UnboundedPreceding, frame.start.type) + assertNull(frame.start.offset) + assertEquals(FrameBoundType.CurrentRow, frame.end.type) + assertNull(frame.end.offset) + } + + @Test + fun rowsShorthandWithoutBetweenEndsAtCurrentRow() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS 2 PRECEDING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameBoundType.Preceding, frame.start.type) + assertEquals(LiteralExpression(Value.IntegerValue(2)), frame.start.offset) + assertEquals(FrameBoundType.CurrentRow, frame.end.type) + } + + @Test + fun rowsShorthandWithUnboundedFollowingFailsBecauseItWouldStartAfterCurrentRow() { + assertFailsWith { + parseOne("SELECT SUM(salary) OVER (ORDER BY id ROWS UNBOUNDED FOLLOWING) FROM users") + } + } + + @Test + fun frameStartAfterFrameEndFailsToParse() { + assertFailsWith { + parseOne("SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN CURRENT ROW AND 1 PRECEDING) FROM users") + } + } + + @Test + fun frameWithoutBetweenOrBoundFailsToParse() { + assertFailsWith { + parseOne("SELECT SUM(salary) OVER (ORDER BY id ROWS) FROM users") + } + } + + @Test + fun windowWithNoFrameLeavesFrameNull() { + val statement = parseOne("SELECT SUM(salary) OVER (ORDER BY id) FROM users") as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertNull(window.frame) + } + + @Test + fun parsesExplicitRangeBetweenFrameWithLiteralOffsets() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount RANGE BETWEEN 5 PRECEDING AND 10 FOLLOWING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameUnit.Range, frame.unit) + assertEquals(FrameBoundType.Preceding, frame.start.type) + assertEquals(LiteralExpression(Value.IntegerValue(5)), frame.start.offset) + assertEquals(FrameBoundType.Following, frame.end.type) + assertEquals(LiteralExpression(Value.IntegerValue(10)), frame.end.offset) + } + + @Test + fun rangeShorthandWithoutBetweenEndsAtCurrentRowAndDefaultsRowsUnitOtherwise() { + val rangeStatement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount RANGE 3 PRECEDING) FROM users" + ) as SelectStatement + val rangeWindow = (rangeStatement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(FrameUnit.Range, rangeWindow.frame!!.unit) + assertEquals(FrameBoundType.CurrentRow, rangeWindow.frame!!.end.type) + + val rowsStatement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount ROWS 3 PRECEDING) FROM users" + ) as SelectStatement + val rowsWindow = (rowsStatement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(FrameUnit.Rows, rowsWindow.frame!!.unit) + } + + @Test + fun parsesExplicitGroupsBetweenFrameWithLiteralOffsets() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 2 FOLLOWING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameUnit.Groups, frame.unit) + assertEquals(FrameBoundType.Preceding, frame.start.type) + assertEquals(LiteralExpression(Value.IntegerValue(1)), frame.start.offset) + assertEquals(FrameBoundType.Following, frame.end.type) + assertEquals(LiteralExpression(Value.IntegerValue(2)), frame.end.offset) + } + + @Test + fun groupsShorthandWithoutBetweenEndsAtCurrentRow() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount GROUPS 3 PRECEDING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(FrameUnit.Groups, window.frame!!.unit) + assertEquals(FrameBoundType.CurrentRow, window.frame!!.end.type) + } + + @Test + fun groupsFrameStartAfterFrameEndFailsToParse() { + assertFailsWith { + parseOne("SELECT SUM(salary) OVER (ORDER BY amount GROUPS BETWEEN CURRENT ROW AND 1 PRECEDING) FROM users") + } + } + + @Test + fun parsesRangeBetweenUnboundedAndCurrentRow() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + val frame = window.frame!! + assertEquals(FrameUnit.Range, frame.unit) + assertEquals(FrameBoundType.UnboundedPreceding, frame.start.type) + assertEquals(FrameBoundType.CurrentRow, frame.end.type) + } + + @Test + fun rangeFrameStartAfterFrameEndFailsToParse() { + assertFailsWith { + parseOne("SELECT SUM(salary) OVER (ORDER BY amount RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) FROM users") + } + } + + @Test + fun frameWithNoExcludeClauseDefaultsToNoOthers() { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + assertEquals(FrameExclude.NoOthers, window.frame!!.exclude) + } + + @Test + fun parsesEveryExcludeMode() { + fun excludeOf(clause: String): FrameExclude { + val statement = parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING $clause) FROM users" + ) as SelectStatement + val window = (statement.projection as ExpressionProjection).items.single().expression as WindowExpression + return window.frame!!.exclude + } + assertEquals(FrameExclude.CurrentRow, excludeOf("EXCLUDE CURRENT ROW")) + assertEquals(FrameExclude.Group, excludeOf("EXCLUDE GROUP")) + assertEquals(FrameExclude.Ties, excludeOf("EXCLUDE TIES")) + assertEquals(FrameExclude.NoOthers, excludeOf("EXCLUDE NO OTHERS")) + } + + @Test + fun excludeWithoutAValidModeFailsToParse() { + assertFailsWith { + parseOne( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE FOO) FROM users" + ) + } + } + + @Test + fun parsesUnionAsASetOperationStatement() { + val statement = parseOne("SELECT id FROM a UNION SELECT id FROM b") as SetOperationStatement + assertEquals(SetOperator.Union, statement.operator) + assertEquals("a", tableName(statement.left)) + assertEquals("b", tableName(statement.right)) + } + + @Test + fun parsesUnionAllIntersectAndExcept() { + assertEquals(SetOperator.UnionAll, (parseOne("SELECT id FROM a UNION ALL SELECT id FROM b") as SetOperationStatement).operator) + assertEquals(SetOperator.Intersect, (parseOne("SELECT id FROM a INTERSECT SELECT id FROM b") as SetOperationStatement).operator) + assertEquals(SetOperator.Except, (parseOne("SELECT id FROM a EXCEPT SELECT id FROM b") as SetOperationStatement).operator) + } + + @Test + fun parsesIntersectAllAndExceptAll() { + assertEquals(SetOperator.IntersectAll, (parseOne("SELECT id FROM a INTERSECT ALL SELECT id FROM b") as SetOperationStatement).operator) + assertEquals(SetOperator.ExceptAll, (parseOne("SELECT id FROM a EXCEPT ALL SELECT id FROM b") as SetOperationStatement).operator) + } + + @Test + fun parsesCreateIndexAndCreateUniqueIndex() { + val plain = parseOne("CREATE INDEX idx ON t (a)") as CreateIndexStatement + assertEquals(false, plain.unique) + assertEquals("idx", plain.indexName) + assertEquals("t", plain.tableName) + assertEquals(listOf("a"), plain.columnNames) + val unique = parseOne("CREATE UNIQUE INDEX idx ON t (a)") as CreateIndexStatement + assertEquals(true, unique.unique) + } + + @Test + fun chainedCompoundOperatorsAreLeftAssociative() { + val statement = parseOne("SELECT id FROM a UNION SELECT id FROM b INTERSECT SELECT id FROM c") as SetOperationStatement + assertEquals(SetOperator.Intersect, statement.operator) + val left = statement.left as SetOperationStatement + assertEquals(SetOperator.Union, left.operator) + assertEquals("a", tableName(left.left)) + assertEquals("b", tableName(left.right)) + assertEquals("c", tableName(statement.right)) + } + + @Test + fun trailingOrderByAndLimitAttachToTheWholeCompoundQuery() { + val statement = parseOne("SELECT id FROM a UNION SELECT id FROM b ORDER BY id DESC LIMIT 3") as SetOperationStatement + val key = statement.orderBy!!.keys.single() + assertEquals("id", (key.expression as ColumnExpression).name) + assertEquals(true, key.descending) + assertEquals(3, statement.limit) + } + + @Test + fun individualSelectCoresInACompoundQueryDoNotOwnOrderByOrLimit() { + val statement = parseOne("SELECT id FROM a UNION SELECT id FROM b") as SetOperationStatement + assertNull((statement.left as SelectStatement).orderBy) + assertNull((statement.left as SelectStatement).limit) + } + + @Test + fun parsesWithClauseAsAWithQuery() { + val statement = parseOne("WITH recent AS (SELECT id FROM a) SELECT id FROM recent") as WithQuery + assertEquals(false, statement.recursive) + val cte = statement.ctes.single() + assertEquals("recent", cte.name) + assertNull(cte.columns) + assertEquals("a", tableName(cte.query)) + assertEquals("recent", tableName(statement.body)) + } + + @Test + fun parsesWithRecursiveAndMultipleCtesWithColumnLists() { + val statement = parseOne( + "WITH RECURSIVE first_cte(x) AS (SELECT id FROM a), second_cte AS (SELECT id FROM b) SELECT x FROM first_cte" + ) as WithQuery + assertEquals(true, statement.recursive) + assertEquals(2, statement.ctes.size) + assertEquals(listOf("x"), statement.ctes[0].columns) + assertNull(statement.ctes[1].columns) + } + + @Test + fun explainAcceptsAUnionQuery() { + val statement = parseOne("EXPLAIN SELECT id FROM a UNION SELECT id FROM b") as ExplainStatement + assertTrue(statement.select is SetOperationStatement) + } + + @Test + fun createViewAcceptsAUnionQuery() { + val statement = parseOne("CREATE VIEW v AS SELECT id FROM a UNION SELECT id FROM b") as CreateViewStatement + assertTrue(statement.query is SetOperationStatement) + } + + @Test + fun derivedTableAcceptsACompoundQuery() { + val statement = parseOne("SELECT * FROM (SELECT id FROM a UNION SELECT id FROM b) AS u") as SelectStatement + val from = statement.from as SubquerySource + assertTrue(from.select is SetOperationStatement) + } + + @Test + fun derivedTableAcceptsAWithQuery() { + val statement = parseOne("SELECT * FROM (WITH x AS (SELECT id FROM a) SELECT id FROM x) AS u") as SelectStatement + val from = statement.from as SubquerySource + assertTrue(from.select is WithQuery) + } + + @Test + fun parsesColumnLevelForeignKeyWithDefaultActions() { + val statement = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))" + ) as CreateTableStatement + val fk = statement.foreignKeys.single() + assertEquals(listOf("dept_id"), fk.columns) + assertEquals("departments", fk.referencedTable) + assertEquals(listOf("id"), fk.referencedColumns) + assertEquals(ForeignKeyAction.NoAction, fk.onDelete) + assertEquals(ForeignKeyAction.NoAction, fk.onUpdate) + } + + @Test + fun parsesForeignKeyActionsInEitherOrder() { + val cascade = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER REFERENCES d(id) ON DELETE CASCADE ON UPDATE RESTRICT)" + ) as CreateTableStatement + val fk1 = cascade.foreignKeys.single() + assertEquals(ForeignKeyAction.Cascade, fk1.onDelete) + assertEquals(ForeignKeyAction.Restrict, fk1.onUpdate) + + val reordered = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER REFERENCES d(id) ON UPDATE SET NULL ON DELETE NO ACTION)" + ) as CreateTableStatement + val fk2 = reordered.foreignKeys.single() + assertEquals(ForeignKeyAction.NoAction, fk2.onDelete) + assertEquals(ForeignKeyAction.SetNull, fk2.onUpdate) + } + + @Test + fun parsesTableLevelForeignKeyConstraint() { + val statement = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, dept_id INTEGER, FOREIGN KEY (dept_id) REFERENCES departments(id))" + ) as CreateTableStatement + assertEquals(2, statement.columns.size) + val fk = statement.foreignKeys.single() + assertEquals(listOf("dept_id"), fk.columns) + assertEquals("departments", fk.referencedTable) + } + + @Test + fun parsesColumnLevelAndTableLevelCheckConstraints() { + val statement = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0), CHECK (id <> 0))" + ) as CreateTableStatement + assertEquals(2, statement.checks.size) + assertTrue(statement.checks[0].expression is BinaryExpression) + assertTrue(statement.checks[1].expression is BinaryExpression) + } + + @Test + fun parsesColumnLevelUniqueConstraint() { + val statement = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)" + ) as CreateTableStatement + val unique = statement.uniqueConstraints.single() + assertEquals(listOf("email"), unique.columns) + } + + @Test + fun parsesTableLevelCompositeUniqueConstraint() { + val statement = parseOne( + "CREATE TABLE t (a INTEGER, b INTEGER, c INTEGER, UNIQUE (a, b, c))" + ) as CreateTableStatement + val unique = statement.uniqueConstraints.single() + assertEquals(listOf("a", "b", "c"), unique.columns) + } + + @Test + fun parsesMultipleUniqueConstraintsMixingColumnAndTableLevel() { + val statement = parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE, a INTEGER, b INTEGER, UNIQUE (a, b))" + ) as CreateTableStatement + assertEquals(2, statement.uniqueConstraints.size) + assertEquals(listOf("email"), statement.uniqueConstraints[0].columns) + assertEquals(listOf("a", "b"), statement.uniqueConstraints[1].columns) + } + + @Test + fun parsesAlterTableAddColumnWithNotNullAndDefault() { + val statement = parseOne("ALTER TABLE t ADD COLUMN active BOOLEAN NOT NULL DEFAULT TRUE") as AlterTableStatement + assertEquals("t", statement.tableName) + val action = statement.action as AddColumn + assertEquals("active", action.column.name) + assertEquals(ColumnType.Boolean, action.column.type) + assertTrue(action.column.notNull) + assertEquals(LiteralExpression(Value.BooleanValue(true)), action.column.default) + } + + @Test + fun parsesAlterTableAddColumnWithoutDefault() { + val statement = parseOne("ALTER TABLE t ADD COLUMN note TEXT") as AlterTableStatement + val action = statement.action as AddColumn + assertEquals("note", action.column.name) + assertFalse(action.column.notNull) + assertNull(action.column.default) + } + + @Test + fun parsesAlterTableDropColumnWithAndWithoutCascade() { + val plain = (parseOne("ALTER TABLE t DROP COLUMN note") as AlterTableStatement).action as DropColumn + assertEquals("note", plain.columnName) + assertEquals(DropMode.Restrict, plain.mode) + val cascaded = (parseOne("ALTER TABLE t DROP COLUMN note CASCADE") as AlterTableStatement).action as DropColumn + assertEquals(DropMode.Cascade, cascaded.mode) + } + + @Test + fun parsesAlterTableRenameColumn() { + val action = (parseOne("ALTER TABLE t RENAME COLUMN a TO b") as AlterTableStatement).action as RenameColumn + assertEquals("a", action.oldName) + assertEquals("b", action.newName) + } + + @Test + fun parsesAlterTableRenameTo() { + val statement = parseOne("ALTER TABLE t RENAME TO t2") as AlterTableStatement + assertEquals("t", statement.tableName) + val action = statement.action as RenameTable + assertEquals("t2", action.newName) + } + + @Test + fun parsesAlterTableAlterColumnTypeWithColumnKeywordAndUsing() { + val action = (parseOne("ALTER TABLE t ALTER COLUMN amount TYPE DECIMAL(10, 2) USING amount") as AlterTableStatement) + .action as ChangeColumnType + assertEquals("amount", action.columnName) + assertEquals(ColumnType.Decimal, action.newType) + assertEquals(10, action.precision) + assertEquals(2, action.scale) + assertEquals(ColumnExpression("amount"), action.using) + } + + @Test + fun parsesAlterTableAlterColumnTypeWithoutColumnKeywordOrUsing() { + val action = (parseOne("ALTER TABLE t ALTER amount TYPE REAL") as AlterTableStatement).action as ChangeColumnType + assertEquals("amount", action.columnName) + assertEquals(ColumnType.Real, action.newType) + assertNull(action.precision) + assertNull(action.using) + } + + @Test + fun alterTableAlterColumnRejectsAMissingTypeKeyword() { + assertFailsWith { parse("ALTER TABLE t ALTER COLUMN amount REAL") } + } + + @Test + fun alterTableAddColumnRejectsPrimaryKeyReferencesUniqueAndCheck() { + assertFailsWith { parse("ALTER TABLE t ADD COLUMN id INTEGER PRIMARY KEY") } + assertFailsWith { parse("ALTER TABLE t ADD COLUMN dept_id INTEGER REFERENCES departments(id)") } + assertFailsWith { parse("ALTER TABLE t ADD COLUMN email TEXT UNIQUE") } + assertFailsWith { parse("ALTER TABLE t ADD COLUMN age INTEGER CHECK (age > 0)") } + } + + @Test + fun parsesColumnLevelDefaultInCreateTable() { + val statement = parseOne("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'new')") as CreateTableStatement + assertEquals(LiteralExpression(Value.TextValue("new")), statement.columns[1].default) + } + + @Test + fun parsesGeneratedAlwaysAsStoredInCreateTable() { + val statement = parseOne("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") as CreateTableStatement + assertEquals(null, statement.columns[0].generated) + assertEquals( + BinaryExpression(BinaryOperator.Multiply, ColumnExpression("a"), LiteralExpression(Value.IntegerValue(2))), + statement.columns[1].generated + ) + } + + @Test + fun generatedColumnRequiresAlwaysKeyword() { + assertFailsWith { parse("CREATE TABLE t (a INTEGER, b INTEGER GENERATED AS (a) STORED)") } + } + + @Test + fun generatedColumnRequiresAsKeyword() { + assertFailsWith { parse("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS (a) STORED)") } + } + + @Test + fun generatedColumnRequiresStoredKeyword() { + assertFailsWith { parse("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a))") } + } + + @Test + fun generatedColumnRejectsCombiningWithDefault() { + assertFailsWith { parse("CREATE TABLE t (a INTEGER, b INTEGER DEFAULT 1 GENERATED ALWAYS AS (a) STORED)") } + } + + @Test + fun generatedColumnRejectsCombiningWithPrimaryKey() { + assertFailsWith { + parse("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a) STORED PRIMARY KEY)") + } + } + + @Test + fun generatedIsAContextualKeywordUsableAsAColumnName() { + val statement = parseOne("CREATE TABLE t (generated INTEGER, always INTEGER, stored INTEGER)") as CreateTableStatement + assertEquals(listOf("generated", "always", "stored"), statement.columns.map { it.name }) + } + + @Test + fun parsesGeneratedAlwaysAsStoredInAlterTableAddColumn() { + val statement = parseOne("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") as AlterTableStatement + val column = (statement.action as AddColumn).column + assertEquals( + BinaryExpression(BinaryOperator.Multiply, ColumnExpression("a"), LiteralExpression(Value.IntegerValue(2))), + column.generated + ) + assertEquals(null, column.default) + } + + @Test + fun alterTableAddColumnGeneratedRejectsCombiningWithDefault() { + assertFailsWith { + parse("ALTER TABLE t ADD COLUMN b INTEGER DEFAULT 1 GENERATED ALWAYS AS (a) STORED") + } + } + + @Test + fun alterTableAddColumnGeneratedStillRejectsPrimaryKey() { + assertFailsWith { + parse("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a) STORED PRIMARY KEY") + } + } + + @Test + fun parsesCreateTriggerWithTimingEventAndBody() { + val statement = parseOne( + "CREATE TRIGGER log AFTER INSERT ON orders FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) as CreateTriggerStatement + assertEquals("log", statement.triggerName) + assertEquals(TriggerTiming.After, statement.timing) + assertEquals(TriggerEvent.Insert, statement.event) + assertEquals("orders", statement.tableName) + assertEquals(null, statement.whenClause) + assertEquals(1, statement.body.size) + assertTrue(statement.body.single() is InsertStatement) + } + + @Test + fun parsesCreateTriggerWithWhenClauseAndMultipleBodyStatements() { + val statement = parseOne( + "CREATE TRIGGER guard BEFORE UPDATE ON accounts FOR EACH ROW WHEN NEW.balance < 0 " + + "BEGIN UPDATE accounts SET balance = 0 WHERE id = NEW.id; DELETE FROM overdrafts WHERE account_id = NEW.id; END" + ) as CreateTriggerStatement + assertEquals(TriggerTiming.Before, statement.timing) + assertEquals(TriggerEvent.Update, statement.event) + assertTrue(statement.whenClause is BinaryExpression) + assertEquals(2, statement.body.size) + assertTrue(statement.body[0] is UpdateStatement) + assertTrue(statement.body[1] is DeleteStatement) + } + + @Test + fun parsesDropTrigger() { + val statement = parseOne("DROP TRIGGER log") as DropTriggerStatement + assertEquals("log", statement.triggerName) + } + + @Test + fun createTriggerRejectsNonDmlBodyStatement() { + assertFailsWith { parse("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN SELECT 1; END") } + } + + @Test + fun createTriggerRequiresForEachRow() { + assertFailsWith { parse("CREATE TRIGGER log AFTER INSERT ON t BEGIN DELETE FROM t WHERE 1 = 0; END") } + } + + @Test + fun createTriggerRequiresMatchingEnd() { + assertFailsWith { parse("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0;") } + } + + @Test + fun createTriggerRequiresTimingAndEventKeywords() { + assertFailsWith { parse("CREATE TRIGGER log ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") } + assertFailsWith { parse("CREATE TRIGGER log AFTER ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") } + } + + @Test + fun parsesCreateSequenceWithNoOptions() { + val statement = parseOne("CREATE SEQUENCE seq") as CreateSequenceStatement + assertEquals("seq", statement.sequenceName) + assertEquals(null, statement.incrementBy) + assertEquals(null, statement.minValue) + assertEquals(null, statement.maxValue) + assertEquals(null, statement.startValue) + assertEquals(false, statement.cycle) + } + + @Test + fun parsesCreateSequenceWithEveryOptionInAnyOrder() { + val statement = parseOne("CREATE SEQUENCE seq CYCLE START WITH 10 MAXVALUE 100 MINVALUE 1 INCREMENT BY 2") as CreateSequenceStatement + assertEquals(2L, statement.incrementBy) + assertEquals(1L, statement.minValue) + assertEquals(100L, statement.maxValue) + assertEquals(10L, statement.startValue) + assertEquals(true, statement.cycle) + } + + @Test + fun parsesCreateSequenceWithNegativeBounds() { + val statement = parseOne("CREATE SEQUENCE seq INCREMENT BY -1 MINVALUE -100 START WITH -1") as CreateSequenceStatement + assertEquals(-1L, statement.incrementBy) + assertEquals(-100L, statement.minValue) + assertEquals(-1L, statement.startValue) + } + + @Test + fun parsesDropSequence() { + val statement = parseOne("DROP SEQUENCE seq") as DropSequenceStatement + assertEquals("seq", statement.sequenceName) + } + + @Test + fun parsesAlterSequenceRestartWithAndIncrementBy() { + val statement = parseOne("ALTER SEQUENCE seq INCREMENT BY 5 RESTART WITH 1") as AlterSequenceStatement + assertEquals("seq", statement.sequenceName) + assertEquals(5L, statement.incrementBy) + assertEquals(true, statement.restart) + assertEquals(1L, statement.restartWith) + } + + @Test + fun parsesAlterSequenceBareRestart() { + val statement = parseOne("ALTER SEQUENCE seq RESTART") as AlterSequenceStatement + assertEquals(true, statement.restart) + assertEquals(null, statement.restartWith) + } + + @Test + fun parsesAlterSequenceNoMinvalueNoMaxvalueNoCycle() { + val statement = parseOne("ALTER SEQUENCE seq NO MINVALUE NO MAXVALUE NO CYCLE") as AlterSequenceStatement + assertEquals(true, statement.clearMinValue) + assertEquals(true, statement.clearMaxValue) + assertEquals(false, statement.cycle) + } + + @Test + fun alterDispatchesToTableOrSequenceByFollowingKeyword() { + assertTrue(parseOne("ALTER TABLE t ADD COLUMN x INTEGER") is AlterTableStatement) + assertTrue(parseOne("ALTER SEQUENCE seq INCREMENT BY 1") is AlterSequenceStatement) + assertFailsWith { parse("ALTER VIEW v RENAME TO v2") } + } + + @Test + fun parsesNextvalAndCurrvalAsDedicatedExpressions() { + val select = parseOne("SELECT NEXTVAL('seq'), CURRVAL('seq') FROM t") as SelectStatement + val items = (select.projection as ExpressionProjection).items + assertEquals(NextValueExpression("seq"), items[0].expression) + assertEquals(CurrentValueExpression("seq"), items[1].expression) + } + + @Test + fun nextvalRequiresASingleStringLiteralArgument() { + assertFailsWith { parse("SELECT NEXTVAL(seq_column)") } + assertFailsWith { parse("SELECT NEXTVAL('a', 'b')") } + assertFailsWith { parse("SELECT NEXTVAL()") } + } + + @Test + fun parsesGroupByRollup() { + val statement = parseOne("SELECT dept, region, SUM(amount) FROM sales GROUP BY ROLLUP (dept, region)") as SelectStatement + val spec = statement.groupingSpec as RollupSpec + assertEquals(listOf("dept", "region"), spec.expressions.map { (it as ColumnExpression).name }) + assertEquals(listOf("dept", "region"), statement.groupBy.map { (it as ColumnExpression).name }) + } + + @Test + fun parsesGroupByCube() { + val statement = parseOne("SELECT dept, region, SUM(amount) FROM sales GROUP BY CUBE (dept, region)") as SelectStatement + val spec = statement.groupingSpec as CubeSpec + assertEquals(listOf("dept", "region"), spec.expressions.map { (it as ColumnExpression).name }) + } + + @Test + fun parsesGroupByGroupingSetsWithAnEmptySet() { + val statement = parseOne( + "SELECT dept, region, SUM(amount) FROM sales GROUP BY GROUPING SETS ((dept, region), (dept), ())" + ) as SelectStatement + val spec = statement.groupingSpec as GroupingSetsSpec + assertEquals(3, spec.sets.size) + assertEquals(listOf("dept", "region"), spec.sets[0].map { (it as ColumnExpression).name }) + assertEquals(listOf("dept"), spec.sets[1].map { (it as ColumnExpression).name }) + assertTrue(spec.sets[2].isEmpty()) + assertEquals(listOf("dept", "region"), statement.groupBy.map { (it as ColumnExpression).name }) + } + + @Test + fun rollupCubeAndGroupingSetsAreNotReservedWordsAsPlainColumnNames() { + val statement = parseOne("SELECT rollup FROM t GROUP BY rollup") as SelectStatement + assertNull(statement.groupingSpec) + assertEquals(listOf("rollup"), statement.groupBy.map { (it as ColumnExpression).name }) + } + + @Test + fun groupByRollupRequiresAClosingParenthesis() { + assertFailsWith { parse("SELECT 1 FROM t GROUP BY ROLLUP (dept") } + } + + @Test + fun groupByGroupingSetsRequiresParenthesizedElements() { + assertFailsWith { parse("SELECT 1 FROM t GROUP BY GROUPING SETS (dept)") } + } + + @Test + fun parsesGroupingFunctionCallLikeAnyOtherFunction() { + val statement = parseOne("SELECT dept, GROUPING(dept) FROM sales GROUP BY ROLLUP (dept)") as SelectStatement + val projection = statement.projection as ExpressionProjection + val call = projection.items[1].expression as CallExpression + assertEquals("GROUPING", call.name) + assertEquals(listOf("dept"), call.arguments.map { (it as ColumnExpression).name }) + } + + @Test + fun plainSelectDefaultsToNotDistinct() { + val statement = parseOne("SELECT dept FROM sales") as SelectStatement + assertFalse(statement.distinct) + assertTrue(statement.distinctOn.isEmpty()) + } + + @Test + fun selectAllParsesTheSameAsNoModifierAtAll() { + val statement = parseOne("SELECT ALL dept FROM sales") as SelectStatement + assertFalse(statement.distinct) + assertTrue(statement.distinctOn.isEmpty()) + } + + @Test + fun selectDistinctSetsTheFlagWithNoDistinctOnExpressions() { + val statement = parseOne("SELECT DISTINCT dept, region FROM sales") as SelectStatement + assertTrue(statement.distinct) + assertTrue(statement.distinctOn.isEmpty()) + val projection = statement.projection as ExpressionProjection + assertEquals(listOf("dept", "region"), projection.items.map { (it.expression as ColumnExpression).name }) + } + + @Test + fun selectDistinctOnParsesItsParenthesizedExpressionList() { + val statement = parseOne("SELECT DISTINCT ON (dept, region) dept, region, amount FROM sales") as SelectStatement + assertTrue(statement.distinct) + assertEquals(listOf("dept", "region"), statement.distinctOn.map { (it as ColumnExpression).name }) + } + + @Test + fun selectDistinctOnAcceptsAnArbitraryExpressionNotJustAColumn() { + val statement = parseOne("SELECT DISTINCT ON (amount + 1) dept FROM sales") as SelectStatement + assertTrue(statement.distinctOn.single() is BinaryExpression) + } + + @Test + fun selectDistinctOnRequiresParenthesesAfterOn() { + assertFailsWith { parse("SELECT DISTINCT ON dept FROM sales") } + } + + @Test + fun selectDistinctOnRequiresAtLeastOneExpression() { + assertFailsWith { parse("SELECT DISTINCT ON () dept FROM sales") } + } + + @Test + fun selectDistinctOnRequiresAClosingParenthesis() { + assertFailsWith { parse("SELECT DISTINCT ON (dept dept FROM sales") } + } + + @Test + fun distinctWorksInsideADerivedTableSubquery() { + val statement = parseOne("SELECT * FROM (SELECT DISTINCT dept FROM sales) AS d") as SelectStatement + val source = statement.from as SubquerySource + assertTrue((source.select as SelectStatement).distinct) + } + + @Test + fun distinctSurvivesOrderByAndLimitBeingAttachedAfterward() { + val statement = parseOne("SELECT DISTINCT dept FROM sales ORDER BY dept LIMIT 5") as SelectStatement + assertTrue(statement.distinct) + assertNotNull(statement.orderBy) + assertEquals(5, statement.limit) + } + + @Test + fun dollarDigitParsesAsAParameterExpression() { + val statement = parseOne("SELECT \$1 FROM sales") as SelectStatement + val projection = statement.projection as ExpressionProjection + assertEquals(ParameterExpression(1), projection.items.single().expression) + } + + @Test + fun multiDigitParameterIndexParses() { + val statement = parseOne("SELECT \$12 FROM sales") as SelectStatement + val projection = statement.projection as ExpressionProjection + assertEquals(ParameterExpression(12), projection.items.single().expression) + } + + @Test + fun parameterIndexZeroIsAParseError() { + assertFailsWith { parse("SELECT \$0 FROM sales") } + } + + @Test + fun prepareParsesTheNameAndWrapsTheTargetStatement() { + val statement = parseOne("PREPARE byId AS SELECT * FROM sales WHERE id = \$1") as PrepareStatement + assertEquals("byId", statement.name) + val target = statement.target as SelectStatement + val where = target.where as BinaryExpression + assertEquals(BinaryOperator.Equal, where.operator) + assertEquals(ParameterExpression(1), where.right) + } + + @Test + fun prepareAcceptsInsertUpdateDeleteAndMergeAsItsTarget() { + assertTrue((parseOne("PREPARE p AS INSERT INTO sales VALUES (\$1)") as PrepareStatement).target is InsertStatement) + assertTrue((parseOne("PREPARE p AS UPDATE sales SET dept = \$1") as PrepareStatement).target is UpdateStatement) + assertTrue((parseOne("PREPARE p AS DELETE FROM sales WHERE dept = \$1") as PrepareStatement).target is DeleteStatement) + assertTrue( + (parseOne("PREPARE p AS MERGE INTO sales USING sales2 ON sales.id = sales2.id WHEN MATCHED THEN DELETE") as PrepareStatement) + .target is MergeStatement + ) + } + + @Test + fun prepareRejectsDdlAndTransactionControlTargets() { + assertFailsWith { parse("PREPARE p AS CREATE TABLE t (id INTEGER)") } + assertFailsWith { parse("PREPARE p AS BEGIN") } + assertFailsWith { parse("PREPARE p AS EXPLAIN SELECT 1") } + } + + @Test + fun executeParsesNameAndPositionalArguments() { + val statement = parseOne("EXECUTE byId(1, 'ada')") as ExecuteStatement + assertEquals("byId", statement.name) + assertEquals(listOf(intLit(1), LiteralExpression(Value.TextValue("ada"))), statement.arguments) + } + + @Test + fun executeWithNoParenthesesHasNoArguments() { + val statement = parseOne("EXECUTE byId") as ExecuteStatement + assertEquals(emptyList(), statement.arguments) + } + + @Test + fun executeWithEmptyParenthesesHasNoArguments() { + val statement = parseOne("EXECUTE byId()") as ExecuteStatement + assertEquals(emptyList(), statement.arguments) + } + + @Test + fun deallocateParsesAName() { + assertEquals(DeallocateStatement("byId"), parseOne("DEALLOCATE byId")) + } + + @Test + fun deallocatePrepareKeywordIsOptional() { + assertEquals(DeallocateStatement("byId"), parseOne("DEALLOCATE PREPARE byId")) + } + + @Test + fun deallocateAllHasNoName() { + assertEquals(DeallocateStatement(null, all = true), parseOne("DEALLOCATE ALL")) + } } diff --git a/src/test/kotlin/kestrel/PersistenceTest.kt b/src/test/kotlin/kestrel/PersistenceTest.kt index 7575dd7..0ebc487 100644 --- a/src/test/kotlin/kestrel/PersistenceTest.kt +++ b/src/test/kotlin/kestrel/PersistenceTest.kt @@ -1,11 +1,17 @@ package kestrel +import java.io.BufferedReader import java.io.File +import java.io.IOException +import java.io.Reader +import java.io.StringReader import java.nio.file.Files import kestrel.cli.Repl import kestrel.persistence.Persistence import kotlin.test.Test import kotlin.test.assertContains +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith import kotlin.test.assertTrue class PersistenceTest { @@ -17,6 +23,34 @@ class PersistenceTest { return output.toString() } + /** + * Delivers [text] like a normal [StringReader], but throws instead of ever signalling EOF - + * simulating a process that vanished (killed, crashed, power loss) right after its last + * statement ran, never reaching either `.exit` or a graceful end of input. [Repl.run] only + * checkpoints (see [Repl.finish]) on one of those two paths, so this is the one way to drive it + * through every statement in [text] while proving NOTHING beyond the incremental WAL write each + * statement already triggered on its own (via [kestrel.cli.Repl.recordForPersistence]) ever + * makes it to disk - exactly the scenario a TEMP table's persistence exclusion must hold up + * under, not just the far more common clean-`.exit`-then-checkpoint path every other test here + * exercises. + */ + private class CrashingReader(text: String) : Reader() { + private val delegate = StringReader(text) + override fun read(cbuf: CharArray, off: Int, len: Int): Int { + val read = delegate.read(cbuf, off, len) + if (read == -1) throw IOException("simulated crash: process died before EOF") + return read + } + override fun close() = delegate.close() + } + + private fun runScriptThatCrashesBeforeItCanCheckpoint(dir: File, script: String) { + val output = StringBuilder() + assertFailsWith { + Repl(output, output, Persistence(dir)).run(BufferedReader(CrashingReader(script))) + } + } + @Test fun dataSurvivesReopenAfterCheckpoint() { val dir = tempDir() @@ -74,6 +108,82 @@ class PersistenceTest { assertTrue(!reopened.contains("2"), "rolled back insert must not persist") } + @Test + fun aMergeSurvivesReopenWithBothItsUpdateAndItsInsertIntact() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER); + CREATE TABLE staging (id INTEGER, amount INTEGER); + INSERT INTO accounts VALUES (1, 100); + INSERT INTO staging VALUES (1, 10), (2, 5); + MERGE INTO accounts USING staging ON accounts.id = staging.id + WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount + WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id, balance FROM accounts ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "110") + assertContains(reopened, "5") + } + + @Test + fun rollbackToSavepointInsideACommittedTransactionDoesNotPersistTheUndoneInsert() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE t (id INTEGER PRIMARY KEY); + BEGIN; + INSERT INTO t VALUES (111); + SAVEPOINT s1; + INSERT INTO t VALUES (222); + ROLLBACK TO SAVEPOINT s1; + INSERT INTO t VALUES (333); + COMMIT; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM t ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "111") + assertContains(reopened, "333") + assertTrue(!reopened.contains("222"), "insert undone by ROLLBACK TO SAVEPOINT must not persist") + } + + @Test + fun insertOnConflictDoUpdateSurvivesReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER); + INSERT INTO accounts VALUES (1, 100); + INSERT INTO accounts VALUES (1, 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT balance FROM accounts WHERE id = 1;\n.exit\n") + assertContains(reopened, "150") + } + + @Test + fun walReplayReconstructsAnInsertOnConflictStatementCorrectly() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER);", + "INSERT INTO accounts VALUES (1, 100);", + "INSERT INTO accounts VALUES (1, 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance;" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run("SELECT balance FROM accounts WHERE id = 1;\n.exit\n".reader().buffered()) + assertContains(output.toString(), "150") + } + @Test fun walReplayRecoversWithoutCheckpoint() { val dir = tempDir() @@ -89,4 +199,851 @@ class PersistenceTest { assertContains(output.toString(), "1") assertContains(output.toString(), "2") } + + @Test + fun viewDefinitionSurvivesReopenAfterCheckpointAndStaysLiveOverNewData() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER); + INSERT INTO users VALUES (1, 'ada', 36), (2, 'linus', 54); + CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40; + .exit + """.trimIndent() + ) + val afterCheckpoint = runScript( + dir, + """ + SELECT name FROM adults ORDER BY id ASC; + INSERT INTO users VALUES (3, 'grace', 85); + .exit + """.trimIndent() + ) + assertContains(afterCheckpoint, "linus") + assertTrue(!afterCheckpoint.contains("ada"), "view predicate must exclude rows under the age threshold") + + val afterInsert = runScript(dir, "SELECT name FROM adults ORDER BY id ASC;\n.views\n.exit\n") + assertContains(afterInsert, "grace") + assertContains(afterInsert, "adults") + } + + @Test + fun dropViewSurvivesReopenWithoutACheckpoint() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE t (id INTEGER);", + "CREATE VIEW v AS SELECT id FROM t;", + "DROP VIEW v;" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run(".views\n.exit\n".reader().buffered()) + assertContains(output.toString(), "no views") + } + + @Test + fun updatesUsingScalarFunctionsAndPredicatesSurviveReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, bio TEXT); + INSERT INTO users VALUES (1, 'ada', 'mathematician'), (2, 'linus', NULL), (3, 'grace', 'admiral'); + UPDATE users SET name = UPPER(name) WHERE bio IS NOT NULL; + DELETE FROM users WHERE name IN ('linus') OR bio LIKE '%admiral'; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT name FROM users ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "ADA") + assertTrue(!reopened.contains("linus"), "row matching NOT IN predicate must stay deleted") + assertTrue(!reopened.contains("grace") && !reopened.contains("GRACE"), "row matching LIKE predicate must stay deleted") + } + + @Test + fun decimalAndDateColumnsSurviveReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE orders (id INTEGER PRIMARY KEY, amount DECIMAL(10, 2), placed DATE); + INSERT INTO orders VALUES (1, 19.9, CAST('2024-03-01' AS DATE)); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT amount, placed FROM orders;\n.exit\n") + assertContains(reopened, "19.90") + assertContains(reopened, "2024-03-01") + } + + @Test + fun decimalAndDateColumnsSurviveReopenWithoutACheckpointViaWalReplay() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, amount DECIMAL(10, 2), placed DATE);", + "INSERT INTO orders VALUES (1, CAST('42.5' AS DECIMAL(10, 2)), CAST('2024-03-01' AS DATE));" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run("SELECT amount, placed FROM orders;\n.exit\n".reader().buffered()) + assertContains(output.toString(), "42.50") + assertContains(output.toString(), "2024-03-01") + } + + @Test + fun foreignKeyAndCheckConstraintsSurviveReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE, age INTEGER CHECK (age > 0)); + INSERT INTO departments VALUES (1, 'eng'); + INSERT INTO employees VALUES (1, 1, 30); + .exit + """.trimIndent() + ) + val afterConstraintChecks = runScript( + dir, + """ + INSERT INTO employees VALUES (2, 99, 30); + .exit + """.trimIndent() + ) + assertContains(afterConstraintChecks, "Error") + assertContains(afterConstraintChecks, "FOREIGN KEY") + + val afterCascade = runScript( + dir, + """ + DELETE FROM departments WHERE id = 1; + SELECT * FROM employees; + .exit + """.trimIndent() + ) + assertContains(afterCascade, "0 rows") + } + + @Test + fun uniqueConstraintsSurviveReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE, a INTEGER, b INTEGER, UNIQUE (a, b)); + INSERT INTO users VALUES (1, 'ada@example.com', 1, 1); + .exit + """.trimIndent() + ) + val afterReopen = runScript( + dir, + """ + INSERT INTO users VALUES (2, 'ada@example.com', 2, 2); + .exit + """.trimIndent() + ) + assertContains(afterReopen, "Error") + assertContains(afterReopen, "UNIQUE constraint") + + val afterCompositeViolation = runScript( + dir, + """ + INSERT INTO users VALUES (3, 'grace@example.com', 1, 1); + .exit + """.trimIndent() + ) + assertContains(afterCompositeViolation, "Error") + assertContains(afterCompositeViolation, "UNIQUE constraint") + } + + @Test + fun uniqueIndexSurvivesReopenAfterCheckpointAndKeepsEnforcingAndAccelerating() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT); + INSERT INTO users VALUES (1, 'ada@example.com'); + CREATE UNIQUE INDEX idx_email ON users (email); + .exit + """.trimIndent() + ) + val afterReopen = runScript( + dir, + """ + INSERT INTO users VALUES (2, 'ada@example.com'); + EXPLAIN SELECT id FROM users WHERE email = 'ada@example.com'; + .schema users + .exit + """.trimIndent() + ) + assertContains(afterReopen, "Error") + assertContains(afterReopen, "UNIQUE constraint") + assertContains(afterReopen, "Index Scan using idx_email") + assertContains(afterReopen, "CREATE UNIQUE INDEX idx_email ON users (email);") + } + + @Test + fun compositeUniqueIndexSurvivesReopenAfterCheckpointAndKeepsEnforcingAndAccelerating() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, seq INTEGER); + INSERT INTO orders VALUES (1, 'EU', 1); + CREATE UNIQUE INDEX idx_region_seq ON orders (region, seq); + .exit + """.trimIndent() + ) + val afterReopen = runScript( + dir, + """ + INSERT INTO orders VALUES (2, 'EU', 1); + INSERT INTO orders VALUES (3, 'US', 1); + EXPLAIN SELECT id FROM orders WHERE region = 'EU' AND seq = 1; + .schema orders + .exit + """.trimIndent() + ) + assertContains(afterReopen, "Error") + assertContains(afterReopen, "UNIQUE constraint") + assertContains(afterReopen, "Index Scan using idx_region_seq") + assertContains(afterReopen, "CREATE UNIQUE INDEX idx_region_seq ON orders (region, seq);") + } + + @Test + fun compositePrimaryKeyAndForeignKeySurviveReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE orders (region TEXT, order_no INTEGER, PRIMARY KEY (region, order_no)); + CREATE TABLE line_items (region TEXT, order_no INTEGER, sku TEXT, FOREIGN KEY (region, order_no) REFERENCES orders (region, order_no) ON DELETE CASCADE); + INSERT INTO orders VALUES ('eu', 1); + INSERT INTO line_items VALUES ('eu', 1, 'SKU1'); + .exit + """.trimIndent() + ) + val duplicateRejected = runScript(dir, "INSERT INTO orders VALUES ('eu', 1);\n.exit\n") + assertContains(duplicateRejected, "Error") + assertContains(duplicateRejected, "UNIQUE constraint") + + val afterCascade = runScript( + dir, + """ + DELETE FROM orders WHERE region = 'eu'; + SELECT * FROM line_items; + .exit + """.trimIndent() + ) + assertTrue(!afterCascade.contains("SKU1"), "cascade delete across a composite FK must survive reopen") + } + + @Test + fun alterTableAddColumnSurvivesReopenAfterCheckpointAndBackfillsAndDefaultsFutureInserts() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO users VALUES (1, 'ada'), (2, 'linus'); + ALTER TABLE users ADD COLUMN status TEXT NOT NULL DEFAULT 'active'; + INSERT INTO users (id, name) VALUES (3, 'grace'); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT name, status FROM users ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "ada") + assertContains(reopened, "active") + val statusCount = Regex("active").findAll(reopened).count() + assertEquals(3, statusCount, "all three rows (backfilled and freshly inserted) must carry the DEFAULT") + } + + @Test + fun generatedColumnSurvivesReopenAfterCheckpointAndKeepsComputingFutureInserts() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED); + INSERT INTO t (a) VALUES (5); + .exit + """.trimIndent() + ) + runScript(dir, "INSERT INTO t (a) VALUES (7);\n.exit\n") + val reopened = runScript(dir, "SELECT b FROM t ORDER BY a ASC;\n.exit\n") + assertContains(reopened, "10") + assertContains(reopened, "14") + } + + @Test + fun generatedColumnSurvivesReopenWithoutACheckpointViaWalReplay() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED);", + "INSERT INTO t (a) VALUES (5);" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run("SELECT b FROM t WHERE a = 5;\n.exit\n".reader().buffered()) + assertContains(output.toString(), "10") + } + + @Test + fun alterTableRenameAndDropColumnSurviveReopenWithoutACheckpointViaWalReplay() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE t (id INTEGER PRIMARY KEY, bio TEXT, scratch TEXT);", + "INSERT INTO t VALUES (1, 'mathematician', 'temp');", + "ALTER TABLE t RENAME COLUMN bio TO biography;", + "ALTER TABLE t DROP COLUMN scratch;" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run("SELECT biography FROM t WHERE id = 1;\n.exit\n".reader().buffered()) + assertContains(output.toString(), "mathematician") + } + + @Test + fun alterTableRenameToSurvivesReopenWithoutACheckpointViaWalReplay() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT);", + "INSERT INTO t VALUES (1, 'ada');", + "ALTER TABLE t RENAME TO people;" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run("SELECT name FROM people WHERE id = 1;\n.tables\n.exit\n".reader().buffered()) + val text = output.toString() + assertContains(text, "ada") + val tableLines = text.lines().map { it.trim() } + assertTrue(tableLines.contains("people"), "renamed table must appear in .tables") + assertTrue(tableLines.none { it.equals("t", ignoreCase = true) }, "old table name must not linger in .tables") + } + + @Test + fun alterTableRenameToPropagatesIntoADependentViewAcrossACheckpointReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO departments VALUES (1, 'eng'); + CREATE VIEW v AS SELECT id, name FROM departments; + ALTER TABLE departments RENAME TO divisions; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT * FROM v;\n.exit\n") + assertContains(reopened, "eng") + } + + @Test + fun stringAggViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE tags (item TEXT, label TEXT); + INSERT INTO tags VALUES ('widget', 'blue'), ('widget', 'small'), ('gadget', 'red'); + CREATE VIEW item_labels AS SELECT item, STRING_AGG(label, ', ' ORDER BY label) AS labels FROM tags GROUP BY item; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT item, labels FROM item_labels ORDER BY item;\n.exit\n") + assertContains(reopened, "blue, small") + assertContains(reopened, "red") + } + + /** + * `COUNT(DISTINCT ...)`'s `distinct` flag is a field on [kestrel.sql.ast.FunctionExpression] just + * like STRING_AGG's `separator`/`orderBy` above - this proves [kestrel.persistence.SqlWriter] + * actually dumps the `DISTINCT` keyword back out (not just the wrapped argument), so a view built + * on it does not silently turn into a plain, non-deduplicating `COUNT` after reopen. + */ + @Test + fun countDistinctViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE sales (dept TEXT, amount INTEGER); + INSERT INTO sales VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', 100); + CREATE VIEW dept_distinct_amounts AS SELECT dept, COUNT(DISTINCT amount) AS n FROM sales GROUP BY dept; + .exit + """.trimIndent() + ) + val reopened = runScript( + dir, + "SELECT n FROM dept_distinct_amounts WHERE dept = 'eng';\n" + + "SELECT n FROM dept_distinct_amounts WHERE dept = 'ops';\n.exit\n" + ) + assertContains(reopened, "| 2 |") + assertContains(reopened, "| 1 |") + } + + /** + * `FILTER (WHERE ...)`'s predicate is a field on [kestrel.sql.ast.FunctionExpression] just like + * STRING_AGG's `separator`/`orderBy` and COUNT's `distinct` above - this proves + * [kestrel.persistence.SqlWriter] actually dumps the `FILTER (WHERE ...)` clause back out, so a + * view built on it does not silently turn into an unfiltered aggregate after reopen. + */ + @Test + fun filteredAggregateViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE sales (dept TEXT, amount INTEGER); + INSERT INTO sales VALUES ('eng', 100), ('eng', 200), ('ops', 100), ('ops', 100); + CREATE VIEW dept_high_value_counts AS SELECT dept, COUNT(*) FILTER (WHERE amount > 100) AS n FROM sales GROUP BY dept; + .exit + """.trimIndent() + ) + val reopened = runScript( + dir, + "SELECT n FROM dept_high_value_counts WHERE dept = 'eng';\n" + + "SELECT n FROM dept_high_value_counts WHERE dept = 'ops';\n.exit\n" + ) + assertContains(reopened, "| 1 |") + assertContains(reopened, "| 0 |") + } + + /** + * [kestrel.catalog.RelationRename] rewrites a renamed table's name everywhere it could appear in a + * dependent view's stored query, including inside a nested subquery - this proves that recursion + * now also reaches a subquery sitting inside an aggregate's `FILTER (WHERE ...)` predicate + * specifically (a [kestrel.sql.ast.FunctionExpression] field distinct from `argument`, easy to miss + * when only `argument` gets rewritten), not just a subquery in WHERE/projection. + */ + @Test + fun renameTablePropagatesIntoAViewsAggregateFilterSubquery() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE items (id INTEGER PRIMARY KEY, amount INTEGER); + INSERT INTO items VALUES (1, 100), (2, 200); + CREATE TABLE config (id INTEGER PRIMARY KEY, threshold INTEGER); + INSERT INTO config VALUES (1, 150); + CREATE VIEW joined AS SELECT COUNT(*) FILTER (WHERE amount > (SELECT threshold FROM config WHERE id = 1)) AS n FROM items; + ALTER TABLE config RENAME TO settings; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT n FROM joined;\n.exit\n") + assertContains(reopened, "| 1 |") + } + + /** + * [kestrel.catalog.RelationRename] rewrites a renamed table's name everywhere it could appear in a + * dependent view's stored query, including inside a nested subquery - this proves that recursion + * now also reaches a subquery sitting inside STRING_AGG's `separator` argument specifically (a + * [kestrel.sql.ast.FunctionExpression] field distinct from `argument`, easy to miss when only + * `argument` gets rewritten), not just a subquery in WHERE/projection. + */ + @Test + fun renameTablePropagatesIntoAViewsStringAggSeparatorSubquery() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE items (id INTEGER PRIMARY KEY, label TEXT); + INSERT INTO items VALUES (1, 'b'), (2, 'a'); + CREATE TABLE config (id INTEGER PRIMARY KEY, sep TEXT); + INSERT INTO config VALUES (1, ','); + CREATE VIEW joined AS SELECT STRING_AGG(label, (SELECT sep FROM config WHERE id = 1) ORDER BY label) AS labels FROM items; + ALTER TABLE config RENAME TO settings; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT labels FROM joined;\n.exit\n") + assertContains(reopened, "a,b") + } + + @Test + fun distinctViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE sales (dept TEXT); + INSERT INTO sales VALUES ('eng'), ('eng'), ('ops'); + CREATE VIEW depts AS SELECT DISTINCT dept FROM sales; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT dept FROM depts ORDER BY dept;\n.exit\n") + assertContains(reopened, "2 rows") + assertContains(reopened, "eng") + assertContains(reopened, "ops") + } + + @Test + fun limitAndOffsetViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE sales (dept TEXT, amount INTEGER); + INSERT INTO sales VALUES ('eng', 300), ('eng', 200), ('ops', 100); + CREATE VIEW second_highest AS SELECT dept, amount FROM sales ORDER BY amount DESC LIMIT 1 OFFSET 1; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT dept, amount FROM second_highest;\n.exit\n") + assertContains(reopened, "1 row") + assertContains(reopened, "200") + assertTrue(!reopened.contains("300"), "amount=300 was skipped by OFFSET and must not survive") + } + + @Test + fun distinctOnViewSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE sales (dept TEXT, amount INTEGER); + INSERT INTO sales VALUES ('eng', 100), ('eng', 200), ('ops', 50); + CREATE VIEW top_per_dept AS SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, amount DESC; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT dept, amount FROM top_per_dept ORDER BY dept;\n.exit\n") + assertContains(reopened, "2 rows") + assertContains(reopened, "200") + assertTrue(!reopened.contains("100"), "amount=100 lost the DISTINCT ON tie-break and must not survive") + } + + /** + * Mirrors [renameTablePropagatesIntoAViewsStringAggSeparatorSubquery] for the newly added + * [kestrel.sql.ast.SelectStatement.distinctOn] field - [kestrel.catalog.RelationRename] must walk + * into it exactly like it already does for `groupBy`/`having`, or a subquery hiding inside + * `DISTINCT ON (...)` would silently keep pointing at a table name that no longer exists after + * reopen (the rewritten AST, not the original text, is what SqlWriter dumps on checkpoint). + */ + @Test + fun renameTablePropagatesIntoAViewsDistinctOnSubquery() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE items (dept TEXT, amount INTEGER); + INSERT INTO items VALUES ('eng', 100), ('eng', 200), ('ops', 50); + CREATE TABLE config (id INTEGER PRIMARY KEY, threshold INTEGER); + INSERT INTO config VALUES (1, 0); + CREATE VIEW top_per_dept AS SELECT DISTINCT ON (dept, (SELECT threshold FROM config WHERE id = 1)) dept, amount FROM items ORDER BY dept, (SELECT threshold FROM config WHERE id = 1), amount DESC; + ALTER TABLE config RENAME TO settings; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT dept, amount FROM top_per_dept ORDER BY dept;\n.exit\n") + assertContains(reopened, "2 rows") + assertContains(reopened, "200") + assertContains(reopened, "50") + } + + /** + * A trigger created before its table's data would refire on every reload (SqlWriter.dump + * intentionally writes CREATE TRIGGER only after every table's rows) - this proves reopening + * does NOT duplicate the audit row the trigger already produced during the first session, + * while a genuinely new INSERT after reopening still fires it normally. + */ + @Test + fun triggerCreationIsDumpedAfterDataSoReloadDoesNotRefireForAlreadyPersistedRows() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER); + CREATE TABLE audit (order_id INTEGER); + CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END; + INSERT INTO orders VALUES (1, 100); + .exit + """.trimIndent() + ) + val afterFirstReopen = runScript(dir, "SELECT order_id FROM audit;\n.exit\n") + assertContains(afterFirstReopen, "1 row") + val afterSecondReopen = runScript( + dir, + "INSERT INTO orders VALUES (2, 200);\nSELECT order_id FROM audit ORDER BY order_id;\n.exit\n" + ) + assertContains(afterSecondReopen, "2 rows") + } + + /** + * The dumped `CREATE SEQUENCE` must bake in the sequence's CURRENT position (via `START WITH`), + * not the one it was originally created with - otherwise a checkpoint reopen would reissue + * already-handed-out values, breaking the one guarantee a sequence exists to provide. + */ + @Test + fun sequencePositionSurvivesReopenAfterCheckpointInsteadOfRestarting() { + val dir = tempDir() + runScript( + dir, + """ + CREATE SEQUENCE seq; + CREATE TABLE t (id INTEGER); + INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "INSERT INTO t VALUES (NEXTVAL('seq'));\nSELECT id FROM t WHERE id = 3;\n.exit\n") + // A restart-from-scratch bug would reissue 1 as the third value (1, 2, 1) instead of + // continuing to 3 - this query would then find zero rows instead of one. + assertContains(reopened, "1 row") + } + + @Test + fun sequenceCreationAndNextvalDefaultSurviveReopenWithoutACheckpointViaWalReplay() { + val dir = tempDir() + val persistence = Persistence(dir) + persistence.appendWal( + listOf( + "CREATE SEQUENCE order_ids;", + "CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('order_ids'), item TEXT);", + "INSERT INTO orders (item) VALUES ('a');" + ) + ) + val output = StringBuilder() + Repl(output, output, Persistence(dir)).run( + "INSERT INTO orders (item) VALUES ('b');\nSELECT item FROM orders WHERE id = 1;\nSELECT item FROM orders WHERE id = 2;\n.exit\n" + .reader().buffered() + ) + val text = output.toString() + assertContains(text, "a") + assertContains(text, "b") + assertEquals(2, text.lines().count { it.trim() == "1 row" }) + } + + @Test + fun dropSequenceSurvivesReopenAfterCheckpoint() { + val dir = tempDir() + runScript( + dir, + """ + CREATE SEQUENCE seq; + DROP SEQUENCE seq; + .exit + """.trimIndent() + ) + val reopened = runScript(dir, ".sequences\n.exit\n") + assertContains(reopened, "no sequences") + } + + /** + * PREPARE/EXECUTE themselves are session-scoped and never written to the WAL (see + * PrepareStatement's own doc) - but an INSERT run THROUGH EXECUTE is real data and must survive + * a reopen exactly like one typed literally would. This is the whole reason + * [kestrel.cli.Repl.executeBuffer] persists EXECUTE's RESOLVED statement instead of its literal + * `EXECUTE name(...)` text: that literal text would be meaningless to replay against a fresh + * session that never re-ran the PREPARE. + */ + @Test + fun executeOfAPreparedInsertPersistsTheResolvedRowAcrossReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER); + PREPARE addUser AS INSERT INTO users VALUES ($1, $2, $3); + EXECUTE addUser(1, 'ada', 36); + EXECUTE addUser(2, 'linus', 54); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT name, age FROM users ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "ada") + assertContains(reopened, "linus") + } + + @Test + fun preparedStatementsDoNotSurviveReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + PREPARE addUser AS INSERT INTO users VALUES ($1, $2); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "EXECUTE addUser(1, 'ada');\n.exit\n") + assertContains(reopened, "Error: no such prepared statement: addUser") + } + + @Test + fun temporaryTableDoesNotSurviveReopenAfterCheckpointButARegularTableAlongsideItDoes() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE regular (id INTEGER PRIMARY KEY, note TEXT); + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, note TEXT); + INSERT INTO regular VALUES (1, 'keepsMe'); + INSERT INTO scratch VALUES (1, 'losesMe'); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT note FROM regular;\nSELECT note FROM scratch;\n.exit\n") + assertContains(reopened, "keepsMe") + assertContains(reopened, "Error: no such table: scratch") + assertTrue(!reopened.contains("losesMe"), "a TEMP table's rows must not survive reopen") + } + + @Test + fun renamingATemporaryTableKeepsItTemporarySoTheRenamedNameAlsoDoesNotSurviveReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY); + ALTER TABLE scratch RENAME TO renamedScratch; + INSERT INTO renamedScratch VALUES (1); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM renamedScratch;\n.exit\n") + assertContains(reopened, "Error: no such table: renamedScratch") + } + + @Test + fun addColumnAndDropColumnOnATemporaryTableKeepItTemporaryAcrossReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, a INTEGER); + ALTER TABLE scratch ADD COLUMN b TEXT; + ALTER TABLE scratch DROP COLUMN a; + INSERT INTO scratch VALUES (1, 'x'); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM scratch;\n.exit\n") + assertContains(reopened, "Error: no such table: scratch") + } + + @Test + fun renamingAColumnOnATemporaryTableKeepsItTemporaryAcrossReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, a INTEGER); + ALTER TABLE scratch RENAME COLUMN a TO b; + INSERT INTO scratch VALUES (1, 2); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM scratch;\n.exit\n") + assertContains(reopened, "Error: no such table: scratch") + } + + @Test + fun aTemporaryTablesForeignKeyBeingDroppedByAnUnrelatedCascadeKeepsItTemporaryAcrossReopen() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TABLE parent (id INTEGER PRIMARY KEY); + CREATE TEMP TABLE child (id INTEGER PRIMARY KEY, parentId INTEGER, FOREIGN KEY (parentId) REFERENCES parent(id)); + DROP TABLE parent CASCADE; + INSERT INTO child VALUES (1, NULL); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM child;\n.exit\n") + assertContains(reopened, "Error: no such table: child") + } + + @Test + fun anIndexOnATemporaryTableNeverPersists() { + val dir = tempDir() + runScript( + dir, + """ + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, tag TEXT); + CREATE INDEX scratchTag ON scratch (tag); + INSERT INTO scratch VALUES (1, 'x'); + .exit + """.trimIndent() + ) + val reopened = runScript(dir, "SELECT id FROM scratch;\n.indexes\n.exit\n") + assertContains(reopened, "Error: no such table: scratch") + assertTrue(!reopened.contains("scratchTag"), "an index on a TEMP table must not survive reopen either") + } + + @Test + fun droppingAnIndexOnATemporaryTableNeverReachesTheIncrementalWalEitherSideOfIt() { + val dir = tempDir() + runScriptThatCrashesBeforeItCanCheckpoint( + dir, + """ + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, tag TEXT); + CREATE INDEX scratchTag ON scratch (tag); + DROP INDEX scratchTag; + INSERT INTO scratch VALUES (1, 'x'); + """.trimIndent() + "\n" + ) + val reopened = runScript(dir, "SELECT id FROM scratch;\n.indexes\n.exit\n") + assertContains(reopened, "Error: no such table: scratch") + assertTrue(!reopened.contains("scratchTag")) + } + + @Test + fun aTemporaryTableNeverReachesTheIncrementalWriteAheadLogEvenBeforeACheckpoint() { + val dir = tempDir() + runScriptThatCrashesBeforeItCanCheckpoint( + dir, + """ + CREATE TABLE regular (id INTEGER PRIMARY KEY); + CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY); + INSERT INTO regular VALUES (111); + INSERT INTO scratch VALUES (222); + """.trimIndent() + "\n" + ) + // No checkpoint ever ran, so this reopen replays raw incremental WAL entries only - if the + // TEMP table's CREATE/INSERT had leaked into that log, replaying "INSERT INTO scratch..." + // would either resurrect it or (if its CREATE alone were skipped but not its INSERT) throw + // "no such table" loudly enough to show up here. + val reopened = runScript(dir, "SELECT id FROM regular;\nSELECT id FROM scratch;\n.exit\n") + assertContains(reopened, "111") + assertContains(reopened, "Error: no such table: scratch") + assertTrue(!reopened.contains("222"), "a TEMP table's row must never reach the WAL, crash or not") + } + + @Test + fun aTruncateNamingBothATemporaryAndARegularTableStillPersistsTheRegularTablesTruncationBeforeACheckpoint() { + val dir = tempDir() + runScriptThatCrashesBeforeItCanCheckpoint( + dir, + """ + CREATE TABLE regular (id INTEGER); + CREATE TEMP TABLE scratch (id INTEGER); + INSERT INTO regular VALUES (111), (222); + INSERT INTO scratch VALUES (333); + TRUNCATE TABLE regular, scratch; + """.trimIndent() + "\n" + ) + // If the TEMP name were not stripped out of the persisted TRUNCATE text, replaying + // "TRUNCATE TABLE regular, scratch" against a reload where scratch was never recreated would + // throw before truncating ANYTHING (Catalog.truncateTables validates every name up front) - + // silently leaving regular's rows intact instead of empty, since replay errors are swallowed + // (see Repl.printResult's `if (replaying) return`). + val reopened = runScript(dir, "SELECT id FROM regular;\n.exit\n") + assertTrue(!reopened.contains("111") && !reopened.contains("222"), "regular's TRUNCATE must survive the crash-before-checkpoint replay") + } } diff --git a/src/test/kotlin/kestrel/PreparedStatementTest.kt b/src/test/kotlin/kestrel/PreparedStatementTest.kt new file mode 100644 index 0000000..9885fdc --- /dev/null +++ b/src/test/kotlin/kestrel/PreparedStatementTest.kt @@ -0,0 +1,237 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.DeallocateResult +import kestrel.engine.DeleteResult +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.PrepareResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * End-to-end tests for `PREPARE`/`EXECUTE`/`DEALLOCATE` on the in-memory engine: `$1`, `$2`, ... + * positional parameters resolved by [kestrel.engine.ParameterSubstitution], driven through the same + * [Lexer]/[Parser]/[Executor] pipeline real SQL text goes through, not the AST directly. + */ +class PreparedStatementTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun seedAccounts() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 'ada', 100), (2, 'linus', 200), (3, 'grace', 300)") + } + + @Test + fun prepareReturnsItsNameAndExecuteRunsAParameterizedSelect() { + seedAccounts() + assertEquals(PrepareResult("byId"), run("PREPARE byId AS SELECT name, balance FROM accounts WHERE id = \$1").single()) + val result = select("EXECUTE byId(2)") + assertEquals(listOf("name", "balance"), result.columns) + assertEquals(listOf(listOf(Value.TextValue("linus"), Value.IntegerValue(200))), result.rows) + } + + @Test + fun theSamePreparedStatementCanBeExecutedRepeatedlyWithDifferentArguments() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + assertEquals(listOf(listOf(Value.TextValue("ada"))), select("EXECUTE byId(1)").rows) + assertEquals(listOf(listOf(Value.TextValue("grace"))), select("EXECUTE byId(3)").rows) + } + + @Test + fun executeSubstitutesMultipleParametersPositionally() { + seedAccounts() + run("PREPARE inRange AS SELECT name FROM accounts WHERE balance >= \$1 AND balance <= \$2 ORDER BY balance") + assertEquals( + listOf(listOf(Value.TextValue("linus")), listOf(Value.TextValue("grace"))), + select("EXECUTE inRange(150, 350)").rows + ) + } + + @Test + fun prepareCanWrapInsertAndExecuteInsertsARealRow() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("PREPARE addAccount AS INSERT INTO accounts VALUES (\$1, \$2, \$3)") + assertEquals(InsertResult(1), run("EXECUTE addAccount(1, 'ada', 100)").single()) + assertEquals( + listOf(listOf(Value.IntegerValue(1), Value.TextValue("ada"), Value.IntegerValue(100))), + select("SELECT * FROM accounts").rows + ) + assertEquals(InsertResult(1), run("EXECUTE addAccount(2, 'linus', 200)").single()) + assertEquals(2, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun prepareCanWrapUpdate() { + seedAccounts() + run("PREPARE credit AS UPDATE accounts SET balance = balance + \$1 WHERE id = \$2") + assertEquals(UpdateResult(1), run("EXECUTE credit(50, 1)").single()) + assertEquals(listOf(listOf(Value.IntegerValue(150))), select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun prepareCanWrapDelete() { + seedAccounts() + run("PREPARE removeAccount AS DELETE FROM accounts WHERE id = \$1") + assertEquals(DeleteResult(1), run("EXECUTE removeAccount(2)").single()) + assertEquals(2, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun prepareCanWrapMergeAndAParameterReachesTheMatchedUpdateAssignment() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + run("INSERT INTO staging VALUES (1, 'ada', 500)") + run( + "PREPARE topUp AS MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = staging.balance + \$1" + ) + run("EXECUTE topUp(10)") + assertEquals(listOf(listOf(Value.IntegerValue(510))), select("SELECT balance FROM accounts WHERE id = 1").rows) + // Executed a second time with a DIFFERENT argument to prove substitution recomputes fresh + // every EXECUTE rather than baking in whatever value the first call used. + run("EXECUTE topUp(20)") + assertEquals(listOf(listOf(Value.IntegerValue(520))), select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun prepareCanWrapMergeAndAParameterReachesTheNotMatchedInsertValues() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("INSERT INTO staging VALUES (1, 'ada', 500)") + run( + "PREPARE seedFromStaging AS MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED THEN INSERT VALUES (staging.id, staging.name, staging.balance + \$1)" + ) + run("EXECUTE seedFromStaging(10)") + assertEquals( + listOf(listOf(Value.IntegerValue(1), Value.TextValue("ada"), Value.IntegerValue(510))), + select("SELECT * FROM accounts").rows + ) + } + + @Test + fun prepareCanWrapMergeAndAParameterInTheOnConditionControlsWhetherARowMatches() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + run("INSERT INTO staging VALUES (1, 'ada', 500)") + // Only a WHEN MATCHED clause - a source row the ON condition rejects has no NOT MATCHED + // clause to fall into, so it is a true no-op rather than an insert attempt. + run( + "PREPARE conditionalTopUp AS MERGE INTO accounts USING staging " + + "ON accounts.id = staging.id AND staging.balance > \$1 " + + "WHEN MATCHED THEN UPDATE SET balance = staging.balance" + ) + run("EXECUTE conditionalTopUp(1000)") + assertEquals(listOf(listOf(Value.IntegerValue(100))), select("SELECT balance FROM accounts WHERE id = 1").rows) + run("EXECUTE conditionalTopUp(10)") + assertEquals(listOf(listOf(Value.IntegerValue(500))), select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun executeWithTheWrongNumberOfArgumentsFailsWithAClearMessage() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + val tooMany = assertFailsWith { run("EXECUTE byId(1, 2)") } + assertTrue(tooMany.message!!.contains("expects 1 parameter"), tooMany.message) + assertFailsWith { run("EXECUTE byId()") } + } + + @Test + fun executeOfAnUnknownNameFails() { + assertFailsWith { run("EXECUTE nope(1)") } + } + + @Test + fun preparingTheSameNameTwiceFails() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + assertFailsWith { run("PREPARE byId AS SELECT balance FROM accounts WHERE id = \$1") } + } + + @Test + fun deallocateRemovesAPreparedStatementSoExecuteNoLongerFindsIt() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + assertEquals(DeallocateResult("byId", all = false), run("DEALLOCATE byId").single()) + assertFailsWith { run("EXECUTE byId(1)") } + } + + @Test + fun deallocatingAnUnknownNameFails() { + assertFailsWith { run("DEALLOCATE nope") } + } + + @Test + fun deallocateAllRemovesEveryPreparedStatement() { + seedAccounts() + run("PREPARE p1 AS SELECT name FROM accounts WHERE id = \$1") + run("PREPARE p2 AS SELECT balance FROM accounts WHERE id = \$1") + assertEquals(DeallocateResult(null, all = true), run("DEALLOCATE ALL").single()) + assertFailsWith { run("EXECUTE p1(1)") } + assertFailsWith { run("EXECUTE p2(1)") } + } + + @Test + fun aNameCanBeReparedAfterBeingDeallocated() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + run("DEALLOCATE byId") + run("PREPARE byId AS SELECT balance FROM accounts WHERE id = \$1") + assertEquals(listOf(listOf(Value.IntegerValue(100))), select("EXECUTE byId(1)").rows) + } + + @Test + fun preparedStatementsAreNotUndoneByRollback() { + seedAccounts() + run("BEGIN") + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + run("ROLLBACK") + // Session-scoped, deliberately outside the transactional catalog snapshot - see + // PrepareStatement's own doc for why. + assertEquals(listOf(listOf(Value.TextValue("ada"))), select("EXECUTE byId(1)").rows) + } + + @Test + fun executeArgumentsMustBeConstantNotColumnReferences() { + seedAccounts() + run("PREPARE byId AS SELECT name FROM accounts WHERE id = \$1") + assertFailsWith { run("EXECUTE byId(id)") } + } + + @Test + fun aParameterInTheSelectListDoesNotNeedToAppearInGroupBy() { + seedAccounts() + run("PREPARE report AS SELECT \$1, COUNT(*) FROM accounts GROUP BY name") + val result = select("EXECUTE report('label')") + assertEquals(3, result.rows.size) + result.rows.forEach { row -> assertEquals(Value.TextValue("label"), row[0]) } + } + + @Test + fun aParameterTypedDirectlyOutsidePrepareParsesButOnlyFailsAtEvaluationTime() { + seedAccounts() + // $1 is structurally legal wherever any expression is (no PREPARE context exists at parse + // time to reject it) - GROUP BY validation lets it through exactly like a literal would, and + // it only fails once evaluation actually needs a real value for it. + val exception = assertFailsWith { run("SELECT \$1, COUNT(*) FROM accounts GROUP BY name") } + assertTrue(exception.message!!.contains("parameter reference"), exception.message) + } +} diff --git a/src/test/kotlin/kestrel/ReplDiskTest.kt b/src/test/kotlin/kestrel/ReplDiskTest.kt new file mode 100644 index 0000000..1a0d560 --- /dev/null +++ b/src/test/kotlin/kestrel/ReplDiskTest.kt @@ -0,0 +1,159 @@ +package kestrel + +import java.io.File +import java.nio.file.Files +import kestrel.cli.Repl +import kestrel.persistence.Persistence +import kestrel.store.sql.DiskCatalog +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * [Repl] wired to the real disk-backed [DiskCatalog]/[kestrel.store.sql.DiskExecutor] pair instead + * of the in-memory [kestrel.catalog.Catalog]/[kestrel.engine.Executor] - the disk engine had been + * fully built out (B+Tree/pager/WAL/2PL, DDL, DML, transactions, EXPLAIN) across many prior sessions + * but was never reachable from the actual user-facing CLI, only from tests calling it directly. This + * file is the end-to-end proof that a SQL session typed at the REPL really persists through the disk + * engine, not just that the disk engine works in isolation. + */ +class ReplDiskTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun runOnDisk(dataFile: File, walFile: File, script: String): String { + val output = StringBuilder() + val catalog = DiskCatalog.open(dataFile, walFile) + Repl(output, output, diskCatalog = catalog).run(script.reader().buffered()) + return output.toString() + } + + @Test + fun gateScenarioRendersOrderedResultThroughTheDiskBackedExecutor() { + val output = runOnDisk( + tempFile("kestrel-repl-data"), + tempFile("kestrel-repl-wal"), + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER); + INSERT INTO users VALUES (1, 'ada', 36), (2, 'linus', 54), (3, 'grace', 85); + SELECT name, age FROM users WHERE age > 40 ORDER BY age DESC; + .exit + """.trimIndent() + ) + assertContains(output, "3 rows inserted") + assertContains(output, "grace") + assertContains(output, "linus") + assertTrue(output.indexOf("grace") < output.indexOf("linus"), "grace should be listed before linus") + } + + @Test + fun metaCommandsReflectDiskCatalogStateForEveryObjectKind() { + val output = runOnDisk( + tempFile("kestrel-repl-data"), + tempFile("kestrel-repl-wal"), + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT NOT NULL); + CREATE UNIQUE INDEX idx_email ON users (email); + CREATE VIEW active_users AS SELECT id FROM users; + CREATE SEQUENCE user_ids; + CREATE TRIGGER log_insert AFTER INSERT ON users FOR EACH ROW BEGIN DELETE FROM users WHERE 1 = 0; END; + .tables + .indexes + .views + .triggers + .sequences + .exit + """.trimIndent() + ) + assertTrue(!output.contains("Error:"), output) + assertContains(output, "users") + assertContains(output, "idx_email") + assertContains(output, "active_users") + assertContains(output, "log_insert") + assertContains(output, "user_ids") + } + + @Test + fun schemaPrintsColumnsForeignKeyCheckUniqueAndSecondaryIndexFromTheDiskCatalog() { + val output = runOnDisk( + tempFile("kestrel-repl-data"), + tempFile("kestrel-repl-wal"), + """ + CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER NOT NULL CHECK (balance >= 0)); + CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + account_id INTEGER, + code TEXT, + UNIQUE (code), + FOREIGN KEY (account_id) REFERENCES accounts (id) + ); + CREATE INDEX idx_orders_code ON orders (code); + .schema orders + .exit + """.trimIndent() + ) + assertContains(output, "CREATE TABLE orders") + assertContains(output, "id INTEGER PRIMARY KEY") + assertContains(output, "FOREIGN KEY (account_id) REFERENCES accounts (id)") + assertContains(output, "UNIQUE (code)") + assertContains(output, "CREATE INDEX idx_orders_code ON orders (code);") + } + + @Test + fun dataSchemaAndIndexesSurviveClosingAndReopeningTheDiskCatalog() { + val dataFile = tempFile("kestrel-repl-data") + val walFile = tempFile("kestrel-repl-wal") + runOnDisk( + dataFile, + walFile, + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE UNIQUE INDEX idx_name ON users (name); + INSERT INTO users VALUES (1, 'ada'), (2, 'grace'); + .exit + """.trimIndent() + ) + val reopened = runOnDisk( + dataFile, + walFile, + "SELECT name FROM users ORDER BY id ASC;\n.indexes\nINSERT INTO users VALUES (3, 'ada');\n.exit\n" + ) + assertContains(reopened, "ada") + assertContains(reopened, "grace") + assertContains(reopened, "idx_name") + assertContains( + reopened, + "Error:", + message = "the UNIQUE index on name must still reject a duplicate name (new id, old name) after reopen" + ) + } + + @Test + fun explicitTransactionLeftOpenAtReplExitIsRolledBackNotSilentlyLost() { + val dataFile = tempFile("kestrel-repl-data") + val walFile = tempFile("kestrel-repl-wal") + runOnDisk( + dataFile, + walFile, + """ + CREATE TABLE t (id INTEGER PRIMARY KEY); + INSERT INTO t VALUES (1); + BEGIN; + INSERT INTO t VALUES (2); + .exit + """.trimIndent() + ) + val reopened = runOnDisk(dataFile, walFile, "SELECT id FROM t ORDER BY id ASC;\n.exit\n") + assertContains(reopened, "1") + assertTrue(!reopened.contains("2"), "the uncommitted INSERT inside BEGIN must not survive the abandoned transaction") + } + + @Test + fun replRejectsCombiningTextBasedPersistenceWithTheDiskBackedCatalog() { + val persistenceDir = Files.createTempDirectory("kestrel-repl-persist").toFile().also { it.deleteOnExit() } + val catalog = DiskCatalog.open(tempFile("kestrel-repl-data"), tempFile("kestrel-repl-wal")) + assertFailsWith { + Repl(persistence = Persistence(persistenceDir), diskCatalog = catalog) + } + } +} diff --git a/src/test/kotlin/kestrel/ReplTest.kt b/src/test/kotlin/kestrel/ReplTest.kt index c055d3d..8a6cd01 100644 --- a/src/test/kotlin/kestrel/ReplTest.kt +++ b/src/test/kotlin/kestrel/ReplTest.kt @@ -26,6 +26,22 @@ class ReplTest { assertTrue(output.indexOf("grace") < output.indexOf("linus"), "grace should be listed before linus") } + @Test + fun mergePrintsHowManyRowsWereInsertedUpdatedAndDeleted() { + val script = """ + CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER); + CREATE TABLE staging (id INTEGER, amount INTEGER); + INSERT INTO accounts VALUES (1, 100), (2, 200); + INSERT INTO staging VALUES (1, 10), (2, -999), (3, 5); + MERGE INTO accounts USING staging ON accounts.id = staging.id + WHEN MATCHED AND accounts.balance + staging.amount < 0 THEN DELETE + WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount + WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount); + """.trimIndent() + val output = runScript(script) + assertContains(output, "1 row inserted, 1 row updated, 1 row deleted") + } + @Test fun metaTablesListsCreatedTables() { val output = runScript("CREATE TABLE alpha (id INTEGER);\nCREATE TABLE beta (id INTEGER);\n.tables\n") @@ -41,6 +57,26 @@ class ReplTest { assertContains(output, "name TEXT NOT NULL") } + @Test + fun metaSchemaListsUniqueIndexesOnTheTable() { + val output = runScript( + "CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT NOT NULL);\n" + + "CREATE UNIQUE INDEX idx_email ON users (email);\n" + + ".schema users\n" + ) + assertContains(output, "CREATE UNIQUE INDEX idx_email ON users (email);") + } + + @Test + fun metaSchemaListsCompositeIndexesOnTheTable() { + val output = runScript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, seq INTEGER);\n" + + "CREATE INDEX idx_region_seq ON orders (region, seq);\n" + + ".schema orders\n" + ) + assertContains(output, "CREATE INDEX idx_region_seq ON orders (region, seq);") + } + @Test fun parseErrorIsReportedAndReplContinues() { val output = runScript("SELCT bad;\nCREATE TABLE t (a INTEGER);\n.tables\n") @@ -54,4 +90,164 @@ class ReplTest { assertContains(output, "Table seen created") assertTrue(!output.contains("unseen"), "statements after .exit must not run") } + + /** + * The REPL buffers input line by line and (before this was fixed) treated ANY line ending in + * `;` as "statement complete" - a trigger body's inner `INSERT ... ;` would have tripped that + * check and been parsed as an incomplete CREATE TRIGGER, well before the closing `END;` was + * even typed. Feeding the trigger definition across several `readLine()` calls (rather than as + * one line) is the point of this test. + */ + @Test + fun multiLineCreateTriggerIsNotExecutedPrematurelyOnAnInnerSemicolon() { + val script = """ + CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER); + CREATE TABLE audit (order_id INTEGER); + CREATE TRIGGER log_insert + AFTER INSERT ON orders + FOR EACH ROW + BEGIN + INSERT INTO audit VALUES (NEW.id); + END; + INSERT INTO orders VALUES (1, 100); + SELECT order_id FROM audit; + """.trimIndent() + val output = runScript(script) + assertTrue(!output.contains("Error:"), output) + assertContains(output, "Trigger log_insert created") + assertContains(output, "1 row") + } + + @Test + fun metaTriggersListsCreatedTriggers() { + val output = runScript( + "CREATE TABLE t (id INTEGER);\n" + + "CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END;\n" + + ".triggers\n" + ) + assertContains(output, "log") + } + + @Test + fun metaSequencesListsCreatedSequencesAndReportsEmpty() { + val empty = runScript(".sequences\n") + assertContains(empty, "no sequences") + val output = runScript("CREATE SEQUENCE seq;\n.sequences\n") + assertContains(output, "Sequence seq created") + assertContains(output, "seq") + } + + @Test + fun createAlterAndDropSequencePrintTheirOwnConfirmations() { + val output = runScript( + """ + CREATE SEQUENCE seq; + ALTER SEQUENCE seq RESTART WITH 5; + DROP SEQUENCE seq; + """.trimIndent() + ) + assertContains(output, "Sequence seq created") + assertContains(output, "Sequence seq altered") + assertContains(output, "Sequence seq dropped") + } + + @Test + fun nextvalDrivenAutoIncrementWorksEndToEndThroughTheRepl() { + val script = """ + CREATE SEQUENCE order_ids; + CREATE TABLE orders (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('order_ids'), item TEXT); + INSERT INTO orders (item) VALUES ('a'), ('b'); + SELECT id, item FROM orders ORDER BY id; + """.trimIndent() + val output = runScript(script) + assertTrue(!output.contains("Error:"), output) + assertContains(output, "2 rows inserted") + assertContains(output, "2 rows") + } + + @Test + fun insertReturningPrintsTheAffectedRowsAsATableAfterTheInsertConfirmation() { + val script = """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO users VALUES (1, 'ada') RETURNING id, name; + """.trimIndent() + val output = runScript(script) + assertContains(output, "1 row inserted") + assertTrue( + output.indexOf("1 row inserted") < output.indexOf("ada"), + "the RETURNING table should be printed after the insert confirmation" + ) + assertContains(output, "ada") + } + + @Test + fun updateAndDeleteReturningAlsoPrintTheirRowsAfterTheirOwnConfirmation() { + val script = """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO users VALUES (1, 'ada'); + UPDATE users SET name = 'grace' WHERE id = 1 RETURNING name; + DELETE FROM users WHERE id = 1 RETURNING id; + """.trimIndent() + val output = runScript(script) + assertContains(output, "1 row updated") + assertContains(output, "grace") + assertContains(output, "1 row deleted") + } + + @Test + fun mergeReturningPrintsTheAffectedRowsAsATableAfterTheMergeConfirmation() { + val script = """ + CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER); + CREATE TABLE staging (id INTEGER, amount INTEGER); + INSERT INTO accounts VALUES (1, 100); + INSERT INTO staging VALUES (1, 10), (2, 5); + MERGE INTO accounts USING staging ON accounts.id = staging.id + WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount + WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount) + RETURNING id, MERGE_ACTION() AS kind; + """.trimIndent() + val output = runScript(script) + assertContains(output, "1 row inserted, 1 row updated, 0 rows deleted") + assertTrue( + output.indexOf("1 row inserted, 1 row updated, 0 rows deleted") < output.indexOf("UPDATE"), + "the RETURNING table should be printed after the merge confirmation" + ) + assertContains(output, "UPDATE") + assertContains(output, "INSERT") + } + + @Test + fun prepareExecuteAndDeallocatePrintTheirOwnConfirmations() { + val script = """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO users VALUES (1, 'ada'); + PREPARE byId AS SELECT name FROM users WHERE id = $1; + EXECUTE byId(1); + DEALLOCATE byId; + """.trimIndent() + val output = runScript(script) + assertContains(output, "Prepared statement byId created") + assertContains(output, "ada") + assertContains(output, "Prepared statement byId deallocated") + } + + @Test + fun metaPreparedListsAndThenForgetsAPreparedStatementName() { + val before = runScript( + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + PREPARE byId AS SELECT name FROM users WHERE id = $1; + .prepared + """.trimIndent() + ) + assertContains(before, "byId") + + val after = runScript( + """ + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + .prepared + """.trimIndent() + ) + assertContains(after, "no prepared statements") + } } diff --git a/src/test/kotlin/kestrel/ReturningTest.kt b/src/test/kotlin/kestrel/ReturningTest.kt new file mode 100644 index 0000000..b45416c --- /dev/null +++ b/src/test/kotlin/kestrel/ReturningTest.kt @@ -0,0 +1,194 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.DeleteResult +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.storage.Value +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +class ReturningTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER NOT NULL)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + } + + @Test + fun insertReturningStarReportsTheFullInsertedRow() { + seed() + val result = exec("INSERT INTO accounts VALUES (2, 'linus', 200) RETURNING *") as InsertResult + assertEquals(1, result.count) + assertEquals(listOf("id", "name", "balance"), result.returning!!.columns) + assertEquals(listOf(listOf(int(2), text("linus"), int(200))), result.returning!!.rows) + } + + @Test + fun insertReturningExpressionListWithAliasProjectsAndLabels() { + seed() + val result = exec("INSERT INTO accounts VALUES (2, 'linus', 200) RETURNING id, balance * 2 AS doubled") as InsertResult + assertEquals(listOf("id", "doubled"), result.returning!!.columns) + assertEquals(listOf(listOf(int(2), int(400))), result.returning!!.rows) + } + + @Test + fun multiRowInsertReturningReportsOneRowPerInsertedRowInOrder() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT NOT NULL)") + val result = exec("INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c') RETURNING id") as InsertResult + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(3))), result.returning!!.rows) + } + + @Test + fun insertWithoutReturningLeavesReturningNull() { + seed() + val result = exec("INSERT INTO accounts VALUES (2, 'linus', 200)") as InsertResult + assertNull(result.returning) + } + + @Test + fun onConflictDoNothingContributesNoRowToReturning() { + seed() + val result = exec( + "INSERT INTO accounts VALUES (1, 'ignored', 999) ON CONFLICT (id) DO NOTHING RETURNING *" + ) as InsertResult + assertEquals(0, result.count) + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun onConflictDoUpdateReturnsThePostUpdateRowNotTheProposedOne() { + seed() + val result = exec( + "INSERT INTO accounts VALUES (1, 'ada', 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance RETURNING balance" + ) as InsertResult + assertEquals(1, result.count) + assertEquals(listOf(listOf(int(150))), result.returning!!.rows) + } + + @Test + fun onConflictDoUpdateWithFailingWhereContributesNoRowToReturning() { + seed() + val result = exec( + "INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999 WHERE excluded.balance < 0 RETURNING *" + ) as InsertResult + assertEquals(0, result.count) + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun mixedConflictAndFreshRowsInOneStatementReturnBothKinds() { + seed() + val result = exec( + "INSERT INTO accounts VALUES (1, 'ada', 5), (2, 'linus', 200) " + + "ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance RETURNING id, balance" + ) as InsertResult + assertEquals( + listOf(listOf(int(1), int(105)), listOf(int(2), int(200))), + result.returning!!.rows + ) + } + + @Test + fun updateReturningReportsThePostUpdateValuesForEveryMatchedRow() { + seed() + run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = exec("UPDATE accounts SET balance = balance + 10 RETURNING id, balance") as UpdateResult + assertEquals(2, result.count) + assertEquals(listOf(listOf(int(1), int(110)), listOf(int(2), int(210))), result.returning!!.rows) + } + + @Test + fun updateReturningWithWhereOnlyReportsMatchedRows() { + seed() + run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = exec("UPDATE accounts SET balance = 0 WHERE id = 2 RETURNING id") as UpdateResult + assertEquals(listOf(listOf(int(2))), result.returning!!.rows) + } + + @Test + fun updateReturningWithNoMatchesReturnsAnEmptyResultNotNull() { + seed() + val result = exec("UPDATE accounts SET balance = 0 WHERE id = 999 RETURNING id") as UpdateResult + assertEquals(0, result.count) + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun deleteReturningReportsThePreDeletionRowValues() { + seed() + run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = exec("DELETE FROM accounts WHERE id = 1 RETURNING *") as DeleteResult + assertEquals(1, result.count) + assertEquals(listOf(listOf(int(1), text("ada"), int(100))), result.returning!!.rows) + assertEquals(1, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun deleteReturningWithNoWhereReturnsEveryDeletedRow() { + seed() + run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = exec("DELETE FROM accounts RETURNING id") as DeleteResult + assertEquals(listOf(listOf(int(1)), listOf(int(2))), result.returning!!.rows) + } + + @Test + fun returningRejectsAnUnknownColumnEvenWhenZeroRowsMatch() { + seed() + assertFailsWith { + exec("UPDATE accounts SET balance = 0 WHERE id = 999 RETURNING nope") + } + } + + @Test + fun returningRejectsAnAggregateFunction() { + seed() + assertFailsWith { + exec("DELETE FROM accounts WHERE id = 1 RETURNING COUNT(*)") + } + } + + @Test + fun returningIsRejectedInsideATriggerBodyStatement() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + run("CREATE TABLE audit (order_id INTEGER)") + assertFailsWith { + run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id) RETURNING order_id; END" + ) + } + } + + @Test + fun aTableWithATriggerStillHonoursReturningOnTheOutermostStatement() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + run("CREATE TABLE audit (order_id INTEGER)") + run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + val result = exec("INSERT INTO orders VALUES (1, 100) RETURNING id, amount") as InsertResult + assertEquals(listOf(listOf(int(1), int(100))), result.returning!!.rows) + assertEquals(listOf(listOf(int(1))), select("SELECT order_id FROM audit").rows) + } +} diff --git a/src/test/kotlin/kestrel/SavepointTest.kt b/src/test/kotlin/kestrel/SavepointTest.kt new file mode 100644 index 0000000..b2ef1ad --- /dev/null +++ b/src/test/kotlin/kestrel/SavepointTest.kt @@ -0,0 +1,209 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class SavepointTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun rowCount(sql: String): Int = select(sql).rows.size + + private fun seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100), (2, 200)") + } + + @Test + fun rollbackToSavepointUndoesOnlyTheInsertsAfterIt() { + seed() + run("BEGIN") + run("INSERT INTO accounts VALUES (3, 300)") + run("SAVEPOINT beforeFour") + run("INSERT INTO accounts VALUES (4, 400)") + assertEquals(4, rowCount("SELECT * FROM accounts")) + run("ROLLBACK TO SAVEPOINT beforeFour") + assertEquals(3, rowCount("SELECT * FROM accounts")) + run("COMMIT") + assertEquals(3, rowCount("SELECT * FROM accounts")) + } + + @Test + fun rollbackToSavepointUndoesUpdatesAndDeletes() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("UPDATE accounts SET balance = 0 WHERE id = 1") + run("DELETE FROM accounts WHERE id = 2") + assertEquals(Value.IntegerValue(0), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + run("ROLLBACK TO SAVEPOINT s1") + assertEquals(Value.IntegerValue(100), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(2, rowCount("SELECT * FROM accounts")) + run("ROLLBACK") + } + + @Test + fun rollbackToSavepointUndoesCreateTable() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("CREATE TABLE scratch (a INTEGER)") + run("ROLLBACK TO SAVEPOINT s1") + assertFailsWith { run("SELECT * FROM scratch") } + run("ROLLBACK") + } + + @Test + fun theSavepointItselfStaysUsableAfterRollingBackToIt() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 300)") + run("ROLLBACK TO SAVEPOINT s1") + run("INSERT INTO accounts VALUES (4, 400)") + assertEquals(3, rowCount("SELECT * FROM accounts")) + run("ROLLBACK TO SAVEPOINT s1") + assertEquals(2, rowCount("SELECT * FROM accounts")) + run("COMMIT") + } + + @Test + fun releaseSavepointForgetsItWithoutUndoingAnything() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 300)") + run("RELEASE SAVEPOINT s1") + assertEquals(3, rowCount("SELECT * FROM accounts")) + assertFailsWith { run("ROLLBACK TO SAVEPOINT s1") } + run("COMMIT") + assertEquals(3, rowCount("SELECT * FROM accounts")) + } + + @Test + fun releaseSavepointAlsoForgetsLaterNestedSavepoints() { + seed() + run("BEGIN") + run("SAVEPOINT spOuter") + run("INSERT INTO accounts VALUES (3, 300)") + run("SAVEPOINT spInner") + run("INSERT INTO accounts VALUES (4, 400)") + run("RELEASE SAVEPOINT spOuter") + assertFailsWith { run("ROLLBACK TO SAVEPOINT spInner") } + assertEquals(4, rowCount("SELECT * FROM accounts")) + run("COMMIT") + } + + @Test + fun reusingASavepointNameShadowsTheOlderOneUntilItIsConsumed() { + seed() + run("BEGIN") + run("SAVEPOINT s") + run("INSERT INTO accounts VALUES (3, 300)") + run("SAVEPOINT s") + run("INSERT INTO accounts VALUES (4, 400)") + run("ROLLBACK TO SAVEPOINT s") + assertEquals(3, rowCount("SELECT * FROM accounts")) + run("ROLLBACK TO SAVEPOINT s") + assertEquals(3, rowCount("SELECT * FROM accounts")) + run("COMMIT") + } + + @Test + fun rollbackRestoresIndexStateEvenAcrossASavepoint() { + seed() + run("CREATE INDEX idxBalance ON accounts (balance)") + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 100)") + assertEquals(2, rowCount("SELECT * FROM accounts WHERE balance = 100")) + run("ROLLBACK TO SAVEPOINT s1") + assertEquals(1, rowCount("SELECT * FROM accounts WHERE balance = 100")) + run("ROLLBACK") + } + + @Test + fun savepointOutsideATransactionIsRejected() { + seed() + assertFailsWith { run("SAVEPOINT s1") } + } + + @Test + fun releaseSavepointOutsideATransactionIsRejected() { + seed() + assertFailsWith { run("RELEASE SAVEPOINT s1") } + } + + @Test + fun rollbackToSavepointOutsideATransactionIsRejected() { + seed() + assertFailsWith { run("ROLLBACK TO SAVEPOINT s1") } + } + + @Test + fun rollbackToAnUnknownSavepointNameIsRejected() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + assertFailsWith { run("ROLLBACK TO SAVEPOINT nope") } + run("ROLLBACK") + } + + @Test + fun releaseAnUnknownSavepointNameIsRejected() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + assertFailsWith { run("RELEASE SAVEPOINT nope") } + run("ROLLBACK") + } + + @Test + fun fullRollbackClearsEveryOutstandingSavepoint() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 300)") + run("ROLLBACK") + run("BEGIN") + assertFailsWith { run("ROLLBACK TO SAVEPOINT s1") } + run("ROLLBACK") + } + + @Test + fun commitClearsEveryOutstandingSavepoint() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 300)") + run("COMMIT") + run("BEGIN") + assertFailsWith { run("ROLLBACK TO SAVEPOINT s1") } + run("ROLLBACK") + assertEquals(3, rowCount("SELECT * FROM accounts")) + } + + @Test + fun releaseKeywordAloneWithoutSavepointKeywordIsAccepted() { + seed() + run("BEGIN") + run("SAVEPOINT s1") + run("INSERT INTO accounts VALUES (3, 300)") + run("RELEASE s1") + assertEquals(3, rowCount("SELECT * FROM accounts")) + run("COMMIT") + } +} diff --git a/src/test/kotlin/kestrel/SequenceTest.kt b/src/test/kotlin/kestrel/SequenceTest.kt new file mode 100644 index 0000000..70797fc --- /dev/null +++ b/src/test/kotlin/kestrel/SequenceTest.kt @@ -0,0 +1,304 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class SequenceTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + + @Test + fun nextvalStartsAtOneAndIncrementsByOneByDefault() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(3))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun createSequenceOptionsAreHonored() { + run("CREATE SEQUENCE seq START WITH 100 INCREMENT BY 10") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(100)), listOf(int(110))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun descendingSequenceDefaultsToMaxvalueAndCountsDown() { + run("CREATE SEQUENCE seq INCREMENT BY -1") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(-2)), listOf(int(-1))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun exhaustedAscendingSequenceWithoutCycleThrows() { + run("CREATE SEQUENCE seq MAXVALUE 2") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertFailsWith { run("INSERT INTO t VALUES (NEXTVAL('seq'))") } + } + + @Test + fun cyclingAscendingSequenceWrapsToMinvalue() { + run("CREATE SEQUENCE seq MAXVALUE 2 CYCLE") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun dropSequenceRemovesItAndFurtherNextvalFails() { + run("CREATE SEQUENCE seq") + run("DROP SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { run("INSERT INTO t VALUES (NEXTVAL('seq'))") } + } + + @Test + fun duplicateCreateSequenceFails() { + run("CREATE SEQUENCE seq") + assertFailsWith { run("CREATE SEQUENCE seq") } + } + + @Test + fun droppingAnUnknownSequenceFails() { + assertFailsWith { run("DROP SEQUENCE nope") } + } + + @Test + fun alterSequenceRestartWithResetsPosition() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + run("ALTER SEQUENCE seq RESTART WITH 50") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(50))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun bareRestartResetsToOriginalStartValue() { + run("CREATE SEQUENCE seq START WITH 5") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + run("ALTER SEQUENCE seq RESTART") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(5)), listOf(int(5)), listOf(int(6))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun alterSequenceIncrementByWithoutRestartPreservesCurrentPosition() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + run("ALTER SEQUENCE seq INCREMENT BY 100") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(101))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun alterSequenceRestartValueOutOfNewBoundsFails() { + run("CREATE SEQUENCE seq MAXVALUE 10") + assertFailsWith { run("ALTER SEQUENCE seq RESTART WITH 20") } + } + + @Test + fun alteringAnUnknownSequenceFails() { + assertFailsWith { run("ALTER SEQUENCE nope INCREMENT BY 2") } + } + + @Test + fun currvalReturnsLastNextvalInThisSession() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER, again INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'), CURRVAL('seq'))") + assertEquals(listOf(listOf(int(1), int(1))), select("SELECT id, again FROM t").rows) + } + + @Test + fun currvalBeforeAnyNextvalInThisSessionFails() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { run("INSERT INTO t VALUES (CURRVAL('seq'))") } + } + + @Test + fun alterSequenceInvalidatesRememberedCurrvalForThatSession() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + run("ALTER SEQUENCE seq INCREMENT BY 1") + assertFailsWith { run("INSERT INTO t VALUES (CURRVAL('seq'))") } + } + + @Test + fun rollbackDoesNotUndoANextvalAdvance() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("BEGIN") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + run("ROLLBACK") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + // The rolled-back row never landed, but NEXTVAL's advance to 1 survives - the surviving + // insert gets 2, not a reused 1. + assertEquals(listOf(listOf(int(2))), select("SELECT id FROM t").rows) + } + + @Test + fun rollbackUndoesCreateSequence() { + run("BEGIN") + run("CREATE SEQUENCE seq") + run("ROLLBACK") + assertFailsWith { run("DROP SEQUENCE seq") } + } + + @Test + fun rollbackUndoesDropSequence() { + run("CREATE SEQUENCE seq") + run("BEGIN") + run("DROP SEQUENCE seq") + run("ROLLBACK") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun rollbackUndoesAlterSequenceButNotTheNextvalAdvanceMadeBeforeIt() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + run("BEGIN") + run("ALTER SEQUENCE seq INCREMENT BY 100") + run("ROLLBACK") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun nextvalAsColumnDefaultActsLikeAnAutoIncrement() { + run("CREATE SEQUENCE order_ids") + run("CREATE TABLE orders (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('order_ids'), item TEXT)") + run("INSERT INTO orders (item) VALUES ('a'), ('b'), ('c')") + assertEquals( + listOf(listOf(int(1), Value.TextValue("a")), listOf(int(2), Value.TextValue("b")), listOf(int(3), Value.TextValue("c"))), + select("SELECT id, item FROM orders ORDER BY id").rows + ) + } + + @Test + fun createTableWithNextvalDefaultRequiresTheSequenceToAlreadyExist() { + assertFailsWith { + run("CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('nope'))") + } + } + + @Test + fun createTableDoesNotConsumeASequenceValueJustToValidateTheDefault() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('seq'))") + run("INSERT INTO orders (id) VALUES (DEFAULT)") + // If CREATE TABLE's own validation had already called NEXTVAL once, this would be 2. + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM orders").rows) + } + + @Test + fun nextvalDefaultOnANonIntegerColumnFails() { + run("CREATE SEQUENCE seq") + assertFailsWith { + run("CREATE TABLE orders (id TEXT DEFAULT NEXTVAL('seq'))") + } + } + + @Test + fun currvalAsColumnDefaultIsRejected() { + run("CREATE SEQUENCE seq") + assertFailsWith { + run("CREATE TABLE orders (id INTEGER DEFAULT CURRVAL('seq'))") + } + } + + @Test + fun addColumnWithNextvalDefaultIsRejected() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("ALTER TABLE t ADD COLUMN seq_id INTEGER DEFAULT NEXTVAL('seq')") + } + } + + @Test + fun nextvalIsUsableInUpdateSet() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NULL)") + run("UPDATE t SET id = NEXTVAL('seq')") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun nextvalInAWhereClauseIsRejected() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (1)") + assertFailsWith { run("UPDATE t SET id = 2 WHERE id = NEXTVAL('seq')") } + } + + @Test + fun nextvalInACheckConstraintIsRejectedAtCreateTableTime() { + run("CREATE SEQUENCE seq") + assertFailsWith { + run("CREATE TABLE t (id INTEGER CHECK (id = NEXTVAL('seq')))") + } + } + + @Test + fun nextvalInATriggerWhenClauseIsRejected() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER trg BEFORE INSERT ON t FOR EACH ROW WHEN (NEXTVAL('seq') = 1) BEGIN DELETE FROM t; END") + } + } + + @Test + fun createSequenceMinvalueExceedingMaxvalueFails() { + assertFailsWith { run("CREATE SEQUENCE seq MINVALUE 10 MAXVALUE 5") } + } + + @Test + fun createSequenceIncrementByZeroFails() { + assertFailsWith { run("CREATE SEQUENCE seq INCREMENT BY 0") } + } + + @Test + fun createSequenceStartOutsideBoundsFails() { + assertFailsWith { run("CREATE SEQUENCE seq MINVALUE 10 MAXVALUE 20 START WITH 5") } + } + + @Test + fun noMinvalueAndNoMaxvalueOnAlterRecomputeTheDefaultBound() { + run("CREATE SEQUENCE seq MINVALUE 1 MAXVALUE 3") + run("ALTER SEQUENCE seq NO MAXVALUE") + run("CREATE TABLE t (id INTEGER)") + // Was capped at 3 before the ALTER; NO MAXVALUE lifts it back to the ascending default. + run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(4, select("SELECT id FROM t").rows.size) + } +} diff --git a/src/test/kotlin/kestrel/SqlWriterTest.kt b/src/test/kotlin/kestrel/SqlWriterTest.kt new file mode 100644 index 0000000..6fa73cc --- /dev/null +++ b/src/test/kotlin/kestrel/SqlWriterTest.kt @@ -0,0 +1,1089 @@ +package kestrel + +import java.math.BigDecimal +import java.time.LocalDate +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.persistence.SqlWriter +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class SqlWriterTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun parseOne(sql: String): Statement = Parser(Lexer(sql).tokenize()).parseProgram().single() + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun text(value: String) = Value.TextValue(value) + + private fun seed() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, category TEXT, bio TEXT)") + run( + "INSERT INTO t VALUES (1, 'a', 'mathematician'), (2, 'b', NULL), (3, 'c', 'rear admiral')" + ) + } + + private fun writeThenReplay(sql: String) { + val statement = parseOne(sql) + val rendered = SqlWriter.write(statement) + run(rendered) + } + + @Test + fun updateWithCaseExpressionRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("UPDATE t SET category = CASE WHEN id = 1 THEN 'first' ELSE category END WHERE id = 1") + assertEquals(listOf(listOf(text("first"))), select("SELECT category FROM t WHERE id = 1").rows) + } + + @Test + fun deleteWithInAndLikeRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("DELETE FROM t WHERE category IN ('b') OR bio LIKE '%admiral'") + val remaining = select("SELECT category FROM t ORDER BY id ASC") + assertEquals(listOf(listOf(text("a"))), remaining.rows) + } + + @Test + fun updateWithIsNullAndBetweenRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("UPDATE t SET category = UPPER(category) WHERE bio IS NULL OR id BETWEEN 3 AND 3") + val result = select("SELECT category FROM t ORDER BY id ASC") + assertEquals(listOf(listOf(text("a")), listOf(text("B")), listOf(text("C"))), result.rows) + } + + @Test + fun truncateRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("TRUNCATE TABLE t") + assertEquals(emptyList(), select("SELECT * FROM t").rows) + } + + @Test + fun truncateRestartIdentityCascadeRoundTripsThroughSqlWriterAgainstAFreshReferencingTable() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE parent (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'))") + run("CREATE TABLE child (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parent(id))") + run("INSERT INTO parent (id) VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + run("INSERT INTO child VALUES (1, 1)") + writeThenReplay("TRUNCATE TABLE parent RESTART IDENTITY CASCADE") + assertEquals(emptyList(), select("SELECT * FROM child").rows) + run("INSERT INTO parent (id) VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(Value.IntegerValue(1))), select("SELECT id FROM parent").rows) + } + + @Test + fun mergeUpdateAndInsertRoundTripThroughSqlWriter() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER, amount INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 10), (2, 5)") + writeThenReplay( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) + assertEquals( + listOf(listOf(Value.IntegerValue(1), Value.IntegerValue(110)), listOf(Value.IntegerValue(2), Value.IntegerValue(5))), + select("SELECT id, balance FROM accounts ORDER BY id").rows + ) + } + + @Test + fun mergeWithAliasesAndADerivedTableSourceRoundTripsThroughSqlWriter() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER, amount INTEGER)") + run("INSERT INTO staging VALUES (1, 10), (1, 20)") + writeThenReplay( + "MERGE INTO accounts AS acc USING (SELECT id, SUM(amount) AS total FROM staging GROUP BY id) AS agg " + + "ON acc.id = agg.id " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (agg.id, agg.total)" + ) + assertEquals(listOf(listOf(Value.IntegerValue(1), Value.IntegerValue(30))), select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun mergeCanPersistAndDeclaresItselfPersistable() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE s (id INTEGER)") + val statement = parseOne("MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE") + assertTrue(SqlWriter.canPersist(statement)) + assertTrue(SqlWriter.write(statement).startsWith("MERGE INTO t USING s ON")) + } + + @Test + fun renderedSqlIsExecutableAsPersistedTextVerbatim() { + seed() + val statement = parseOne("UPDATE t SET category = CONCAT(category, category) WHERE id NOT IN (2)") + val rendered = SqlWriter.write(statement) + val reparsed = Parser(Lexer(rendered).tokenize()).parseProgram().single() + executor.execute(reparsed) + val result = select("SELECT category FROM t ORDER BY id ASC") + assertEquals(listOf(listOf(text("aa")), listOf(text("b")), listOf(text("cc"))), result.rows) + } + + @Test + fun createViewWithMultiColumnOrderByRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("CREATE VIEW ordered AS SELECT id, category FROM t ORDER BY category ASC, id DESC") + val result = select("SELECT category FROM ordered") + assertEquals(listOf(listOf(text("a")), listOf(text("b")), listOf(text("c"))), result.rows) + } + + @Test + fun createViewWithNullsFirstRoundTripsThroughSqlWriter() { + run("CREATE TABLE scored (id INTEGER PRIMARY KEY, score INTEGER)") + run("INSERT INTO scored VALUES (1, 10), (2, NULL), (3, 30), (4, NULL)") + val statement = parseOne("CREATE VIEW ranked AS SELECT id, score FROM scored ORDER BY score DESC NULLS FIRST") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ORDER BY score DESC NULLS FIRST"), rendered) + run(rendered) + val result = select("SELECT id FROM ranked") + assertEquals( + listOf(Value.IntegerValue(2), Value.IntegerValue(4), Value.IntegerValue(3), Value.IntegerValue(1)) + .map { listOf(it) }, + result.rows + ) + } + + @Test + fun createViewRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("CREATE VIEW letters AS SELECT id, category FROM t WHERE category IS NOT NULL") + val result = select("SELECT category FROM letters ORDER BY id ASC") + assertEquals(listOf(listOf(text("a")), listOf(text("b")), listOf(text("c"))), result.rows) + } + + @Test + fun createOrReplaceViewRoundTripsThroughSqlWriter() { + seed() + run("CREATE VIEW letters AS SELECT id FROM t") + writeThenReplay("CREATE OR REPLACE VIEW letters AS SELECT id FROM t WHERE id > 1") + val result = select("SELECT id FROM letters ORDER BY id ASC") + assertEquals(listOf(listOf(Value.IntegerValue(2)), listOf(Value.IntegerValue(3))), result.rows) + } + + @Test + fun createViewOverARollupGroupByRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 100), ('eng', 200), ('ops', 50)") + writeThenReplay("CREATE VIEW dept_totals AS SELECT dept, SUM(amount) AS total FROM sales GROUP BY ROLLUP (dept)") + val result = select("SELECT dept, total FROM dept_totals ORDER BY dept") + assertEquals( + listOf( + listOf(Value.NullValue, Value.IntegerValue(350)), + listOf(text("eng"), Value.IntegerValue(300)), + listOf(text("ops"), Value.IntegerValue(50)) + ), + result.rows + ) + } + + @Test + fun createViewOverStringAggWithASeparatorAndAnInternalOrderByRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, region TEXT)") + run("INSERT INTO sales VALUES ('eng', 'us'), ('eng', 'eu')") + writeThenReplay( + "CREATE VIEW dept_regions AS SELECT dept, STRING_AGG(region, '/' ORDER BY region) AS regions " + + "FROM sales GROUP BY dept" + ) + val result = select("SELECT dept, regions FROM dept_regions") + assertEquals(listOf(listOf(text("eng"), text("eu/us"))), result.rows) + } + + @Test + fun createViewOverACountDistinctAggregateRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100)") + writeThenReplay("CREATE VIEW dept_distinct_amounts AS SELECT dept, COUNT(DISTINCT amount) AS n FROM sales GROUP BY dept") + val result = select("SELECT dept, n FROM dept_distinct_amounts ORDER BY dept") + assertEquals( + listOf(listOf(text("eng"), Value.IntegerValue(2)), listOf(text("ops"), Value.IntegerValue(1))), + result.rows + ) + } + + @Test + fun createViewOverAFilteredAggregateRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100)") + writeThenReplay( + "CREATE VIEW dept_high_value_counts AS SELECT dept, COUNT(*) FILTER (WHERE amount > 100) AS n " + + "FROM sales GROUP BY dept" + ) + val result = select("SELECT dept, n FROM dept_high_value_counts ORDER BY dept") + assertEquals( + listOf(listOf(text("eng"), Value.IntegerValue(1)), listOf(text("ops"), Value.IntegerValue(0))), + result.rows + ) + } + + @Test + fun createViewOverDistinctRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, region TEXT)") + run("INSERT INTO sales VALUES ('eng', 'eu'), ('eng', 'eu'), ('ops', 'eu')") + writeThenReplay("CREATE VIEW depts AS SELECT DISTINCT dept FROM sales") + val result = select("SELECT dept FROM depts ORDER BY dept") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops"))), result.rows) + } + + @Test + fun createViewOverLimitAndOffsetRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 300), ('eng', 200), ('ops', 100)") + writeThenReplay("CREATE VIEW second_highest AS SELECT dept, amount FROM sales ORDER BY amount DESC LIMIT 1 OFFSET 1") + val result = select("SELECT dept, amount FROM second_highest") + assertEquals(listOf(listOf(text("eng"), Value.IntegerValue(200))), result.rows) + } + + @Test + fun createViewOverDistinctOnRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 100), ('eng', 200), ('ops', 50)") + writeThenReplay( + "CREATE VIEW top_per_dept AS SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, amount DESC" + ) + val result = select("SELECT dept, amount FROM top_per_dept ORDER BY dept") + assertEquals( + listOf(listOf(text("eng"), Value.IntegerValue(200)), listOf(text("ops"), Value.IntegerValue(50))), + result.rows + ) + } + + @Test + fun createViewOverGroupingSetsRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (dept TEXT, region TEXT, amount INTEGER)") + run("INSERT INTO sales VALUES ('eng', 'eu', 100), ('eng', 'us', 200), ('ops', 'eu', 50)") + writeThenReplay( + "CREATE VIEW totals AS SELECT dept, region, SUM(amount) AS total FROM sales GROUP BY GROUPING SETS ((dept), (region))" + ) + val result = select("SELECT dept, region, total FROM totals ORDER BY dept, region") + assertEquals( + listOf( + listOf(Value.NullValue, text("eu"), Value.IntegerValue(150)), + listOf(Value.NullValue, text("us"), Value.IntegerValue(200)), + listOf(text("eng"), Value.NullValue, Value.IntegerValue(300)), + listOf(text("ops"), Value.NullValue, Value.IntegerValue(50)) + ), + result.rows + ) + } + + @Test + fun dropViewRoundTripsThroughSqlWriter() { + seed() + run("CREATE VIEW letters AS SELECT id FROM t") + writeThenReplay("DROP VIEW letters") + assertEquals(emptyList(), catalog.viewNames) + } + + @Test + fun createViewOverADerivedTableRoundTripsThroughSqlWriter() { + seed() + writeThenReplay( + "CREATE VIEW letters AS SELECT s.id, s.category FROM (SELECT id, category FROM t WHERE bio IS NULL) AS s" + ) + val result = select("SELECT id FROM letters") + assertEquals(listOf(listOf(Value.IntegerValue(2))), result.rows) + } + + @Test + fun createViewOverALateralJoinRoundTripsThroughSqlWriter() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'sales')") + run( + "INSERT INTO employees VALUES (1, 1, 'ada', 300), (2, 1, 'linus', 250), (3, 2, 'sam', 150)" + ) + val statement = parseOne( + "CREATE VIEW top_earners AS SELECT d.name AS dept, top.name AS emp FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("LATERAL ("), rendered) + run(rendered) + val result = select("SELECT dept, emp FROM top_earners ORDER BY dept ASC") + assertEquals(listOf(listOf(text("eng"), text("ada")), listOf(text("sales"), text("sam"))), result.rows) + } + + @Test + fun dumpAndReplayOfALateralViewPreservesTheLateralKeyword() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1, 'ada', 300), (2, 1, 'linus', 250)") + run( + "CREATE VIEW top_earners AS SELECT d.name AS dept, top.name AS emp FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE" + ) + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("LATERAL ("), dump) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + val result = Parser(Lexer("SELECT emp FROM top_earners").tokenize()).parseProgram().single() + .let { freshExecutor.execute(it) } as SelectResult + assertEquals(listOf(text("ada")), result.rows.single()) + } + + @Test + fun createViewOverARightFullAndCrossJoinRoundTripsThroughSqlWriterWithCorrectKeywords() { + run("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT)") + run("CREATE TABLE orders3 (id INTEGER PRIMARY KEY, customerId INTEGER)") + run("INSERT INTO customers VALUES (1, 'ada'), (2, 'grace')") + // customerId 99 matches no customer, and customer 'grace' (id 2) has no order - so a plain + // INNER JOIN would drop both, letting RIGHT/FULL prove they actually kept what INNER cannot. + run("INSERT INTO orders3 VALUES (10, 1), (11, 99)") + + val rightStatement = parseOne("CREATE VIEW r AS SELECT customers.name, orders3.id FROM customers RIGHT JOIN orders3 ON customers.id = orders3.customerId") + val rightRendered = SqlWriter.write(rightStatement) + assertTrue(rightRendered.contains("RIGHT JOIN"), rightRendered) + run(rightRendered) + assertEquals( + setOf(listOf(text("ada"), Value.IntegerValue(10)), listOf(Value.NullValue, Value.IntegerValue(11))), + select("SELECT name, id FROM r").rows.toSet() + ) + + val fullStatement = parseOne("CREATE VIEW f AS SELECT customers.name, orders3.id FROM customers FULL JOIN orders3 ON customers.id = orders3.customerId") + val fullRendered = SqlWriter.write(fullStatement) + assertTrue(fullRendered.contains("FULL JOIN"), fullRendered) + run(fullRendered) + assertEquals(3, select("SELECT name, id FROM f").rows.size) + + val crossStatement = parseOne("CREATE VIEW x AS SELECT customers.name, orders3.id FROM customers CROSS JOIN orders3") + val crossRendered = SqlWriter.write(crossStatement) + assertTrue(crossRendered.contains("CROSS JOIN"), crossRendered) + assertTrue(!crossRendered.contains(" ON "), crossRendered) + run(crossRendered) + assertEquals(4, select("SELECT name, id FROM x").rows.size) + } + + @Test + fun createViewOverAUnionQueryRoundTripsThroughSqlWriter() { + seed() + writeThenReplay( + "CREATE VIEW letters AS SELECT category FROM t WHERE id = 1 UNION SELECT category FROM t WHERE id = 2" + ) + val result = select("SELECT category FROM letters ORDER BY category") + assertEquals(listOf(listOf(text("a")), listOf(text("b"))), result.rows) + } + + @Test + fun createViewOverAnIntersectAllQueryRoundTripsThroughSqlWriter() { + run("CREATE TABLE bag_a (v INTEGER)") + run("CREATE TABLE bag_b (v INTEGER)") + run("INSERT INTO bag_a VALUES (1), (1), (2)") + run("INSERT INTO bag_b VALUES (1), (2), (2)") + writeThenReplay("CREATE VIEW shared AS SELECT v FROM bag_a INTERSECT ALL SELECT v FROM bag_b") + val result = select("SELECT v FROM shared ORDER BY v") + assertEquals(listOf(listOf(Value.IntegerValue(1)), listOf(Value.IntegerValue(2))), result.rows) + } + + @Test + fun createViewOverARecursiveCteRoundTripsThroughSqlWriter() { + run("CREATE TABLE dual (x INTEGER)") + run("INSERT INTO dual VALUES (1)") + writeThenReplay( + "CREATE VIEW counter_view AS WITH RECURSIVE counter(n) AS (" + + "SELECT 1 FROM dual UNION ALL SELECT n + 1 FROM counter WHERE n < 3" + + ") SELECT n FROM counter" + ) + val result = select("SELECT n FROM counter_view ORDER BY n") + assertEquals(listOf(listOf(Value.IntegerValue(1)), listOf(Value.IntegerValue(2)), listOf(Value.IntegerValue(3))), result.rows) + } + + @Test + fun dumpIncludesViewDefinitionsAfterTablesAndIndexes() { + seed() + run("CREATE VIEW letters AS SELECT id, category FROM t WHERE category IS NOT NULL") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("CREATE VIEW letters AS")) + assertTrue(dump.indexOf("CREATE TABLE t") < dump.indexOf("CREATE VIEW letters")) + } + + @Test + fun createTableWithDecimalAndDateColumnsRoundTripsThroughSqlWriter() { + val rendered = SqlWriter.write(parseOne("CREATE TABLE priced (amount DECIMAL(10, 2), born DATE)")) + assertTrue(rendered.contains("DECIMAL(10, 2)")) + assertTrue(rendered.contains("DATE")) + writeThenReplay("CREATE TABLE priced (id INTEGER PRIMARY KEY, amount DECIMAL(10, 2), born DATE)") + run("INSERT INTO priced VALUES (1, 5, CAST('2024-01-01' AS DATE))") + val result = select("SELECT amount FROM priced") + assertEquals(listOf(listOf(Value.DecimalValue(BigDecimal("5.00")))), result.rows) + } + + @Test + fun castExpressionRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("UPDATE t SET category = CAST(id AS TEXT) WHERE id = 1") + assertEquals(listOf(listOf(text("1"))), select("SELECT category FROM t WHERE id = 1").rows) + } + + @Test + fun dropTableAndDropViewCascadeRoundTripThroughSqlWriter() { + seed() + run("CREATE VIEW letters AS SELECT id FROM t") + writeThenReplay("DROP TABLE t CASCADE") + assertTrue(catalog.tableNames.isEmpty()) + assertTrue(catalog.viewNames.isEmpty()) + } + + @Test + fun dumpAndReplayPreservesDecimalAndDateValuesExactly() { + run("CREATE TABLE priced (id INTEGER PRIMARY KEY, amount DECIMAL(12, 4), born DATE)") + run("INSERT INTO priced VALUES (1, CAST('123456.7890' AS DECIMAL(12, 4)), CAST('2024-01-15' AS DATE))") + val dump = SqlWriter.dump(catalog) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + val result = freshExecutor.execute( + Parser(Lexer("SELECT amount, born FROM priced").tokenize()).parseProgram().single() + ) as SelectResult + assertEquals( + listOf(listOf(Value.DecimalValue(BigDecimal("123456.7890")), Value.DateValue(LocalDate.of(2024, 1, 15)))), + result.rows + ) + } + + @Test + fun createTableWithForeignKeyAndCheckRoundTripsThroughSqlWriter() { + run(SqlWriter.write(parseOne("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)"))) + run( + SqlWriter.write( + parseOne( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER " + + "REFERENCES departments(id) ON DELETE CASCADE, age INTEGER CHECK (age > 0))" + ) + ) + ) + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1, 30)") + assertFailsWith { run("INSERT INTO employees VALUES (2, 1, -1)") } + run("DELETE FROM departments WHERE id = 1") + assertTrue(select("SELECT * FROM employees").rows.isEmpty()) + } + + @Test + fun dumpAndReplayPreservesForeignKeyAndCheckConstraints() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) " + + "ON DELETE CASCADE, age INTEGER CHECK (age > 0))" + ) + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1, 30)") + val dump = SqlWriter.dump(catalog) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + Parser(Lexer("DELETE FROM departments WHERE id = 1").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + val remaining = freshExecutor.execute( + Parser(Lexer("SELECT * FROM employees").tokenize()).parseProgram().single() + ) as SelectResult + assertTrue(remaining.rows.isEmpty()) + } + + @Test + fun createTableWithUniqueConstraintsRoundTripsThroughSqlWriter() { + run( + SqlWriter.write( + parseOne( + "CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE, a INTEGER, b INTEGER, UNIQUE (a, b))" + ) + ) + ) + run("INSERT INTO t VALUES (1, 'ada@example.com', 1, 1)") + assertFailsWith { run("INSERT INTO t VALUES (2, 'ada@example.com', 2, 2)") } + assertFailsWith { run("INSERT INTO t VALUES (3, 'grace@example.com', 1, 1)") } + val result = exec("INSERT INTO t VALUES (4, 'grace@example.com', 1, 2)") + assertEquals(InsertResult(1), result) + } + + @Test + fun dumpAndReplayPreservesUniqueConstraints() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO t VALUES (1, 'ada@example.com')") + val dump = SqlWriter.dump(catalog) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + assertFailsWith { + Parser(Lexer("INSERT INTO t VALUES (2, 'ada@example.com')").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + } + } + + @Test + fun createUniqueIndexRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT)") + writeThenReplay("CREATE UNIQUE INDEX idx_email ON t (email)") + run("INSERT INTO t VALUES (1, 'ada@example.com')") + assertFailsWith { run("INSERT INTO t VALUES (2, 'ada@example.com')") } + } + + @Test + fun dumpAndReplayPreservesAUniqueIndex() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT)") + run("CREATE UNIQUE INDEX idx_email ON t (email)") + run("INSERT INTO t VALUES (1, 'ada@example.com')") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("CREATE UNIQUE INDEX idx_email ON t (email);")) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + assertFailsWith { + Parser(Lexer("INSERT INTO t VALUES (2, 'ada@example.com')").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + } + } + + @Test + fun alterTableAddColumnWithDefaultRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("ALTER TABLE t ADD COLUMN status TEXT NOT NULL DEFAULT 'new'") + assertEquals(listOf(text("new"), text("new"), text("new")), select("SELECT status FROM t").rows.map { it.single() }) + run("INSERT INTO t (id) VALUES (4)") + assertEquals(listOf(text("new")), select("SELECT status FROM t WHERE id = 4").rows.single()) + } + + @Test + fun alterTableDropColumnRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("ALTER TABLE t DROP COLUMN bio") + val columns = (select("SELECT * FROM t WHERE id = 1")).columns + assertTrue(columns.none { it.equals("bio", ignoreCase = true) }) + } + + @Test + fun alterTableRenameColumnRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("ALTER TABLE t RENAME COLUMN bio TO biography") + assertEquals(listOf(text("mathematician")), select("SELECT biography FROM t WHERE id = 1").rows.single()) + } + + @Test + fun alterTableRenameToRoundTripsThroughSqlWriter() { + seed() + writeThenReplay("ALTER TABLE t RENAME TO renamed") + assertEquals(listOf(text("mathematician")), select("SELECT bio FROM renamed WHERE id = 1").rows.single()) + } + + @Test + fun dumpAndReplayOfARenamedTableWithADependentViewPreservesTheRewrittenReference() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO departments VALUES (1, 'eng')") + run("CREATE VIEW v AS SELECT id, name FROM departments") + run("ALTER TABLE departments RENAME TO divisions") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("FROM divisions"), dump) + assertTrue(!dump.contains("FROM departments"), dump) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + val result = Parser(Lexer("SELECT * FROM v").tokenize()).parseProgram().single() + .let { freshExecutor.execute(it) } as SelectResult + assertEquals(listOf(text("eng")), listOf(result.rows.single()[1])) + } + + @Test + fun dumpAndReplayPreservesColumnDefaultValue() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + run("INSERT INTO t (id) VALUES (1)") + val dump = SqlWriter.dump(catalog) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + freshExecutor.execute(Parser(Lexer("INSERT INTO t (id) VALUES (2)").tokenize()).parseProgram().single()) + + val result = freshExecutor.execute( + Parser(Lexer("SELECT status FROM t ORDER BY id ASC").tokenize()).parseProgram().single() + ) as SelectResult + assertEquals(listOf(listOf(text("pending")), listOf(text("pending"))), result.rows) + } + + @Test + fun dumpAndReplayPreservesAGeneratedColumnsExpressionNotJustItsAlreadyComputedValue() { + run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + run("INSERT INTO t (a) VALUES (5)") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("GENERATED ALWAYS AS"), dump) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + // A row inserted only AFTER replay proves the GENERATED expression itself round-tripped + // (not just the value 10 already sitting in the dump's INSERT statement). + freshExecutor.execute(Parser(Lexer("INSERT INTO t (a) VALUES (7)").tokenize()).parseProgram().single()) + + val result = freshExecutor.execute( + Parser(Lexer("SELECT b FROM t ORDER BY a ASC").tokenize()).parseProgram().single() + ) as SelectResult + assertEquals(listOf(listOf(Value.IntegerValue(10)), listOf(Value.IntegerValue(14))), result.rows) + } + + @Test + fun dumpAfterAlterTableAddColumnReflectsTheNewShape() { + seed() + run("ALTER TABLE t ADD COLUMN status TEXT DEFAULT 'new'") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("status TEXT DEFAULT 'new'")) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + val result = freshExecutor.execute( + Parser(Lexer("SELECT status FROM t WHERE id = 1").tokenize()).parseProgram().single() + ) as SelectResult + assertEquals(listOf(listOf(text("new"))), result.rows) + } + + @Test + fun dumpAndReplayAfterAlterTableAddColumnGeneratedPreservesTheExpressionAndBackfilledValues() { + run("CREATE TABLE t (a INTEGER)") + run("INSERT INTO t (a) VALUES (5)") + run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("GENERATED ALWAYS AS"), dump) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + // A row inserted only AFTER replay proves the GENERATED expression itself round-tripped, + // not just the value backfilled into row a=5 before the dump was taken. + freshExecutor.execute(Parser(Lexer("INSERT INTO t (a) VALUES (7)").tokenize()).parseProgram().single()) + + val result = freshExecutor.execute( + Parser(Lexer("SELECT b FROM t ORDER BY a ASC").tokenize()).parseProgram().single() + ) as SelectResult + assertEquals(listOf(listOf(Value.IntegerValue(10)), listOf(Value.IntegerValue(14))), result.rows) + } + + @Test + fun createCompositeUniqueIndexRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER)") + writeThenReplay("CREATE UNIQUE INDEX idx_ab ON t (a, b)") + run("INSERT INTO t VALUES (1, 1, 1)") + assertFailsWith { run("INSERT INTO t VALUES (2, 1, 1)") } + run("INSERT INTO t VALUES (3, 1, 2)") + } + + @Test + fun dumpAndReplayPreservesACompositeUniqueIndex() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER)") + run("CREATE UNIQUE INDEX idx_ab ON t (a, b)") + run("INSERT INTO t VALUES (1, 1, 1)") + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("CREATE UNIQUE INDEX idx_ab ON t (a, b);")) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + + assertFailsWith { + Parser(Lexer("INSERT INTO t VALUES (2, 1, 1)").tokenize()).parseProgram() + .forEach { freshExecutor.execute(it) } + } + } + + @Test + fun createViewOverAWindowFunctionRoundTripsThroughSqlWriter() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept TEXT, salary INTEGER)") + run( + "INSERT INTO employees VALUES (1, 'eng', 300), (2, 'eng', 400), (3, 'sales', 150)" + ) + val statement = parseOne( + "CREATE VIEW ranked AS SELECT id, dept, " + + "RANK() OVER (PARTITION BY dept ORDER BY salary DESC) AS r FROM employees" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("OVER (PARTITION BY dept ORDER BY salary DESC)"), rendered) + run(rendered) + val result = select("SELECT id, r FROM ranked ORDER BY id ASC") + assertEquals( + listOf( + listOf(Value.IntegerValue(1), Value.IntegerValue(2)), + listOf(Value.IntegerValue(2), Value.IntegerValue(1)), + listOf(Value.IntegerValue(3), Value.IntegerValue(1)) + ), + result.rows + ) + } + + @Test + fun dumpAndReplayOfAWindowFunctionViewPreservesTheOverClause() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept TEXT, salary INTEGER)") + run("INSERT INTO employees VALUES (1, 'eng', 300), (2, 'eng', 400)") + run( + "CREATE VIEW totals AS SELECT id, SUM(salary) OVER (PARTITION BY dept) AS deptTotal FROM employees" + ) + val dump = SqlWriter.dump(catalog) + assertTrue(dump.contains("OVER (PARTITION BY dept)"), dump) + + val freshCatalog = Catalog() + val freshExecutor = Executor(freshCatalog) + Parser(Lexer(dump).tokenize()).parseProgram().forEach { freshExecutor.execute(it) } + val result = Parser(Lexer("SELECT deptTotal FROM totals ORDER BY id ASC").tokenize()).parseProgram().single() + .let { freshExecutor.execute(it) } as SelectResult + assertEquals(listOf(listOf(Value.IntegerValue(700)), listOf(Value.IntegerValue(700))), result.rows) + } + + @Test + fun createViewOverLagWithOffsetAndDefaultRoundTripsThroughSqlWriter() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept TEXT, salary INTEGER)") + run("INSERT INTO employees VALUES (1, 'eng', 400), (2, 'eng', 300)") + val statement = parseOne( + "CREATE VIEW withPrev AS SELECT id, dept, " + + "LAG(salary, 1, 0) OVER (PARTITION BY dept ORDER BY salary DESC) AS prevSalary FROM employees" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("LAG(salary, 1, 0) OVER (PARTITION BY dept ORDER BY salary DESC)"), rendered) + run(rendered) + val result = select("SELECT id, prevSalary FROM withPrev ORDER BY id ASC") + assertEquals( + listOf( + listOf(Value.IntegerValue(1), Value.IntegerValue(0)), + listOf(Value.IntegerValue(2), Value.IntegerValue(400)) + ), + result.rows + ) + } + + @Test + fun createViewOverAnExplicitRowsFrameRoundTripsThroughSqlWriter() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept TEXT, salary INTEGER)") + run("INSERT INTO employees VALUES (1, 'eng', 300), (2, 'eng', 400), (3, 'eng', 100)") + val statement = parseOne( + "CREATE VIEW moving AS SELECT id, dept, " + + "SUM(salary) OVER (PARTITION BY dept ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees" + ) + val rendered = SqlWriter.write(statement) + assertTrue( + rendered.contains("OVER (PARTITION BY dept ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)"), + rendered + ) + run(rendered) + val result = select("SELECT id, s FROM moving ORDER BY id ASC") + assertEquals( + listOf( + listOf(Value.IntegerValue(1), Value.IntegerValue(300)), + listOf(Value.IntegerValue(2), Value.IntegerValue(700)), + listOf(Value.IntegerValue(3), Value.IntegerValue(500)) + ), + result.rows + ) + } + + @Test + fun rowsFrameShorthandIsWrittenBackOutInCanonicalBetweenForm() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, salary INTEGER)") + run("INSERT INTO employees VALUES (1, 400), (2, 300)") + val statement = parseOne( + "CREATE VIEW shorthand AS SELECT id, SUM(salary) OVER (ORDER BY id ROWS 1 PRECEDING) AS s FROM employees" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ROWS BETWEEN 1 PRECEDING AND CURRENT ROW"), rendered) + } + + @Test + fun unboundedFrameBoundsRoundTripThroughSqlWriter() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, salary INTEGER)") + run("INSERT INTO employees VALUES (1, 400), (2, 300)") + val statement = parseOne( + "CREATE VIEW wholePartition AS SELECT id, " + + "SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS s FROM employees" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"), rendered) + } + + @Test + fun createViewOverAnExplicitRangeFrameRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20), (3, 20), (4, 30), (5, 50)") + val statement = parseOne( + "CREATE VIEW valueTotals AS SELECT id, " + + "SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s FROM sales" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("OVER (ORDER BY amount RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING)"), rendered) + run(rendered) + val result = select("SELECT id, s FROM valueTotals ORDER BY id ASC") + assertEquals( + listOf(50L, 80L, 80L, 70L, 50L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun createViewOverAnExcludeCurrentRowFrameRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20), (3, 20), (4, 30), (5, 50)") + val statement = parseOne( + "CREATE VIEW excludingSelf AS SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW" + + ") AS s FROM sales" + ) + val rendered = SqlWriter.write(statement) + assertTrue( + rendered.contains("ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW"), + rendered + ) + run(rendered) + val result = select("SELECT id, s FROM excludingSelf ORDER BY id ASC") + assertEquals( + listOf(120L, 110L, 110L, 100L, 80L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun createViewOverAnExplicitGroupsFrameRoundTripsThroughSqlWriter() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20), (3, 20), (4, 30), (5, 50)") + val statement = parseOne( + "CREATE VIEW groupTotals AS SELECT id, " + + "SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s FROM sales" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)"), rendered) + run(rendered) + val result = select("SELECT id, s FROM groupTotals ORDER BY id ASC") + assertEquals( + listOf(50L, 80L, 80L, 120L, 80L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun excludeNoOthersIsOmittedFromRenderedSqlSinceItIsTheDefault() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20)") + val statement = parseOne( + "CREATE VIEW totals AS SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE NO OTHERS" + + ") AS s FROM sales" + ) + val rendered = SqlWriter.write(statement) + assertTrue(!rendered.contains("EXCLUDE"), rendered) + } + + @Test + fun insertOnConflictDoNothingRoundTripsThroughSqlWriter() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + val statement = parseOne("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO NOTHING") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ON CONFLICT (id) DO NOTHING"), rendered) + run(rendered) + assertEquals(listOf(listOf(Value.IntegerValue(100))), select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun insertOnConflictDoUpdateWithExcludedAndWhereRoundTripsThroughSqlWriter() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + val statement = parseOne( + "INSERT INTO accounts VALUES (1, 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance WHERE excluded.balance > 0" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ON CONFLICT (id) DO UPDATE SET balance = (accounts.balance + excluded.balance) WHERE (excluded.balance > 0)"), rendered) + run(rendered) + assertEquals(listOf(listOf(Value.IntegerValue(150))), select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun createTriggerRoundTripsThroughSqlWriter() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + run("CREATE TABLE audit (order_id INTEGER)") + val statement = parseOne( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW WHEN NEW.amount > 0 " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW WHEN"), rendered) + assertTrue(rendered.contains("INSERT INTO audit VALUES (NEW.id)"), rendered) + run(rendered) + run("INSERT INTO orders VALUES (1, 50)") + assertEquals(listOf(listOf(Value.IntegerValue(1))), select("SELECT order_id FROM audit").rows) + } + + @Test + fun dropTriggerRoundTripsThroughSqlWriter() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + val rendered = SqlWriter.write(parseOne("DROP TRIGGER log")) + assertEquals("DROP TRIGGER log;", rendered) + run(rendered) + assertFailsWith { run("DROP TRIGGER log") } + } + + @Test + fun createSequenceRoundTripsThroughSqlWriter() { + val statement = parseOne("CREATE SEQUENCE seq START WITH 5 INCREMENT BY 2 MAXVALUE 100 CYCLE") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("CREATE SEQUENCE seq"), rendered) + assertTrue(rendered.contains("INCREMENT BY 2"), rendered) + assertTrue(rendered.contains("MAXVALUE 100"), rendered) + assertTrue(rendered.contains("START WITH 5"), rendered) + assertTrue(rendered.contains("CYCLE"), rendered) + run(rendered) + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(Value.IntegerValue(5))), select("SELECT id FROM t").rows) + } + + @Test + fun dropSequenceRoundTripsThroughSqlWriter() { + run("CREATE SEQUENCE seq") + val rendered = SqlWriter.write(parseOne("DROP SEQUENCE seq")) + assertEquals("DROP SEQUENCE seq;", rendered) + run(rendered) + assertFailsWith { run("DROP SEQUENCE seq") } + } + + @Test + fun alterSequenceRoundTripsThroughSqlWriter() { + run("CREATE SEQUENCE seq") + val statement = parseOne("ALTER SEQUENCE seq INCREMENT BY 5 RESTART WITH 20") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("ALTER SEQUENCE seq"), rendered) + assertTrue(rendered.contains("INCREMENT BY 5"), rendered) + assertTrue(rendered.contains("RESTART WITH 20"), rendered) + run(rendered) + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(Value.IntegerValue(20))), select("SELECT id FROM t").rows) + } + + @Test + fun nextvalDefaultRoundTripsThroughSqlWriter() { + run("CREATE SEQUENCE seq") + val statement = parseOne("CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('seq'))") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("DEFAULT NEXTVAL('seq')"), rendered) + run(rendered) + run("INSERT INTO orders (id) VALUES (DEFAULT)") + assertEquals(listOf(listOf(Value.IntegerValue(1))), select("SELECT id FROM orders").rows) + } + + @Test + fun currvalRoundTripsThroughSqlWriter() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER)") + run("INSERT INTO t VALUES (NEXTVAL('seq'))") + val statement = parseOne("UPDATE t SET id = CURRVAL('seq')") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.contains("CURRVAL('seq')"), rendered) + run(rendered) + assertEquals(listOf(listOf(Value.IntegerValue(1))), select("SELECT id FROM t").rows) + } + + /** + * `RETURNING` is a statement-level, query-time-only clause: its output was already consumed by + * whoever issued the statement (the REPL printed it, or a caller read the [StatementResult]), so + * replaying the write on reopen has no reason to recompute or re-print it - only the write itself + * needs to survive. [SqlWriter] therefore never round-trips it, matching that a persisted dump + * replayed on reopen is a silent process ([kestrel.cli.Repl]'s own `replaying` flag suppresses + * [kestrel.cli.Repl.printResult] during it) with nothing to print RETURNING's rows to anyway. + */ + @Test + fun returningClauseIsDroppedFromThePersistedSql() { + seed() + val insert = parseOne("INSERT INTO t VALUES (4, 'd', 'baroness') RETURNING id") + val renderedInsert = SqlWriter.write(insert) + assertTrue(!renderedInsert.contains("RETURNING"), renderedInsert) + run(renderedInsert) + assertEquals(listOf(listOf(text("d"))), select("SELECT category FROM t WHERE id = 4").rows) + + val update = parseOne("UPDATE t SET category = 'z' WHERE id = 4 RETURNING category") + assertTrue(!SqlWriter.write(update).contains("RETURNING")) + + val delete = parseOne("DELETE FROM t WHERE id = 4 RETURNING id") + assertTrue(!SqlWriter.write(delete).contains("RETURNING")) + } + + /** Same reasoning as [returningClauseIsDroppedFromThePersistedSql], now for MERGE's own RETURNING. */ + @Test + fun mergeReturningClauseIsDroppedFromThePersistedSqlButTheMutationStillReplays() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("CREATE TABLE staging (id INTEGER, amount INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + run("INSERT INTO staging VALUES (1, 10)") + val merge = parseOne( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "RETURNING id, MERGE_ACTION()" + ) + val rendered = SqlWriter.write(merge) + assertTrue(!rendered.contains("RETURNING"), rendered) + assertTrue(!rendered.contains("MERGE_ACTION"), rendered) + run(rendered) + assertEquals(listOf(listOf(Value.IntegerValue(110))), select("SELECT balance FROM accounts").rows) + } + + @Test + fun writeOfATemporaryCreateTableIncludesTheTemporaryKeyword() { + val statement = parseOne("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY)") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.startsWith("CREATE TEMPORARY TABLE scratch ("), rendered) + } + + @Test + fun writeOfARegularCreateTableOmitsTheTemporaryKeyword() { + val statement = parseOne("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val rendered = SqlWriter.write(statement) + assertTrue(rendered.startsWith("CREATE TABLE t ("), rendered) + } + + /** + * The direct, single-function counterpart to [kestrel.PersistenceTest]'s full + * checkpoint-reopen tests: proves [SqlWriter.dump] itself never mentions a TEMP table (or its + * rows, or an index/trigger defined on it), rather than relying on the round trip through + * [kestrel.persistence.Persistence]/[kestrel.cli.Repl] to demonstrate the same thing indirectly. + */ + @Test + fun dumpOmitsATemporaryTableItsRowsItsIndexAndItsTriggerEntirely() { + run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, tag TEXT)") + run("CREATE INDEX scratchTag ON scratch (tag)") + run("CREATE TRIGGER scratchGuard BEFORE INSERT ON scratch FOR EACH ROW BEGIN DELETE FROM scratch WHERE id = -1; END") + run("INSERT INTO scratch VALUES (1, 'x')") + run("CREATE TABLE regular (id INTEGER PRIMARY KEY)") + run("INSERT INTO regular VALUES (9)") + + val dump = SqlWriter.dump(catalog) + assertTrue(!dump.contains("scratch", ignoreCase = true), dump) + assertTrue(dump.contains("CREATE TABLE regular ("), dump) + assertTrue(dump.contains("INSERT INTO regular VALUES (9)"), dump) + } + + private fun exec(sql: String): StatementResult = run(sql).single() +} diff --git a/src/test/kotlin/kestrel/SubqueryTest.kt b/src/test/kotlin/kestrel/SubqueryTest.kt new file mode 100644 index 0000000..c2e4143 --- /dev/null +++ b/src/test/kotlin/kestrel/SubqueryTest.kt @@ -0,0 +1,235 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class SubqueryTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seed() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL, budget INTEGER)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, department_id INTEGER, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng', 1000), (2, 'sales', 500), (3, 'ops', 0)") + run( + "INSERT INTO employees VALUES " + + "(1, 'ada', 1, 120), (2, 'linus', 1, 90), (3, 'grace', 2, 80), (4, 'guido', 2, 70)" + ) + } + + @Test + fun scalarSubqueryComparesAgainstAggregateOfAnotherQuery() { + seed() + val result = select("SELECT name FROM employees WHERE salary > (SELECT AVG(salary) FROM employees) ORDER BY name") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun scalarSubqueryReturningNoRowsEvaluatesToNull() { + seed() + val result = select("SELECT (SELECT id FROM departments WHERE name = 'nope') FROM departments WHERE id = 1") + assertEquals(listOf(listOf(Value.NullValue)), result.rows) + } + + @Test + fun scalarSubqueryReturningMultipleRowsFails() { + seed() + assertFailsWith { + select("SELECT (SELECT salary FROM employees) FROM departments WHERE id = 1") + } + } + + @Test + fun scalarSubqueryReturningMultipleColumnsFails() { + seed() + assertFailsWith { + select("SELECT (SELECT id, name FROM departments) FROM departments WHERE id = 1") + } + } + + @Test + fun correlatedScalarSubqueryInProjectionCountsPerRow() { + seed() + val result = select( + "SELECT d.name, (SELECT COUNT(*) FROM employees e WHERE e.department_id = d.id) " + + "FROM departments d ORDER BY d.id ASC" + ) + assertEquals( + listOf( + listOf(text("eng"), int(2)), + listOf(text("sales"), int(2)), + listOf(text("ops"), int(0)) + ), + result.rows + ) + } + + @Test + fun correlatedScalarSubqueryInWhereComparesAgainstOwnGroupAverage() { + seed() + val result = select( + "SELECT name FROM employees e WHERE e.salary > " + + "(SELECT AVG(salary) FROM employees e2 WHERE e2.department_id = e.department_id) ORDER BY name ASC" + ) + assertEquals(listOf(listOf(text("ada")), listOf(text("grace"))), result.rows) + } + + @Test + fun inSubqueryMatchesUncorrelatedResultSet() { + seed() + val result = select("SELECT name FROM departments WHERE id IN (SELECT department_id FROM employees) ORDER BY name ASC") + assertEquals(listOf(listOf(text("eng")), listOf(text("sales"))), result.rows) + } + + @Test + fun notInSubqueryExcludesMatchingRows() { + seed() + val result = select("SELECT name FROM departments WHERE id NOT IN (SELECT department_id FROM employees)") + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun notInSubqueryWithNullInResultSetExcludesEverythingByThreeValuedLogic() { + run("CREATE TABLE d (id INTEGER)") + run("CREATE TABLE e (dept INTEGER)") + run("INSERT INTO d VALUES (1), (2)") + run("INSERT INTO e VALUES (1), (NULL)") + val result = select("SELECT id FROM d WHERE id NOT IN (SELECT dept FROM e)") + assertEquals(emptyList(), result.rows) + } + + @Test + fun inSubqueryRequiresExactlyOneColumn() { + seed() + assertFailsWith { + select("SELECT name FROM departments WHERE id IN (SELECT id, name FROM departments)") + } + } + + @Test + fun existsWithoutCorrelationChecksWhetherSubqueryHasAnyRows() { + seed() + val result = select("SELECT name FROM departments WHERE EXISTS (SELECT 1 FROM employees) ORDER BY name ASC") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops")), listOf(text("sales"))), result.rows) + } + + @Test + fun correlatedExistsFindsDepartmentsWithEmployees() { + seed() + val result = select( + "SELECT name FROM departments d WHERE EXISTS " + + "(SELECT 1 FROM employees e WHERE e.department_id = d.id) ORDER BY name ASC" + ) + assertEquals(listOf(listOf(text("eng")), listOf(text("sales"))), result.rows) + } + + @Test + fun correlatedNotExistsFindsDepartmentsWithoutEmployees() { + seed() + val result = select( + "SELECT name FROM departments d WHERE NOT EXISTS " + + "(SELECT 1 FROM employees e WHERE e.department_id = d.id)" + ) + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun correlatedScalarSubqueryUsableInUpdateSet() { + seed() + run("CREATE TABLE dept_stats (id INTEGER PRIMARY KEY, headcount INTEGER)") + run("INSERT INTO dept_stats VALUES (1, 0), (2, 0), (3, 0)") + run( + "UPDATE dept_stats SET headcount = " + + "(SELECT COUNT(*) FROM employees WHERE employees.department_id = dept_stats.id)" + ) + val result = select("SELECT headcount FROM dept_stats ORDER BY id ASC") + assertEquals(listOf(listOf(int(2)), listOf(int(2)), listOf(int(0))), result.rows) + } + + @Test + fun inSubqueryUsableInDeleteWhereClause() { + seed() + run("INSERT INTO employees VALUES (5, 'temp', 3, 10)") + run("DELETE FROM employees WHERE department_id IN (SELECT id FROM departments WHERE budget = 0)") + val result = select("SELECT name FROM employees ORDER BY name ASC") + assertEquals( + listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("guido")), listOf(text("linus"))), + result.rows + ) + } + + @Test + fun scalarSubqueryUsableAsInsertValue() { + seed() + run("CREATE TABLE summary (id INTEGER PRIMARY KEY, employee_count INTEGER)") + run("INSERT INTO summary VALUES (1, (SELECT COUNT(*) FROM employees))") + val result = select("SELECT employee_count FROM summary") + assertEquals(listOf(listOf(int(4))), result.rows) + } + + @Test + fun correlatedScalarSubqueryInUpdateSetRoundTripsThroughSqlWriter() { + seed() + run("CREATE TABLE dept_stats (id INTEGER PRIMARY KEY, headcount INTEGER)") + run("INSERT INTO dept_stats VALUES (1, 0), (2, 0), (3, 0)") + val statement = Parser( + Lexer( + "UPDATE dept_stats SET headcount = " + + "(SELECT COUNT(*) FROM employees WHERE employees.department_id = dept_stats.id)" + ).tokenize() + ).parseProgram().single() + val rendered = kestrel.persistence.SqlWriter.write(statement) + val reparsed = Parser(Lexer(rendered).tokenize()).parseProgram().single() + executor.execute(reparsed) + val result = select("SELECT headcount FROM dept_stats ORDER BY id ASC") + assertEquals(listOf(listOf(int(2)), listOf(int(2)), listOf(int(0))), result.rows) + } + + @Test + fun inSubqueryInDeleteWhereRoundTripsThroughSqlWriter() { + seed() + run("INSERT INTO employees VALUES (5, 'temp', 3, 10)") + val statement = Parser( + Lexer("DELETE FROM employees WHERE department_id IN (SELECT id FROM departments WHERE budget = 0)").tokenize() + ).parseProgram().single() + val rendered = kestrel.persistence.SqlWriter.write(statement) + val reparsed = Parser(Lexer(rendered).tokenize()).parseProgram().single() + executor.execute(reparsed) + val result = select("SELECT name FROM employees ORDER BY name ASC") + assertEquals( + listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("guido")), listOf(text("linus"))), + result.rows + ) + } + + @Test + fun aggregateOverUncorrelatedScalarSubqueryEvaluatesItPerRow() { + seed() + val result = select("SELECT SUM((SELECT budget FROM departments WHERE id = 1)) FROM employees") + assertEquals(listOf(listOf(int(1000 * 4))), result.rows) + } + + @Test + fun existsResultCannotBeUsedAsAnArithmeticOperand() { + seed() + assertFailsWith { + select("SELECT 1 + EXISTS (SELECT 1 FROM employees) FROM departments WHERE id = 1") + } + } +} diff --git a/src/test/kotlin/kestrel/TriggerTest.kt b/src/test/kotlin/kestrel/TriggerTest.kt new file mode 100644 index 0000000..eb82f94 --- /dev/null +++ b/src/test/kotlin/kestrel/TriggerTest.kt @@ -0,0 +1,337 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class TriggerTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + @Test + fun afterInsertTriggerRunsBodyForEachInsertedRow() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + run("CREATE TABLE audit (order_id INTEGER)") + run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + run("INSERT INTO orders VALUES (1, 100), (2, 200)") + assertEquals( + listOf(listOf(int(1)), listOf(int(2))), + select("SELECT order_id FROM audit ORDER BY order_id").rows + ) + } + + @Test + fun beforeInsertTriggerCanAbortTheWholeStatementBeforeThisEnginesOwnConstraintsRun() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE guard (x INTEGER NOT NULL)") + run("CREATE TRIGGER reject_all BEFORE INSERT ON t FOR EACH ROW BEGIN INSERT INTO guard VALUES (NULL); END") + // The row itself is perfectly valid for t's own PRIMARY KEY - only the trigger rejects it. + assertFailsWith { run("INSERT INTO t VALUES (1)") } + assertEquals(0, select("SELECT * FROM t").rows.size) + } + + @Test + fun beforeTriggerSideEffectsAreRolledBackWhenTheFiringStatementItselfFails() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY CHECK (id > 0))") + run("CREATE TABLE counter (n INTEGER)") + run("INSERT INTO counter VALUES (0)") + run("CREATE TRIGGER bump BEFORE INSERT ON t FOR EACH ROW BEGIN UPDATE counter SET n = n + 1; END") + assertFailsWith { run("INSERT INTO t VALUES (-1)") } + assertEquals(listOf(listOf(int(0))), select("SELECT n FROM counter").rows) + assertEquals(0, select("SELECT * FROM t").rows.size) + } + + @Test + fun afterInsertTriggerFailureRollsBackTheOriginalInsertToo() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE audit (order_id INTEGER NOT NULL)") + run("CREATE TRIGGER log_fail AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NULL); END") + assertFailsWith { run("INSERT INTO t VALUES (1)") } + assertEquals(0, select("SELECT * FROM t").rows.size) + assertEquals(0, select("SELECT * FROM audit").rows.size) + } + + @Test + fun whenClauseGatesFiringPerRow() { + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + run("CREATE TABLE audit (order_id INTEGER)") + run( + "CREATE TRIGGER log_big AFTER INSERT ON orders FOR EACH ROW WHEN NEW.amount > 100 " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + run("INSERT INTO orders VALUES (1, 50), (2, 200)") + assertEquals(listOf(listOf(int(2))), select("SELECT order_id FROM audit").rows) + } + + @Test + fun updateTriggerSeesBothOldAndNew() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("CREATE TABLE audit (account_id INTEGER, old_balance INTEGER, new_balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + run( + "CREATE TRIGGER log_balance_change AFTER UPDATE ON accounts FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id, OLD.balance, NEW.balance); END" + ) + run("UPDATE accounts SET balance = 150 WHERE id = 1") + assertEquals(listOf(listOf(int(1), int(100), int(150))), select("SELECT * FROM audit").rows) + } + + @Test + fun updateTriggerFiresForEveryMatchedRowEvenWhenTheValueIsUnchanged() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + run("CREATE TABLE audit (n INTEGER)") + run("INSERT INTO t VALUES (1, 5)") + run("CREATE TRIGGER log AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (1); END") + run("UPDATE t SET x = 5 WHERE id = 1") + assertEquals(1, select("SELECT * FROM audit").rows.size) + } + + @Test + fun deleteTriggerSeesOld() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + run("CREATE TABLE audit (label TEXT)") + run("INSERT INTO t VALUES (1, 'gone')") + run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (OLD.label); END") + run("DELETE FROM t WHERE id = 1") + assertEquals(listOf(listOf(text("gone"))), select("SELECT * FROM audit").rows) + } + + @Test + fun multipleTriggersOnSameEventFireInCreationOrder() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TABLE audit (step TEXT)") + run("CREATE TRIGGER first_trigger AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES ('first'); END") + run("CREATE TRIGGER second_trigger AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES ('second'); END") + run("INSERT INTO t VALUES (1)") + assertEquals(listOf(listOf(text("first")), listOf(text("second"))), select("SELECT * FROM audit").rows) + } + + @Test + fun cascadingTriggerFiresATriggerOnADifferentTable() { + run("CREATE TABLE a (id INTEGER)") + run("CREATE TABLE b (id INTEGER)") + run("CREATE TABLE c (id INTEGER)") + run("CREATE TRIGGER a_to_b AFTER INSERT ON a FOR EACH ROW BEGIN INSERT INTO b VALUES (NEW.id); END") + run("CREATE TRIGGER b_to_c AFTER INSERT ON b FOR EACH ROW BEGIN INSERT INTO c VALUES (NEW.id); END") + run("INSERT INTO a VALUES (7)") + assertEquals(listOf(listOf(int(7))), select("SELECT * FROM b").rows) + assertEquals(listOf(listOf(int(7))), select("SELECT * FROM c").rows) + } + + @Test + fun recursiveTriggerHitsTheDepthGuardInsteadOfLoopingForeverAndRollsBackCleanly() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TRIGGER loopy AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO t VALUES (NEW.id + 1); END") + val exception = assertFailsWith { run("INSERT INTO t VALUES (1)") } + assertTrue(exception.message!!.contains("recursion", ignoreCase = true), exception.message!!) + assertEquals(0, select("SELECT * FROM t").rows.size) + } + + @Test + fun dropTriggerStopsItFromFiringAndCannotBeDroppedTwice() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TABLE audit (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + run("DROP TRIGGER log") + run("INSERT INTO t VALUES (1)") + assertEquals(0, select("SELECT * FROM audit").rows.size) + assertFailsWith { run("DROP TRIGGER log") } + } + + @Test + fun droppingTheTableDropsItsTriggersToo() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN UPDATE t SET id = id; END") + run("DROP TABLE t") + assertFailsWith { run("DROP TRIGGER log") } + } + + @Test + fun renamingTheTableKeepsItsTriggersFiring() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TABLE audit (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + run("ALTER TABLE t RENAME TO t2") + run("INSERT INTO t2 VALUES (42)") + assertEquals(listOf(listOf(int(42))), select("SELECT * FROM audit").rows) + } + + @Test + fun rollbackUndoesCreateTrigger() { + run("CREATE TABLE t (id INTEGER)") + run("BEGIN") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + run("ROLLBACK") + assertFailsWith { run("DROP TRIGGER log") } + } + + @Test + fun createTriggerRejectsUnknownTable() { + assertFailsWith { + run("CREATE TRIGGER log AFTER INSERT ON missing FOR EACH ROW BEGIN DELETE FROM missing WHERE 1 = 0; END") + } + } + + @Test + fun createTriggerRejectsDuplicateName() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + assertFailsWith { + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsAnUnqualifiedColumn() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsNewOnADeleteTrigger() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER log BEFORE DELETE ON t FOR EACH ROW WHEN NEW.id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsOldOnAnInsertTrigger() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER log BEFORE INSERT ON t FOR EACH ROW WHEN OLD.id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseAcceptsBothSidesOnAnUpdateTrigger() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TRIGGER log BEFORE UPDATE ON t FOR EACH ROW WHEN NEW.id > OLD.id BEGIN DELETE FROM t WHERE 1 = 0; END") + } + + @Test + fun whenClauseRejectsAnUnknownColumn() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN NEW.nope > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsASubquery() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run( + "CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN EXISTS (SELECT 1 FROM t) " + + "BEGIN DELETE FROM t WHERE 1 = 0; END" + ) + } + } + + @Test + fun bodyStatementRejectsAnUnknownTargetTable() { + run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM missing WHERE 1 = 0; END") + } + } + + @Test + fun onConflictRowThatLandsWithNoConflictFiresInsertTriggers() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + run("CREATE TABLE audit (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + run("INSERT INTO t VALUES (1, 1) ON CONFLICT (id) DO NOTHING") + assertEquals(listOf(listOf(int(1))), select("SELECT * FROM audit").rows) + } + + @Test + fun onConflictDoNothingFiresNoTriggerAtAllForTheConflictingRow() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + run("INSERT INTO t VALUES (1, 1)") + run("CREATE TABLE audit (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + run("INSERT INTO t VALUES (1, 99) ON CONFLICT (id) DO NOTHING") + assertEquals(0, select("SELECT * FROM audit").rows.size) + } + + @Test + fun onConflictDoUpdateFiresUpdateTriggersNotInsertTriggers() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + run("CREATE TABLE inserts (id INTEGER)") + run("CREATE TABLE updates (id INTEGER, old_balance INTEGER, new_balance INTEGER)") + run("CREATE TRIGGER log_insert AFTER INSERT ON accounts FOR EACH ROW BEGIN INSERT INTO inserts VALUES (NEW.id); END") + run( + "CREATE TRIGGER log_update AFTER UPDATE ON accounts FOR EACH ROW " + + "BEGIN INSERT INTO updates VALUES (NEW.id, OLD.balance, NEW.balance); END" + ) + run("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + assertEquals(0, select("SELECT * FROM inserts").rows.size) + assertEquals(listOf(listOf(int(1), int(100), int(999))), select("SELECT * FROM updates").rows) + } + + @Test + fun onConflictBeforeUpdateTriggerCanRejectTheWholeStatement() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100)") + run("CREATE TABLE guard (x INTEGER NOT NULL)") + run("CREATE TRIGGER reject BEFORE UPDATE ON accounts FOR EACH ROW BEGIN INSERT INTO guard VALUES (NULL); END") + assertFailsWith { + run("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + } + assertEquals(int(100), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun onConflictMultiRowStatementFiresTheRightTriggerPerRow() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, hits INTEGER)") + run("INSERT INTO t VALUES (1, 10)") + run("CREATE TABLE inserts (id INTEGER)") + run("CREATE TABLE updates (id INTEGER)") + run("CREATE TRIGGER log_insert AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO inserts VALUES (NEW.id); END") + run("CREATE TRIGGER log_update AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO updates VALUES (NEW.id); END") + run( + """ + INSERT INTO t VALUES (1, 1), (2, 2) + ON CONFLICT (id) DO UPDATE SET hits = t.hits + excluded.hits + """.trimIndent() + ) + assertEquals(listOf(listOf(int(2))), select("SELECT * FROM inserts").rows) + assertEquals(listOf(listOf(int(1))), select("SELECT * FROM updates").rows) + } + + @Test + fun bodyStatementInAnInsertTriggerCanOnlySeeNewNotOld() { + run("CREATE TABLE t (id INTEGER)") + run("CREATE TABLE audit (id INTEGER)") + run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + run("INSERT INTO t VALUES (5)") + assertEquals(listOf(listOf(int(5))), select("SELECT * FROM audit").rows) + // OLD is simply not bound for an INSERT trigger, so an OLD.id reference in the body falls + // through exactly like any other unresolvable column in an INSERT's VALUES list would. + run("CREATE TRIGGER bad AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (OLD.id); END") + assertFailsWith { run("INSERT INTO t VALUES (6)") } + } +} diff --git a/src/test/kotlin/kestrel/TruncateTest.kt b/src/test/kotlin/kestrel/TruncateTest.kt new file mode 100644 index 0000000..5a9050b --- /dev/null +++ b/src/test/kotlin/kestrel/TruncateTest.kt @@ -0,0 +1,209 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.TruncateTableResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class TruncateTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + + @Test + fun truncateRemovesEveryRowButKeepsTheTableAndItsSchema() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT NOT NULL)") + run("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + val result = exec("TRUNCATE TABLE t") as TruncateTableResult + assertEquals(listOf("t"), result.tableNames) + assertEquals(emptyList(), select("SELECT * FROM t").rows) + run("INSERT INTO t VALUES (1, 'fresh')") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun tableKeywordIsOptional() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + exec("TRUNCATE t") + assertEquals(emptyList(), select("SELECT * FROM t").rows) + } + + @Test + fun multipleTablesInOneStatementAreAllTruncated() { + run("CREATE TABLE a (id INTEGER PRIMARY KEY)") + run("CREATE TABLE b (id INTEGER PRIMARY KEY)") + run("INSERT INTO a VALUES (1)") + run("INSERT INTO b VALUES (1)") + val result = exec("TRUNCATE TABLE a, b") as TruncateTableResult + assertEquals(setOf("a", "b"), result.tableNames.toSet()) + assertEquals(emptyList(), select("SELECT * FROM a").rows) + assertEquals(emptyList(), select("SELECT * FROM b").rows) + } + + @Test + fun truncatingAnUnknownTableFails() { + assertFailsWith { run("TRUNCATE TABLE nope") } + } + + @Test + fun truncatingAViewFailsWithAViewSpecificMessage() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE VIEW v AS SELECT * FROM t") + val error = assertFailsWith { run("TRUNCATE TABLE v") } + assertTrue(error.message!!.contains("view"), error.message) + } + + @Test + fun continueIdentityIsTheDefaultAndLeavesTheSequencePositionAlone() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + run("INSERT INTO t (label) VALUES ('a'), ('b')") + exec("TRUNCATE TABLE t") + run("INSERT INTO t (label) VALUES ('c')") + assertEquals(listOf(listOf(int(3))), select("SELECT id FROM t").rows) + } + + @Test + fun continueIdentityIsAlsoTheResultOfWritingItOutExplicitly() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + run("INSERT INTO t (label) VALUES ('a')") + exec("TRUNCATE TABLE t CONTINUE IDENTITY") + run("INSERT INTO t (label) VALUES ('b')") + assertEquals(listOf(listOf(int(2))), select("SELECT id FROM t").rows) + } + + @Test + fun restartIdentityResetsTheOwningSequenceBackToItsStartValue() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + run("INSERT INTO t (label) VALUES ('a'), ('b'), ('c')") + exec("TRUNCATE TABLE t RESTART IDENTITY") + run("INSERT INTO t (label) VALUES ('fresh')") + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM t").rows) + } + + @Test + fun restartIdentityIgnoresAColumnWhoseDefaultIsNotNextval() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT DEFAULT 'x')") + run("INSERT INTO t VALUES (1, 'a')") + exec("TRUNCATE TABLE t RESTART IDENTITY") + run("INSERT INTO t VALUES (2, DEFAULT)") + assertEquals(listOf(listOf(int(2))), select("SELECT id FROM t").rows) + } + + @Test + fun truncateIsRestrictedByDefaultWhenAnotherTableHasAForeignKeyIntoIt() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 1)") + val error = assertFailsWith { run("TRUNCATE TABLE departments") } + assertTrue(error.message!!.contains("employees"), error.message) + assertEquals(listOf(listOf(int(1))), select("SELECT id FROM departments").rows) + } + + @Test + fun truncateRestrictionAppliesEvenWhenTheForeignKeyDeclaresOnDeleteCascade() { + // TRUNCATE consults no per-row ON DELETE action at all - RESTRICT blocks regardless of it. + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE)") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + assertFailsWith { run("TRUNCATE TABLE departments") } + } + + @Test + fun truncatingBothTablesInTheSameStatementNeedsNoCascade() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + val result = exec("TRUNCATE TABLE departments, employees") as TruncateTableResult + assertEquals(setOf("departments", "employees"), result.tableNames.toSet()) + } + + @Test + fun truncateCascadeAlsoEmptiesEveryTransitiveReferencer() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + run("CREATE TABLE timesheets (id INTEGER PRIMARY KEY, employee_id INTEGER REFERENCES employees(id))") + run("INSERT INTO departments VALUES (1)") + run("INSERT INTO employees VALUES (1, 1)") + run("INSERT INTO timesheets VALUES (1, 1)") + val result = exec("TRUNCATE TABLE departments CASCADE") as TruncateTableResult + assertEquals(setOf("departments", "employees", "timesheets"), result.tableNames.toSet()) + assertEquals(emptyList(), select("SELECT * FROM timesheets").rows) + } + + @Test + fun selfReferencingForeignKeyDoesNotBlockTruncatingItsOwnTable() { + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, manager_id INTEGER REFERENCES employees(id))") + run("INSERT INTO employees VALUES (1, NULL), (2, 1)") + exec("TRUNCATE TABLE employees") + assertEquals(emptyList(), select("SELECT * FROM employees").rows) + } + + @Test + fun truncateNeverFiresRowLevelTriggersUnlikeAWhereLessDelete() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("CREATE TABLE audit (n INTEGER)") + run("INSERT INTO audit VALUES (0)") + run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN UPDATE audit SET n = n + 1; END") + run("INSERT INTO t VALUES (1), (2), (3)") + exec("TRUNCATE TABLE t") + assertEquals(listOf(listOf(int(0))), select("SELECT n FROM audit").rows) + assertEquals(emptyList(), select("SELECT * FROM t").rows) + } + + @Test + fun rollbackAfterTruncateRestoresTheRows() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1), (2)") + run("BEGIN") + exec("TRUNCATE TABLE t") + run("ROLLBACK") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), select("SELECT id FROM t").rows) + } + + @Test + fun rollbackAfterTruncateRestartIdentityAlsoRestoresTheSequencePosition() { + run("CREATE SEQUENCE seq") + run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + run("INSERT INTO t (label) VALUES ('a'), ('b')") + run("BEGIN") + exec("TRUNCATE TABLE t RESTART IDENTITY") + run("ROLLBACK") + // Both the rows (1, 2) AND the sequence's advanced position come back - if RESTART IDENTITY's + // reset had survived the ROLLBACK, this INSERT would collide with the just-restored id 1. + run("INSERT INTO t (label) VALUES ('c')") + assertEquals(listOf(int(1), int(2), int(3)), select("SELECT id FROM t").rows.map { it.single() }) + } + + @Test + fun truncateInsideATransactionCanStillBeCommitted() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + run("INSERT INTO t VALUES (1)") + run("BEGIN") + exec("TRUNCATE TABLE t") + run("COMMIT") + assertEquals(emptyList(), select("SELECT * FROM t").rows) + } +} diff --git a/src/test/kotlin/kestrel/UniqueConstraintTest.kt b/src/test/kotlin/kestrel/UniqueConstraintTest.kt new file mode 100644 index 0000000..3547261 --- /dev/null +++ b/src/test/kotlin/kestrel/UniqueConstraintTest.kt @@ -0,0 +1,157 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class UniqueConstraintTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + @Test + fun columnLevelUniqueRejectsDuplicateValue() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO users VALUES (1, 'ada@example.com')") + val error = assertFailsWith { + run("INSERT INTO users VALUES (2, 'ada@example.com')") + } + assertTrue(error.message!!.contains("UNIQUE constraint"), error.message!!) + } + + @Test + fun columnLevelUniqueAllowsMultipleNullValues() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO users VALUES (1, NULL)") + val result = exec("INSERT INTO users VALUES (2, NULL)") + assertEquals(InsertResult(1), result) + } + + @Test + fun tableLevelCompositeUniqueRejectsDuplicateCombination() { + run( + "CREATE TABLE enrollments (" + + "student_id INTEGER, course_id INTEGER, term TEXT, " + + "UNIQUE (student_id, course_id, term))" + ) + run("INSERT INTO enrollments VALUES (1, 10, 'fall')") + assertFailsWith { + run("INSERT INTO enrollments VALUES (1, 10, 'fall')") + } + } + + @Test + fun tableLevelCompositeUniqueAllowsDifferingCombinations() { + run( + "CREATE TABLE enrollments (" + + "student_id INTEGER, course_id INTEGER, term TEXT, " + + "UNIQUE (student_id, course_id, term))" + ) + run("INSERT INTO enrollments VALUES (1, 10, 'fall')") + val bySameStudentDifferentCourse = exec("INSERT INTO enrollments VALUES (1, 11, 'fall')") + assertEquals(InsertResult(1), bySameStudentDifferentCourse) + val bySameStudentAndCourseDifferentTerm = exec("INSERT INTO enrollments VALUES (1, 10, 'spring')") + assertEquals(InsertResult(1), bySameStudentAndCourseDifferentTerm) + } + + @Test + fun tableLevelCompositeUniqueExemptsRowsWithAnyNullColumn() { + run( + "CREATE TABLE enrollments (" + + "student_id INTEGER, course_id INTEGER, term TEXT, " + + "UNIQUE (student_id, course_id, term))" + ) + run("INSERT INTO enrollments VALUES (1, 10, NULL)") + val result = exec("INSERT INTO enrollments VALUES (1, 10, NULL)") + assertEquals(InsertResult(1), result) + } + + @Test + fun updateRejectsUniqueViolationAgainstAnotherRow() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO users VALUES (1, 'ada@example.com'), (2, 'grace@example.com')") + assertFailsWith { + run("UPDATE users SET email = 'ada@example.com' WHERE id = 2") + } + } + + @Test + fun updateToSameRowsOwnValueIsNotAViolation() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("INSERT INTO users VALUES (1, 'ada@example.com')") + val result = exec("UPDATE users SET email = 'ada@example.com' WHERE id = 1") + assertEquals(UpdateResult(1), result) + } + + @Test + fun uniqueConstraintRejectsUnknownColumn() { + assertFailsWith { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, UNIQUE (missing))") + } + } + + @Test + fun foreignKeyCanReferenceAUniqueNonPrimaryKeyColumn() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("CREATE TABLE profiles (id INTEGER PRIMARY KEY, user_email TEXT REFERENCES users(email))") + run("INSERT INTO users VALUES (1, 'ada@example.com')") + val result = exec("INSERT INTO profiles VALUES (1, 'ada@example.com')") + assertEquals(InsertResult(1), result) + assertFailsWith { + run("INSERT INTO profiles VALUES (2, 'nobody@example.com')") + } + } + + @Test + fun foreignKeyRejectsReferencingAColumnWithoutEnforcedUniqueness() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + val error = assertFailsWith { + run("CREATE TABLE profiles (id INTEGER PRIMARY KEY, user_email TEXT REFERENCES users(email))") + } + assertTrue(error.message!!.contains("PRIMARY KEY or UNIQUE"), error.message!!) + } + + @Test + fun cascadeUpdatePropagatesThroughAUniqueReferencedColumn() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run( + "CREATE TABLE profiles (id INTEGER PRIMARY KEY, " + + "user_email TEXT REFERENCES users(email) ON UPDATE CASCADE)" + ) + run("INSERT INTO users VALUES (1, 'old@example.com')") + run("INSERT INTO profiles VALUES (1, 'old@example.com')") + run("UPDATE users SET email = 'new@example.com' WHERE id = 1") + val result = select("SELECT user_email FROM profiles WHERE id = 1") + assertEquals(listOf(listOf(text("new@example.com"))), result.rows) + } + + @Test + fun rollbackAfterAFailedInsertLeavesUniqueStateReusable() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + run("BEGIN") + run("INSERT INTO users VALUES (1, 'ada@example.com')") + run("ROLLBACK") + val result = exec("INSERT INTO users VALUES (1, 'ada@example.com')") + assertEquals(InsertResult(1), result) + assertEquals(listOf(listOf(int(1), text("ada@example.com"))), select("SELECT id, email FROM users").rows) + } +} diff --git a/src/test/kotlin/kestrel/UpsertTest.kt b/src/test/kotlin/kestrel/UpsertTest.kt new file mode 100644 index 0000000..39cc28a --- /dev/null +++ b/src/test/kotlin/kestrel/UpsertTest.kt @@ -0,0 +1,234 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.Executor +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class UpsertTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun insertCount(sql: String): Int = (exec(sql) as InsertResult).count + + private fun seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER NOT NULL)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + } + + @Test + fun noConflictInsertsNormallyAndReportsOneRow() { + seed() + val count = insertCount("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (id) DO NOTHING") + assertEquals(1, count) + assertEquals(2, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictOnPrimaryKeyDoesNothingAndReportsZero() { + seed() + val count = insertCount("INSERT INTO accounts VALUES (1, 'ignored', 999) ON CONFLICT (id) DO NOTHING") + assertEquals(0, count) + assertEquals(Value.IntegerValue(100), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals("ada", (select("SELECT name FROM accounts WHERE id = 1").rows[0][0] as Value.TextValue).value) + } + + @Test + fun conflictWithBareOnConflictWithNoTargetCatchesAnyUniqueViolation() { + seed() + run("CREATE UNIQUE INDEX idxName ON accounts (name)") + val count = insertCount("INSERT INTO accounts VALUES (2, 'ada', 300) ON CONFLICT DO NOTHING") + assertEquals(0, count) + assertEquals(1, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictDoUpdateAppliesTheAssignments() { + seed() + insertCount("INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999") + assertEquals(Value.IntegerValue(999), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(1, select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictDoUpdateCanReferenceExcludedForTheProposedRow() { + seed() + insertCount("INSERT INTO accounts VALUES (1, 'ada', 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance") + assertEquals(Value.IntegerValue(150), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictDoUpdateWithFailingWhereBehavesLikeDoNothingForThatRow() { + seed() + val count = insertCount( + "INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999 WHERE excluded.balance < 0" + ) + assertEquals(0, count) + assertEquals(Value.IntegerValue(100), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictDoUpdateWithPassingWhereApplies() { + seed() + val count = insertCount( + "INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999 WHERE excluded.balance > 0" + ) + assertEquals(1, count) + assertEquals(Value.IntegerValue(999), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun multiRowInsertHandlesEachRowsConflictIndependently() { + seed() + val count = insertCount( + """ + INSERT INTO accounts VALUES (1, 'ada', 5), (2, 'linus', 200), (3, 'grace', 300) + ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance + """.trimIndent() + ) + assertEquals(3, count) + assertEquals(Value.IntegerValue(105), select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(Value.IntegerValue(200), select("SELECT balance FROM accounts WHERE id = 2").rows[0][0]) + assertEquals(Value.IntegerValue(300), select("SELECT balance FROM accounts WHERE id = 3").rows[0][0]) + } + + @Test + fun laterRowInTheSameStatementSeesAnEarlierRowsInsert() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, hits INTEGER NOT NULL)") + val count = insertCount( + """ + INSERT INTO t VALUES (1, 1), (1, 1) + ON CONFLICT (id) DO UPDATE SET hits = t.hits + excluded.hits + """.trimIndent() + ) + assertEquals(2, count) + assertEquals(Value.IntegerValue(2), select("SELECT hits FROM t WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictTargetMustMatchAnExistingUniqueOrPrimaryKey() { + seed() + val error = assertFailsWith { + run("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (name) DO NOTHING") + } + assertTrue(error.message!!.contains("no unique or primary key constraint"), error.message!!) + } + + @Test + fun conflictTargetMatchesADeclaredUniqueConstraintNotJustThePrimaryKey() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE, name TEXT)") + run("INSERT INTO users VALUES (1, 'ada@example.com', 'ada')") + val count = insertCount( + "INSERT INTO users VALUES (2, 'ada@example.com', 'ada2') ON CONFLICT (email) DO UPDATE SET name = excluded.name" + ) + assertEquals(1, count) + assertEquals("ada2", (select("SELECT name FROM users WHERE email = 'ada@example.com'").rows[0][0] as Value.TextValue).value) + assertEquals(1, select("SELECT * FROM users").rows.size) + } + + @Test + fun conflictOnADifferentUnrelatedUniqueColumnStillThrows() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE, tag TEXT UNIQUE)") + run("INSERT INTO users VALUES (1, 'ada@example.com', 'x')") + val error = assertFailsWith { + run("INSERT INTO users VALUES (2, 'new@example.com', 'x') ON CONFLICT (email) DO NOTHING") + } + assertTrue(error.message!!.contains("UNIQUE constraint"), error.message!!) + } + + @Test + fun conflictDoUpdateEnforcesNotNullOnTheResultingRow() { + seed() + val error = assertFailsWith { + run("INSERT INTO accounts VALUES (1, 'ada', 100) ON CONFLICT (id) DO UPDATE SET name = NULL") + } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + } + + @Test + fun conflictDoUpdateEnforcesCheckConstraintsOnTheResultingRow() { + run( + "CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER CHECK (balance >= 0))" + ) + run("INSERT INTO accounts VALUES (1, 100)") + val error = assertFailsWith { + run("INSERT INTO accounts VALUES (1, 100) ON CONFLICT (id) DO UPDATE SET balance = -5") + } + assertTrue(error.message!!.contains("CHECK constraint"), error.message!!) + } + + @Test + fun conflictDoUpdateRunsForeignKeyValidationOnTheUpdatedRow() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("INSERT INTO departments VALUES (1)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept INTEGER REFERENCES departments(id))" + ) + run("INSERT INTO employees VALUES (1, 1)") + val error = assertFailsWith { + run("INSERT INTO employees VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET dept = 999") + } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message!!) + } + + @Test + fun conflictDoUpdateCascadesToChildTablesWhenTheReferencedKeyChanges() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + run("INSERT INTO departments VALUES (1)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept INTEGER REFERENCES departments(id) ON UPDATE CASCADE)" + ) + run("INSERT INTO employees VALUES (1, 1)") + insertCount("INSERT INTO departments VALUES (1) ON CONFLICT (id) DO UPDATE SET id = 2") + assertEquals(Value.IntegerValue(2), select("SELECT dept FROM employees WHERE id = 1").rows[0][0]) + } + + @Test + fun compositePrimaryKeyConflictTargetIsRecognised() { + run( + "CREATE TABLE grades (student INTEGER, course INTEGER, score INTEGER, PRIMARY KEY (student, course))" + ) + run("INSERT INTO grades VALUES (1, 10, 50)") + val count = insertCount( + "INSERT INTO grades VALUES (1, 10, 90) ON CONFLICT (student, course) DO UPDATE SET score = excluded.score" + ) + assertEquals(1, count) + assertEquals(Value.IntegerValue(90), select("SELECT score FROM grades WHERE student = 1 AND course = 10").rows[0][0]) + } + + @Test + fun conflictDoUpdateDefaultKeywordFallsBackToColumnDefault() { + run("CREATE TABLE t (id INTEGER PRIMARY KEY, note TEXT DEFAULT 'fresh')") + run("INSERT INTO t VALUES (1, 'stale')") + insertCount("INSERT INTO t VALUES (1, 'stale') ON CONFLICT (id) DO UPDATE SET note = DEFAULT") + assertEquals("fresh", (select("SELECT note FROM t WHERE id = 1").rows[0][0] as Value.TextValue).value) + } + + @Test + fun tableStateAfterAConflictResolutionStillDumpsAndReplaysCleanly() { + seed() + run("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + val dumped = kestrel.persistence.SqlWriter.dump(catalog) + val replayCatalog = Catalog() + val replayExecutor = Executor(replayCatalog) + Parser(Lexer(dumped).tokenize()).parseProgram().forEach { replayExecutor.execute(it) } + val replaySelect = Parser(Lexer("SELECT * FROM accounts").tokenize()).parseProgram() + .map { replayExecutor.execute(it) }.single() as SelectResult + assertEquals(2, replaySelect.rows.size) + } +} diff --git a/src/test/kotlin/kestrel/ViewTest.kt b/src/test/kotlin/kestrel/ViewTest.kt new file mode 100644 index 0000000..a819437 --- /dev/null +++ b/src/test/kotlin/kestrel/ViewTest.kt @@ -0,0 +1,288 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.CreateViewResult +import kestrel.engine.DropViewResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class ViewTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun exec(sql: String): StatementResult = run(sql).single() + + private fun select(sql: String): SelectResult = exec(sql) as SelectResult + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private fun seedUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER, dept TEXT)") + run( + "INSERT INTO users VALUES " + + "(1, 'ada', 36, 'eng'), (2, 'linus', 54, 'eng'), (3, 'grace', 85, 'navy'), (4, 'alan', 41, NULL)" + ) + } + + @Test + fun createViewReturnsResultAndRegistersName() { + seedUsers() + val result = exec("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40") as CreateViewResult + assertEquals("adults", result.viewName) + assertEquals(listOf("adults"), catalog.viewNames) + } + + @Test + fun queryingAViewRunsItsDefiningQueryFresh() { + seedUsers() + run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40") + val first = select("SELECT name FROM adults ORDER BY id ASC") + assertEquals(listOf(text("linus"), text("grace"), text("alan")), first.rows.map { it.single() }) + + run("INSERT INTO users VALUES (5, 'margaret', 61, 'navy')") + val second = select("SELECT name FROM adults ORDER BY id ASC") + assertEquals( + listOf(text("linus"), text("grace"), text("alan"), text("margaret")), + second.rows.map { it.single() } + ) + } + + @Test + fun viewCanBeUsedAsAJoinTarget() { + seedUsers() + run("CREATE TABLE bonuses (dept TEXT, amount INTEGER)") + run("INSERT INTO bonuses VALUES ('eng', 1000), ('navy', 500)") + run("CREATE VIEW eng_staff AS SELECT id, name, dept FROM users WHERE dept = 'eng'") + val result = select( + "SELECT eng_staff.name, bonuses.amount FROM eng_staff " + + "JOIN bonuses ON eng_staff.dept = bonuses.dept ORDER BY eng_staff.id ASC" + ) + assertEquals( + listOf(listOf(text("ada"), int(1000)), listOf(text("linus"), int(1000))), + result.rows + ) + } + + @Test + fun viewOverAggregateQueryWorks() { + seedUsers() + run("CREATE VIEW dept_counts AS SELECT dept, COUNT(*) AS n FROM users GROUP BY dept") + val result = select("SELECT dept, n FROM dept_counts WHERE n > 1 ORDER BY dept ASC") + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } + + @Test + fun viewDefinitionCanReferenceAnotherView() { + seedUsers() + run("CREATE VIEW adults AS SELECT id, name, age FROM users WHERE age >= 40") + run("CREATE VIEW seniors AS SELECT id, name FROM adults WHERE age >= 60") + val result = select("SELECT name FROM seniors") + assertEquals(listOf(listOf(text("grace"))), result.rows) + } + + @Test + fun creatingAViewValidatesColumnsEagerly() { + seedUsers() + assertFailsWith { run("CREATE VIEW broken AS SELECT nope FROM users") } + assertTrue(catalog.viewNames.isEmpty()) + } + + @Test + fun creatingAViewOverAnUnknownTableFails() { + assertFailsWith { run("CREATE VIEW v AS SELECT * FROM missing") } + } + + @Test + fun duplicateViewNameFailsWithoutOrReplace() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 21") } + } + + @Test + fun createOrReplaceViewSwapsTheDefinition() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("CREATE OR REPLACE VIEW adults AS SELECT id FROM users WHERE age >= 50") + val result = select("SELECT id FROM adults ORDER BY id ASC") + assertEquals(listOf(listOf(int(2)), listOf(int(3))), result.rows) + } + + @Test + fun dropViewRemovesItAndItStopsBeingQueryable() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + val result = exec("DROP VIEW adults") as DropViewResult + assertEquals("adults", result.viewName) + assertFailsWith { run("SELECT * FROM adults") } + } + + @Test + fun droppingAnUnknownViewFails() { + assertFailsWith { run("DROP VIEW nope") } + } + + @Test + fun cannotCreateTableWithSameNameAsAnExistingView() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { run("CREATE TABLE adults (id INTEGER)") } + } + + @Test + fun cannotCreateViewWithSameNameAsAnExistingTable() { + seedUsers() + assertFailsWith { run("CREATE VIEW users AS SELECT id FROM users") } + } + + @Test + fun dropTableRefusesAViewNameAndDropViewRefusesATableName() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { run("DROP TABLE adults") } + assertFailsWith { run("DROP VIEW users") } + } + + @Test + fun insertUpdateDeleteAgainstAViewAreRejected() { + seedUsers() + run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") + assertFailsWith { run("INSERT INTO adults VALUES (9, 'x')") } + assertFailsWith { run("UPDATE adults SET name = 'x'") } + assertFailsWith { run("DELETE FROM adults") } + } + + @Test + fun creatingAnIndexOnAViewIsRejected() { + seedUsers() + run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") + assertFailsWith { run("CREATE INDEX idx ON adults (id)") } + } + + @Test + fun mutuallyRecursiveViewsFailInsteadOfOverflowingTheStack() { + seedUsers() + run("CREATE VIEW a AS SELECT id FROM users") + run("CREATE VIEW b AS SELECT id FROM a") + run("CREATE OR REPLACE VIEW a AS SELECT id FROM b") + assertFailsWith { run("SELECT * FROM a") } + } + + @Test + fun selfReferencingViewFailsCleanly() { + seedUsers() + run("CREATE VIEW a AS SELECT id FROM users") + run("CREATE OR REPLACE VIEW a AS SELECT id FROM a") + assertFailsWith { run("SELECT * FROM a") } + } + + @Test + fun rollbackUndoesCreateViewAndDropView() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("BEGIN") + run("CREATE VIEW seniors AS SELECT id FROM users WHERE age >= 60") + run("DROP VIEW adults") + run("ROLLBACK") + assertEquals(listOf("adults"), catalog.viewNames) + } + + @Test + fun commitKeepsViewChanges() { + seedUsers() + run("BEGIN") + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("COMMIT") + assertEquals(listOf("adults"), catalog.viewNames) + } + + @Test + fun dropTableWithADependentViewFailsByDefaultAndAlsoWithExplicitRestrict() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { run("DROP TABLE users") } + assertFailsWith { run("DROP TABLE users RESTRICT") } + assertEquals(listOf("users"), catalog.tableNames) + assertEquals(listOf("adults"), catalog.viewNames) + } + + @Test + fun dropTableCascadeDropsTheTableAndItsDependentView() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("DROP TABLE users CASCADE") + assertTrue(catalog.tableNames.isEmpty()) + assertTrue(catalog.viewNames.isEmpty()) + } + + @Test + fun dropViewWithADependentViewFailsByDefault() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("CREATE VIEW seniorAdults AS SELECT id FROM adults WHERE id > 0") + assertFailsWith { run("DROP VIEW adults") } + assertEquals(listOf("adults", "seniorAdults"), catalog.viewNames) + } + + @Test + fun dropViewCascadeDropsATransitiveChainOfDependentViews() { + seedUsers() + run("CREATE VIEW a AS SELECT id FROM users WHERE age >= 18") + run("CREATE VIEW b AS SELECT id FROM a WHERE id > 0") + run("CREATE VIEW c AS SELECT id FROM b WHERE id > 0") + run("DROP VIEW a CASCADE") + assertTrue(catalog.viewNames.isEmpty()) + } + + @Test + fun dropTableCascadeHandlesADiamondOfDependentViewsWithoutFailing() { + seedUsers() + run("CREATE VIEW b AS SELECT id FROM users WHERE age >= 18") + run("CREATE VIEW c AS SELECT id FROM users WHERE age >= 21") + run("CREATE VIEW d AS SELECT b.id FROM b JOIN c ON b.id = c.id") + run("DROP TABLE users CASCADE") + assertTrue(catalog.tableNames.isEmpty()) + assertTrue(catalog.viewNames.isEmpty()) + } + + @Test + fun rollbackUndoesACascadeDrop() { + seedUsers() + run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + run("BEGIN") + run("DROP TABLE users CASCADE") + run("ROLLBACK") + assertEquals(listOf("users"), catalog.tableNames) + assertEquals(listOf("adults"), catalog.viewNames) + val result = select("SELECT id FROM adults ORDER BY id ASC") + assertTrue(result.rows.isNotEmpty()) + } + + @Test + fun explainOverAViewDescribesTheOuterQuery() { + seedUsers() + run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") + val explain = exec("EXPLAIN SELECT * FROM adults") as kestrel.engine.ExplainResult + assertTrue(explain.plan.any { it.contains("adults") }) + } + + @Test + fun columnTypeOfAViewIsInferredFromActualData() { + seedUsers() + run("CREATE VIEW names AS SELECT name FROM users") + val result = select("SELECT name FROM names WHERE name = 'ada'") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } +} diff --git a/src/test/kotlin/kestrel/WindowFunctionTest.kt b/src/test/kotlin/kestrel/WindowFunctionTest.kt new file mode 100644 index 0000000..0096c8c --- /dev/null +++ b/src/test/kotlin/kestrel/WindowFunctionTest.kt @@ -0,0 +1,937 @@ +package kestrel + +import kestrel.catalog.Catalog +import kestrel.engine.ExplainResult +import kestrel.engine.Executor +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class WindowFunctionTest { + private val catalog = Catalog() + private val executor = Executor(catalog) + + private fun run(sql: String): List = + Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + + private fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + private fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + + private fun text(value: String) = Value.TextValue(value) + private fun int(value: Long) = Value.IntegerValue(value) + + /** + * eng (deptId 1): ada/linus tie at 400 (ada inserted first), grace at 300. + * sales (deptId 2): sam/kay tie at 200 (sam inserted first), meg at 100. + * Salaries also happen to be in strict descending order by id across the whole table, which + * keeps the no-PARTITION-BY tests below easy to eyeball. + */ + private fun seed() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'sales')") + run( + "INSERT INTO employees VALUES " + + "(1, 1, 'ada', 400), (2, 1, 'linus', 400), (3, 1, 'grace', 300), " + + "(4, 2, 'sam', 200), (5, 2, 'kay', 200), (6, 2, 'meg', 100)" + ) + } + + @Test + fun rowNumberAssignsSequentialNumbersWithinEachPartition() { + seed() + val result = select( + "SELECT id, ROW_NUMBER() OVER (PARTITION BY deptId ORDER BY salary DESC) AS rn " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(2)), listOf(int(3), int(3)), + listOf(int(4), int(1)), listOf(int(5), int(2)), listOf(int(6), int(3)) + ), + result.rows + ) + } + + @Test + fun rowNumberWithoutOverOrderByFallsBackToPartitionArrivalOrder() { + seed() + val result = select("SELECT id, ROW_NUMBER() OVER (PARTITION BY deptId) AS rn FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(2)), listOf(int(3), int(3)), + listOf(int(4), int(1)), listOf(int(5), int(2)), listOf(int(6), int(3)) + ), + result.rows + ) + } + + @Test + fun rankLeavesGapsAfterTiesButDenseRankDoesNot() { + seed() + val result = select( + "SELECT id, RANK() OVER (PARTITION BY deptId ORDER BY salary DESC) AS r, " + + "DENSE_RANK() OVER (PARTITION BY deptId ORDER BY salary DESC) AS dr " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1), int(1)), + listOf(int(2), int(1), int(1)), + listOf(int(3), int(3), int(2)), + listOf(int(4), int(1), int(1)), + listOf(int(5), int(1), int(1)), + listOf(int(6), int(3), int(2)) + ), + result.rows + ) + } + + @Test + fun rankWithoutPartitionOrdersTheWholeResultSet() { + seed() + val result = select("SELECT id, RANK() OVER (ORDER BY salary DESC) AS r FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(1)), listOf(int(3), int(3)), + listOf(int(4), int(4)), listOf(int(5), int(4)), listOf(int(6), int(6)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithoutOrderByRepeatsTheWholePartitionTotalOnEveryRow() { + seed() + val result = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId) AS deptTotal FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1100)), listOf(int(2), int(1100)), listOf(int(3), int(1100)), + listOf(int(4), int(500)), listOf(int(5), int(500)), listOf(int(6), int(500)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithOrderByRunsCumulativelyAndTiesSharePeerGroupTotal() { + seed() + val result = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS running " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(800)), listOf(int(2), int(800)), listOf(int(3), int(1100)), + listOf(int(4), int(400)), listOf(int(5), int(400)), listOf(int(6), int(500)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithUniqueOrderKeyProducesARunningTotal() { + seed() + val result = select("SELECT id, SUM(salary) OVER (ORDER BY id) AS running FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(400)), listOf(int(2), int(800)), listOf(int(3), int(1100)), + listOf(int(4), int(1300)), listOf(int(5), int(1500)), listOf(int(6), int(1600)) + ), + result.rows + ) + } + + @Test + fun countStarOverPartitionCountsRowsRegardlessOfNulls() { + seed() + run("INSERT INTO employees VALUES (7, 3, 'unpaid', NULL)") + val result = select( + "SELECT COUNT(*) OVER (PARTITION BY deptId) AS n, COUNT(salary) OVER (PARTITION BY deptId) AS paid " + + "FROM employees WHERE deptId = 3" + ) + assertEquals(listOf(listOf(int(1), int(0))), result.rows) + } + + @Test + fun windowFunctionOverAJoinPartitionsByTheJoinedColumn() { + seed() + val result = select( + "SELECT e.name, d.name AS dept, " + + "ROW_NUMBER() OVER (PARTITION BY d.name ORDER BY e.salary DESC) AS rn " + + "FROM employees e JOIN departments d ON e.deptId = d.id " + + "WHERE d.name = 'eng' ORDER BY rn ASC" + ) + assertEquals( + listOf( + listOf(text("ada"), text("eng"), int(1)), + listOf(text("linus"), text("eng"), int(2)), + listOf(text("grace"), text("eng"), int(3)) + ), + result.rows + ) + } + + @Test + fun orderByCanReferenceAWindowFunctionByItsProjectionAlias() { + seed() + val result = select( + "SELECT id, RANK() OVER (ORDER BY salary DESC) AS r FROM employees ORDER BY r ASC, id ASC" + ) + assertEquals((1..6).map { it.toLong() }, result.rows.map { (it[0] as Value.IntegerValue).value }) + } + + @Test + fun windowFunctionIsRejectedInWhere() { + seed() + val error = assertFailsWith { + run("SELECT id FROM employees WHERE ROW_NUMBER() OVER (ORDER BY id) = 1") + } + assertTrue(error.message!!.contains("WHERE"), error.message!!) + } + + @Test + fun windowFunctionIsRejectedInGroupBy() { + seed() + assertFailsWith { + run("SELECT COUNT(*) FROM employees GROUP BY ROW_NUMBER() OVER (ORDER BY id)") + } + } + + @Test + fun windowFunctionIsRejectedInHaving() { + seed() + assertFailsWith { + run("SELECT deptId, COUNT(*) FROM employees GROUP BY deptId HAVING ROW_NUMBER() OVER (ORDER BY deptId) > 1") + } + } + + @Test + fun windowFunctionCannotBeCombinedWithGroupByInTheSameQuery() { + seed() + val error = assertFailsWith { + run("SELECT deptId, COUNT(*), ROW_NUMBER() OVER (ORDER BY deptId) FROM employees GROUP BY deptId") + } + assertTrue(error.message!!.contains("combined"), error.message!!) + } + + @Test + fun rankRequiresAnOrderByInsideOver() { + seed() + val error = assertFailsWith { + run("SELECT RANK() OVER (PARTITION BY deptId) FROM employees") + } + assertTrue(error.message!!.contains("ORDER BY"), error.message!!) + } + + @Test + fun denseRankRequiresAnOrderByInsideOver() { + seed() + assertFailsWith { + run("SELECT DENSE_RANK() OVER (PARTITION BY deptId) FROM employees") + } + } + + @Test + fun nestedWindowFunctionsAreRejected() { + seed() + val error = assertFailsWith { + run("SELECT SUM(salary) OVER (ORDER BY ROW_NUMBER() OVER (ORDER BY id)) FROM employees") + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun explainDoesNotFailOnAQueryWithAWindowFunction() { + seed() + val plan = explain("EXPLAIN SELECT id, ROW_NUMBER() OVER (PARTITION BY deptId ORDER BY salary DESC) FROM employees") + assertTrue(plan.isNotEmpty()) + } + + @Test + fun selectStarWithAWindowFunctionOnlyInOrderByDoesNotLeakTheExtraColumn() { + seed() + val result = select("SELECT * FROM departments ORDER BY ROW_NUMBER() OVER (ORDER BY id) DESC") + assertEquals(listOf("id", "name"), result.columns) + assertEquals(listOf(listOf(int(2), text("sales")), listOf(int(1), text("eng"))), result.rows) + } + + @Test + fun lagReturnsThePreviousRowsValueWithinEachPartitionAndNullAtTheEdge() { + seed() + val result = select( + "SELECT id, LAG(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS prev " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(null, 400L, 400L, null, 200L, 200L), + result.rows.map { (it[1] as? Value.IntegerValue)?.value } + ) + } + + @Test + fun leadReturnsTheNextRowsValueWithinEachPartitionAndNullAtTheEdge() { + seed() + val result = select( + "SELECT id, LEAD(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS next " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(400L, 300L, null, 200L, 100L, null), + result.rows.map { (it[1] as? Value.IntegerValue)?.value } + ) + } + + @Test + fun lagWithAnExplicitOffsetSkipsMultipleRows() { + seed() + val result = select( + "SELECT id, LAG(salary, 2) OVER (PARTITION BY deptId ORDER BY salary DESC) AS prev2 " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(null, null, 400L, null, null, 200L), + result.rows.map { (it[1] as? Value.IntegerValue)?.value } + ) + } + + @Test + fun lagPastThePartitionEdgeFallsBackToTheDefaultExpressionInsteadOfNull() { + seed() + val result = select( + "SELECT id, LAG(salary, 1, 0) OVER (PARTITION BY deptId ORDER BY salary DESC) AS prev " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(0L, 400L, 400L, 0L, 200L, 200L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun lagOffsetMustNotBeNegative() { + seed() + val error = assertFailsWith { + run("SELECT LAG(salary, -1) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun lagOffsetMustBeAnInteger() { + seed() + val error = assertFailsWith { + run("SELECT LAG(salary, 1.5) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun firstValueReturnsTheFirstRowsValueForEveryRowInThePartition() { + seed() + val result = select( + "SELECT id, FIRST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS top " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(400L, 400L, 400L, 200L, 200L, 200L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun lastValueSharesOneValuePerPeerGroupRatherThanTheWholePartition() { + seed() + val result = select( + "SELECT id, LAST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS lv " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(400L, 400L, 300L, 200L, 200L, 100L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun lastValueWithoutOverOrderByUsesTheWholePartitionsLastRow() { + seed() + val result = select( + "SELECT id, LAST_VALUE(salary) OVER (PARTITION BY deptId) AS lv FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(300L, 300L, 300L, 100L, 100L, 100L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileSplitsEachPartitionIntoAsEvenBucketsAsPossible() { + seed() + val result = select( + "SELECT id, NTILE(2) OVER (PARTITION BY deptId ORDER BY salary DESC) AS bucket " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(1L, 1L, 2L, 1L, 1L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileWithMoreBucketsThanRowsLeavesTheHighestBucketsUnused() { + seed() + val result = select( + "SELECT id, NTILE(5) OVER (PARTITION BY deptId ORDER BY salary DESC) AS bucket " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(1L, 2L, 3L, 1L, 2L, 3L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileBucketCountMustBePositive() { + seed() + val error = assertFailsWith { + run("SELECT NTILE(0) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("positive"), error.message!!) + } + + @Test + fun ntileBucketCountMustBeAnInteger() { + seed() + val error = assertFailsWith { + run("SELECT NTILE(1.5) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun windowFunctionNestedInsideLagsOffsetIsRejected() { + seed() + val error = assertFailsWith { + run("SELECT LAG(salary, ROW_NUMBER() OVER (ORDER BY id)) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun explicitRowsFrameProducesAMovingAverageInsteadOfARunningTotal() { + seed() + val result = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(400L, 800L, 700L, 200L, 400L, 300L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun rowsBetweenCurrentRowAndFollowingLooksForward() { + seed() + val result = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf(800L, 700L, 300L, 400L, 300L, 100L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + @Test + fun rowsBetweenUnboundedPrecedingAndCurrentRowMatchesTheImplicitDefaultOnAUniqueOrderKey() { + seed() + val explicit = select( + "SELECT id, SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + val implicit = select("SELECT id, SUM(salary) OVER (ORDER BY id) AS s FROM employees ORDER BY id ASC") + assertEquals(implicit.rows, explicit.rows) + } + + @Test + fun frameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + seed() + val result = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 5 FOLLOWING AND 9 FOLLOWING) AS s, " + + "COUNT(*) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 5 FOLLOWING AND 9 FOLLOWING) AS c " + + "FROM employees WHERE deptId = 1 ORDER BY id ASC" + ) + assertEquals( + listOf(listOf(int(1), Value.NullValue, int(0)), listOf(int(2), Value.NullValue, int(0)), listOf(int(3), Value.NullValue, int(0))), + result.rows + ) + } + + @Test + fun firstValueAndLastValueRespectAnExplicitFrameInsteadOfThePeerGroupDefault() { + seed() + val result = select( + "SELECT id, " + + "FIRST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS f, " + + "LAST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS l " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(400), int(400)), + listOf(int(2), int(400), int(300)), + listOf(int(3), int(400), int(300)), + listOf(int(4), int(200), int(200)), + listOf(int(5), int(200), int(100)), + listOf(int(6), int(200), int(100)) + ), + result.rows + ) + } + + @Test + fun rowsFrameShorthandWithoutBetweenEndsAtCurrentRow() { + seed() + val explicit = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS 1 PRECEDING) AS s " + + "FROM employees ORDER BY id ASC" + ) + val spelledOut = select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(spelledOut.rows, explicit.rows) + } + + @Test + fun rowsFrameOffsetMustNotBeNegative() { + seed() + val error = assertFailsWith { + run("SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN -1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun rowsFrameOffsetMustBeAnInteger() { + seed() + val error = assertFailsWith { + run("SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN 1.5 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun rowsFrameIsRejectedOnRankingAndOffsetWindowFunctions() { + seed() + val error = assertFailsWith { + run("SELECT ROW_NUMBER() OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("frame"), error.message!!) + } + + @Test + fun windowFunctionNestedInsideAFrameOffsetIsRejected() { + seed() + val error = assertFailsWith { + run( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN ROW_NUMBER() OVER (ORDER BY id) PRECEDING AND CURRENT ROW) " + + "FROM employees" + ) + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun explicitFrameSurvivesViewDefinitionRoundTrip() { + seed() + run( + "CREATE VIEW movingTotals AS SELECT id, " + + "SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees" + ) + val result = select("SELECT id, s FROM movingTotals ORDER BY id ASC") + assertEquals( + listOf(400L, 800L, 700L, 200L, 400L, 300L), + result.rows.map { (it[1] as Value.IntegerValue).value } + ) + } + + /** amount is monotonic in id (10, 20, 20, 30, 50) so `ORDER BY id` and `ORDER BY amount` agree, with a tie at 20. */ + private fun seedSales() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20), (3, 20), (4, 30), (5, 50)") + } + + @Test + fun rangeBetweenPrecedingAndFollowingIncludesRowsWithinValueDistanceRegardlessOfPhysicalRowCount() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(50L, 80L, 80L, 70L, 50L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameGivesTheSameRowSetRegardlessOfOrderDirectionForASymmetricWindow() { + seedSales() + val ascending = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ASC RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + val descending = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount DESC RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(ascending.rows, descending.rows) + } + + @Test + fun rangeBetweenPrecedingAndFollowingOnADescendingOrderByFlipsWhichDirectionIsPreceding() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount DESC RANGE BETWEEN 15 PRECEDING AND 5 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(50L, 70L, 70L, 30L, 50L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeCurrentRowExpandsToTheWholePeerGroupUnlikeRowsCurrentRow() { + seedSales() + val range = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM sales ORDER BY id ASC" + ) + val rows = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(10L, 50L, 50L, 80L, 130L), range.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(listOf(10L, 30L, 50L, 80L, 130L), rows.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameMeasuresDistanceOnADecimalColumn() { + run("CREATE TABLE prices (id INTEGER PRIMARY KEY, price DECIMAL(10, 2))") + run("INSERT INTO prices VALUES (1, 1.00), (2, 1.40), (3, 1.50), (4, 2.00)") + val result = select( + "SELECT id, COUNT(*) OVER (ORDER BY price RANGE BETWEEN 0.5 PRECEDING AND 0.5 FOLLOWING) AS c " + + "FROM prices ORDER BY id ASC" + ) + assertEquals(listOf(3L, 3L, 4L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameMeasuresDistanceInDaysOnADateColumn() { + run("CREATE TABLE events (id INTEGER PRIMARY KEY, day DATE)") + run( + "INSERT INTO events VALUES " + + "(1, CAST('2024-01-01' AS DATE)), (2, CAST('2024-01-02' AS DATE)), " + + "(3, CAST('2024-01-05' AS DATE)), (4, CAST('2024-01-06' AS DATE)), (5, CAST('2024-01-10' AS DATE))" + ) + val result = select( + "SELECT id, COUNT(*) OVER (ORDER BY day RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS c " + + "FROM events ORDER BY id ASC" + ) + assertEquals(listOf(2L, 2L, 2L, 2L, 1L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS s, " + + "COUNT(*) OVER (ORDER BY amount RANGE BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS c " + + "FROM sales ORDER BY id ASC" + ) + assertEquals( + (1L..5L).map { listOf(int(it), Value.NullValue, int(0)) }, + result.rows + ) + } + + @Test + fun rangeFrameWithAPrecedingBoundRequiresExactlyOneOrderByColumn() { + seedSales() + val error = assertFailsWith { + run( + "SELECT SUM(amount) OVER (ORDER BY amount, id RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) " + + "FROM sales" + ) + } + assertTrue(error.message!!.contains("exactly one ORDER BY column"), error.message!!) + } + + @Test + fun rangeFrameWithAPrecedingBoundRequiresANumericOrDateOrderColumn() { + seed() + val error = assertFailsWith { + run("SELECT SUM(salary) OVER (ORDER BY name RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("numeric or DATE"), error.message!!) + } + + @Test + fun rangeFrameOffsetMustBeNumeric() { + seedSales() + val error = assertFailsWith { + run("SELECT SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 'x' PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("numeric"), error.message!!) + } + + @Test + fun rangeFrameOffsetMustNotBeNegative() { + seedSales() + val error = assertFailsWith { + run("SELECT SUM(amount) OVER (ORDER BY amount RANGE BETWEEN -1 PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun rangeFrameSurvivesViewDefinitionRoundTrip() { + seedSales() + run( + "CREATE VIEW valueTotals AS SELECT id, " + + "SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s " + + "FROM sales" + ) + val result = select("SELECT id, s FROM valueTotals ORDER BY id ASC") + assertEquals(listOf(50L, 80L, 80L, 70L, 50L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeCurrentRowRemovesOnlyTheCurrentRowFromTheWholePartitionFrame() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(120L, 110L, 110L, 100L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeGroupRemovesTheWholePeerGroupIncludingTheCurrentRow() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(120L, 90L, 90L, 100L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeTiesRemovesOnlyTheOtherTiedRowsAndKeepsTheCurrentRow() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE TIES" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(130L, 110L, 110L, 130L, 130L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeCurrentRowInteractsWithABoundedFrameInsteadOfOnlyTheWholePartition() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(20L, 30L, 50L, 70L, 30L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeNoOthersExplicitlyWrittenOutBehavesLikeNoExcludeClauseAtAll() { + seedSales() + val explicit = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE NO OTHERS" + + ") AS s FROM sales ORDER BY id ASC" + ) + val implicit = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(implicit.rows, explicit.rows) + } + + @Test + fun excludeGroupWithNoOrderByTreatsTheWholePartitionAsOnePeerGroup() { + seedSales() + val result = select( + "SELECT id, COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP) AS c " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(0L, 0L, 0L, 0L, 0L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeClauseSurvivesViewDefinitionRoundTrip() { + seedSales() + run( + "CREATE VIEW excludingSelf AS SELECT id, " + + "SUM(amount) OVER (ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW) AS s " + + "FROM sales" + ) + val result = select("SELECT id, s FROM excludingSelf ORDER BY id ASC") + assertEquals(listOf(120L, 110L, 110L, 100L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + // ---- GROUPS frame unit ---- + // sales (ordered by amount): id1=10, {id2=20, id3=20} tied, id4=30, id5=50 -> 4 distinct peer + // groups. GROUPS counts THESE groups, unlike ROWS (physical rows) or RANGE (value distance). + + @Test + fun groupsBetweenPrecedingAndFollowingCountsPeerGroupsNotPhysicalRows() { + seedSales() + val groups = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + val rows = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + // id2/id3's tied peer group makes the GROUPS frame span more physical rows than ROWS for + // their neighbours (id1 and id4), even though both ask for "1 PRECEDING AND 1 FOLLOWING". + assertEquals(listOf(50L, 80L, 80L, 120L, 80L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(listOf(30L, 50L, 70L, 100L, 80L), rows.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsFrameBehavesExactlyLikeRowsFrameWhenNoRowsAreTied() { + run("CREATE TABLE prices (id INTEGER PRIMARY KEY, price DECIMAL(10, 2))") + run("INSERT INTO prices VALUES (1, 1.00), (2, 1.40), (3, 1.50), (4, 2.00)") + val groups = select( + "SELECT id, COUNT(*) OVER (ORDER BY price GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS c " + + "FROM prices ORDER BY id ASC" + ) + val rows = select( + "SELECT id, COUNT(*) OVER (ORDER BY price ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS c " + + "FROM prices ORDER BY id ASC" + ) + assertEquals(rows.rows, groups.rows) + } + + @Test + fun groupsCurrentRowExpandsToTheWholePeerGroupJustLikeRangeCurrentRow() { + seedSales() + val groups = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM sales ORDER BY id ASC" + ) + val range = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM sales ORDER BY id ASC" + ) + val rows = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(10L, 50L, 50L, 80L, 130L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(groups.rows, range.rows) + assertEquals(listOf(10L, 30L, 50L, 80L, 130L), rows.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsFrameWorksWithAMultiColumnNonNumericOrderByUnlikeRange() { + seed() + val result = select( + "SELECT id, COUNT(*) OVER (ORDER BY deptId, name GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS c " + + "FROM employees ORDER BY deptId ASC, name ASC" + ) + // ada, grace, linus (dept 1) then kay, meg, sam (dept 2) - every row is its own peer group + // since no two rows share the same (deptId, name), so this is exactly the sliding-window-of-3 + // pattern (2 at each edge, 3 in the middle). + assertEquals( + listOf(int(1), int(3), int(2), int(5), int(6), int(4)), + result.rows.map { it[0] } + ) + assertEquals(listOf(2L, 3L, 3L, 3L, 3L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsFrameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + seedSales() + val result = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS s, " + + "COUNT(*) OVER (ORDER BY amount GROUPS BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS c " + + "FROM sales ORDER BY id ASC" + ) + assertEquals((1L..5L).map { listOf(int(it), Value.NullValue, int(0)) }, result.rows) + } + + @Test + fun groupsFrameOffsetMustBeAnInteger() { + seedSales() + val error = assertFailsWith { + run("SELECT SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 1.5 PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun groupsFrameOffsetMustNotBeNegative() { + seedSales() + val error = assertFailsWith { + run("SELECT SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN -1 PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun groupsFrameWithUnboundedPrecedingAndFollowingCoversTheWholePartitionJustLikeRowsAndRange() { + seedSales() + val groups = select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS s " + + "FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(130L, 130L, 130L, 130L, 130L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsFrameIsRejectedOnRankingAndOffsetWindowFunctions() { + seed() + val error = assertFailsWith { + run("SELECT ROW_NUMBER() OVER (ORDER BY id GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("GROUPS"), error.message!!) + } + + @Test + fun excludeGroupCombinesWithAGroupsFrameJustLikeItDoesWithRange() { + seedSales() + val groups = select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(120L, 90L, 90L, 100L, 80L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun nestedWindowFunctionInAGroupsFrameOffsetIsRejected() { + seedSales() + val error = assertFailsWith { + run( + "SELECT SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN ROW_NUMBER() OVER (ORDER BY amount) PRECEDING AND CURRENT ROW) " + + "FROM sales" + ) + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun groupsFrameSurvivesViewDefinitionRoundTrip() { + seedSales() + run( + "CREATE VIEW groupTotals AS SELECT id, " + + "SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s " + + "FROM sales" + ) + val result = select("SELECT id, s FROM groupTotals ORDER BY id ASC") + assertEquals(listOf(50L, 80L, 80L, 120L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } +} diff --git a/src/test/kotlin/kestrel/store/btree/BPlusTreeTest.kt b/src/test/kotlin/kestrel/store/btree/BPlusTreeTest.kt new file mode 100644 index 0000000..dedd792 --- /dev/null +++ b/src/test/kotlin/kestrel/store/btree/BPlusTreeTest.kt @@ -0,0 +1,184 @@ +package kestrel.store.btree + +import kestrel.store.codec.KeyCodec +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import kestrel.storage.Value +import java.nio.file.Files +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class BPlusTreeTest { + private fun newPool(capacity: Int = 64): BufferPool { + val file = Files.createTempFile("kestrel-btree", ".db").toFile().also { it.deleteOnExit() } + return BufferPool(DiskManager(file), capacity) + } + + private fun key(value: Long): ByteArray = KeyCodec.encode(Value.IntegerValue(value)) + + private fun decodeKey(bytes: ByteArray): Long = (KeyCodec.decodeKey(bytes)[0] as Value.IntegerValue).value + + @Test + fun searchOnEmptyTreeReturnsNull() { + val tree = BPlusTree.create(newPool()) + assertNull(tree.search(key(1))) + } + + @Test + fun insertAndSearchSingleEntry() { + val tree = BPlusTree.create(newPool()) + tree.insert(key(1), 100L) + assertEquals(100L, tree.search(key(1))) + assertNull(tree.search(key(2))) + } + + @Test + fun insertingSameKeyTwiceUpserts() { + val tree = BPlusTree.create(newPool()) + tree.insert(key(5), 1L) + tree.insert(key(5), 2L) + assertEquals(2L, tree.search(key(5))) + assertEquals(listOf(5L to 2L), tree.rangeScan().asSequence().map { decodeKey(it.first) to it.second }.toList()) + } + + @Test + fun deleteRemovesEntryAndReturnsFalseWhenAbsent() { + val tree = BPlusTree.create(newPool()) + tree.insert(key(1), 10L) + assertTrue(tree.delete(key(1))) + assertNull(tree.search(key(1))) + assertFalse(tree.delete(key(1))) + assertFalse(tree.delete(key(99))) + } + + @Test + fun rangeScanOnEmptyTreeYieldsNothing() { + val tree = BPlusTree.create(newPool()) + assertTrue(tree.rangeScan().asSequence().toList().isEmpty()) + } + + @Test + fun rangeScanRespectsInclusiveAndExclusiveUpperBound() { + val tree = BPlusTree.create(newPool()) + for (i in 1L..10L) tree.insert(key(i), i * 10) + val exclusive = tree.rangeScan(key(3), key(7)).asSequence().map { decodeKey(it.first) }.toList() + assertEquals((3L..6L).toList(), exclusive) + val inclusive = tree.rangeScan(key(3), key(7), toInclusive = true).asSequence().map { decodeKey(it.first) }.toList() + assertEquals((3L..7L).toList(), inclusive) + } + + @Test + fun deletingEveryEntryLeavesAnEmptySearchableTree() { + val tree = BPlusTree.create(newPool()) + for (i in 1L..20L) tree.insert(key(i), i) + for (i in 1L..20L) assertTrue(tree.delete(key(i))) + assertTrue(tree.rangeScan().asSequence().toList().isEmpty()) + assertNull(tree.search(key(1))) + tree.insert(key(1), 42L) + assertEquals(42L, tree.search(key(1))) + } + + @Test + fun splitsAcrossMultipleLevelsAndPersistsThroughBufferEviction() { + val pool = newPool(capacity = 8) // small pool forces heavy eviction/reload traffic + val tree = BPlusTree.create(pool) + val n = 3000 + for (i in 0 until n) tree.insert(key(i.toLong()), i.toLong() * 7) + for (i in 0 until n) assertEquals(i.toLong() * 7, tree.search(key(i.toLong())), "key $i") + val scanned = tree.rangeScan().asSequence().map { decodeKey(it.first) }.toList() + assertEquals((0 until n).map { it.toLong() }, scanned) + assertTrue(tree.rootPageId >= 0) + } + + @Test + fun stressInsertSearchDeleteAndRangeScanAgainstReferenceMap() { + val pool = newPool(capacity = 96) + val tree = BPlusTree.create(pool) + val random = Random(1234) + val reference = java.util.TreeMap() + + val keys = generateSequence { random.nextLong(-2_000_000_000L, 2_000_000_000L) } + .distinct() + .take(12000) + .toList() + for (k in keys) { + val rowId = k * 3 + 1 + tree.insert(key(k), rowId) + reference[k] = rowId + } + + for ((k, v) in reference) assertEquals(v, tree.search(key(k)), "expected key $k") + assertNull(tree.search(key(9_999_999_999L))) + + val fullScan = tree.rangeScan().asSequence().map { decodeKey(it.first) to it.second }.toList() + assertEquals(reference.entries.map { it.key to it.value }, fullScan) + + val lo = reference.keys.elementAt(reference.size / 4) + val hi = reference.keys.elementAt(reference.size / 2) + val expectedRange = reference.subMap(lo, true, hi, true).entries.map { it.key to it.value } + val actualRange = tree.rangeScan(key(lo), key(hi), toInclusive = true) + .asSequence().map { decodeKey(it.first) to it.second }.toList() + assertEquals(expectedRange, actualRange) + + val toDelete = keys.filterIndexed { index, _ -> index % 2 == 0 } + for (k in toDelete) { + assertTrue(tree.delete(key(k)), "expected to delete $k") + reference.remove(k) + } + for (k in toDelete.take(50)) assertFalse(tree.delete(key(k))) + + for ((k, v) in reference) assertEquals(v, tree.search(key(k)), "expected surviving key $k") + for (k in toDelete) assertNull(tree.search(key(k)), "expected deleted key $k to be gone") + + val afterDeleteScan = tree.rangeScan().asSequence().map { decodeKey(it.first) to it.second }.toList() + assertEquals(reference.entries.map { it.key to it.value }, afterDeleteScan) + } + + @Test + fun treeReopensFromPersistedRootAfterBufferFlush() { + val file = Files.createTempFile("kestrel-btree-reopen", ".db").toFile().also { it.deleteOnExit() } + val pool1 = BufferPool(DiskManager(file), 32) + val tree1 = BPlusTree.create(pool1) + for (i in 0 until 500) tree1.insert(key(i.toLong()), i.toLong()) + pool1.flushAll() + val rootId = tree1.rootPageId + + val manager2 = DiskManager(file) + val pool2 = BufferPool(manager2, 32) + val tree2 = BPlusTree.open(pool2, rootId) + for (i in 0 until 500) assertEquals(i.toLong(), tree2.search(key(i.toLong()))) + val scanned = tree2.rangeScan().asSequence().map { decodeKey(it.first) }.toList() + assertEquals((0 until 500).map { it.toLong() }, scanned) + } + + @Test + fun allPageIdsOnAFreshTreeIsJustTheRoot() { + val tree = BPlusTree.create(newPool()) + assertEquals(listOf(tree.rootPageId), tree.allPageIds()) + } + + @Test + fun allPageIdsCoversEveryInternalAndLeafPageAfterSplits() { + val pool = newPool() + val tree = BPlusTree.create(pool) + val n = 500 + for (i in 0 until n) tree.insert(key(i.toLong()), i.toLong()) + + val pageIds = tree.allPageIds() + assertTrue(pageIds.contains(tree.rootPageId)) + assertEquals(pageIds.size, pageIds.toSet().size, "expected no page id to be visited twice") + assertTrue(pageIds.size > 1, "expected $n inserts to have split the tree across more than one page") + + val leafKeys = pageIds.flatMap { pageId -> + val page = pool.fetchPage(pageId) + val node = NodeCodec.readFrom(page) + pool.unpinPage(pageId, false) + if (node is BTreeNode.Leaf) node.keys.map { decodeKey(it) } else emptyList() + } + assertEquals((0 until n).map { it.toLong() }.sorted(), leafKeys.sorted(), "every leaf page must be reachable from allPageIds") + } +} diff --git a/src/test/kotlin/kestrel/store/codec/ByteCursorTest.kt b/src/test/kotlin/kestrel/store/codec/ByteCursorTest.kt new file mode 100644 index 0000000..26c43f1 --- /dev/null +++ b/src/test/kotlin/kestrel/store/codec/ByteCursorTest.kt @@ -0,0 +1,73 @@ +package kestrel.store.codec + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ByteCursorTest { + @Test + fun writesAndReadsVarIntsAcrossByteBoundaries() { + val samples = listOf(0L, 1L, 127L, 128L, 300L, 16384L, Int.MAX_VALUE.toLong(), Long.MAX_VALUE) + val writer = ByteWriter() + for (sample in samples) writer.writeVarInt(sample) + val reader = ByteReader(writer.toByteArray()) + for (sample in samples) assertEquals(sample, reader.readVarInt()) + assertFalse(reader.hasRemaining()) + } + + @Test + fun zigZagRoundTripsNegativeAndPositiveValues() { + val samples = listOf(0L, -1L, 1L, -64L, 64L, Long.MIN_VALUE, Long.MAX_VALUE) + val writer = ByteWriter() + for (sample in samples) writer.writeZigZag(sample) + val reader = ByteReader(writer.toByteArray()) + for (sample in samples) assertEquals(sample, reader.readZigZag()) + } + + @Test + fun negativeNumbersUseFewerBytesThanUnsignedVarIntWouldViaZigZag() { + val writer = ByteWriter() + writer.writeZigZag(-1L) + assertEquals(1, writer.length) + } + + @Test + fun fixedWidthLongAndDoubleRoundTrip() { + val writer = ByteWriter() + writer.writeFixedLong(-42L) + writer.writeFixedDouble(3.14159) + val reader = ByteReader(writer.toByteArray()) + assertEquals(-42L, reader.readFixedLong()) + assertEquals(3.14159, reader.readFixedDouble()) + } + + @Test + fun lengthPrefixedBytesRoundTrip() { + val writer = ByteWriter() + writer.writeLengthPrefixed(byteArrayOf(1, 2, 3)) + writer.writeLengthPrefixed(ByteArray(0)) + writer.writeByte(9) + val reader = ByteReader(writer.toByteArray()) + assertTrue(reader.readLengthPrefixed().contentEquals(byteArrayOf(1, 2, 3))) + assertTrue(reader.readLengthPrefixed().contentEquals(ByteArray(0))) + assertEquals(9, reader.readByte()) + } + + @Test + fun readingPastEndThrows() { + val reader = ByteReader(byteArrayOf(1)) + reader.readByte() + assertFailsWith { reader.readByte() } + } + + @Test + fun growsBufferBeyondInitialCapacity() { + val writer = ByteWriter(2) + repeat(500) { writer.writeByte(it and 0xFF) } + assertEquals(500, writer.length) + val reader = ByteReader(writer.toByteArray()) + repeat(500) { assertEquals(it and 0xFF, reader.readByte()) } + } +} diff --git a/src/test/kotlin/kestrel/store/codec/Crc32CTest.kt b/src/test/kotlin/kestrel/store/codec/Crc32CTest.kt new file mode 100644 index 0000000..4312025 --- /dev/null +++ b/src/test/kotlin/kestrel/store/codec/Crc32CTest.kt @@ -0,0 +1,33 @@ +package kestrel.store.codec + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals + +class Crc32CTest { + @Test + fun sameBytesProduceSameChecksum() { + val a = "hello kestrel".toByteArray() + val b = "hello kestrel".toByteArray() + assertEquals(Crc32C.of(a), Crc32C.of(b)) + } + + @Test + fun differentBytesProduceDifferentChecksum() { + assertNotEquals(Crc32C.of("hello".toByteArray()), Crc32C.of("hellp".toByteArray())) + } + + @Test + fun detectsSingleBitFlip() { + val original = ByteArray(4096) { (it % 251).toByte() } + val corrupted = original.copyOf() + corrupted[2000] = (corrupted[2000].toInt() xor 0x01).toByte() + assertNotEquals(Crc32C.of(original), Crc32C.of(corrupted)) + } + + @Test + fun offsetAndLengthCoverOnlyTheRequestedSlice() { + val buffer = "xxhelloxx".toByteArray() + assertEquals(Crc32C.of("hello".toByteArray()), Crc32C.of(buffer, 2, 5)) + } +} diff --git a/src/test/kotlin/kestrel/store/codec/KeyCodecTest.kt b/src/test/kotlin/kestrel/store/codec/KeyCodecTest.kt new file mode 100644 index 0000000..95fcfed Binary files /dev/null and b/src/test/kotlin/kestrel/store/codec/KeyCodecTest.kt differ diff --git a/src/test/kotlin/kestrel/store/codec/TupleCodecTest.kt b/src/test/kotlin/kestrel/store/codec/TupleCodecTest.kt new file mode 100644 index 0000000..e82d8a3 --- /dev/null +++ b/src/test/kotlin/kestrel/store/codec/TupleCodecTest.kt @@ -0,0 +1,90 @@ +package kestrel.store.codec + +import java.math.BigDecimal +import java.math.BigInteger +import java.time.LocalDate +import kestrel.storage.Value +import kotlin.test.Test +import kotlin.test.assertEquals + +class TupleCodecTest { + @Test + fun roundTripsAllValueTypes() { + val row = listOf( + Value.IntegerValue(42), + Value.TextValue("hello world"), + Value.BooleanValue(true), + Value.RealValue(2.5), + Value.DecimalValue(BigDecimal("19.99")), + Value.DateValue(LocalDate.of(2024, 1, 15)), + Value.NullValue + ) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsDecimalExactlyPreservingScaleAndSign() { + val row = listOf( + Value.DecimalValue(BigDecimal.ZERO), + Value.DecimalValue(BigDecimal("-19.990")), + Value.DecimalValue(BigDecimal("1200")), + Value.DecimalValue(BigDecimal(BigInteger("123456789012345678901234567890"), 10)) + ) + val decoded = TupleCodec.decode(TupleCodec.encode(row)) + assertEquals(row, decoded) + // Exact scale round-trip (not just numeric equality): -19.990 must stay 3-decimal, not collapse to -19.99. + assertEquals(3, (decoded[1] as Value.DecimalValue).value.scale()) + } + + @Test + fun roundTripsDatesIncludingBeforeTheEpoch() { + val row = listOf( + Value.DateValue(LocalDate.of(1970, 1, 1)), + Value.DateValue(LocalDate.of(1900, 1, 1)), + Value.DateValue(LocalDate.of(2999, 12, 31)) + ) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsNegativeAndExtremeIntegers() { + val row = listOf(Value.IntegerValue(Long.MIN_VALUE), Value.IntegerValue(Long.MAX_VALUE), Value.IntegerValue(-1)) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsAllNullRow() { + val row = listOf(Value.NullValue, Value.NullValue, Value.NullValue) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsEmptyRow() { + assertEquals(emptyList(), TupleCodec.decode(TupleCodec.encode(emptyList()))) + } + + @Test + fun nullBitmapSpansMultipleBytesForWideRows() { + val row = (0 until 20).map { if (it % 3 == 0) Value.NullValue else Value.IntegerValue(it.toLong()) } + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsUnicodeText() { + val row = listOf(Value.TextValue("zolw kruk ósmórog 🐦")) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun roundTripsEmptyText() { + val row = listOf(Value.TextValue("")) + assertEquals(row, TupleCodec.decode(TupleCodec.encode(row))) + } + + @Test + fun encodingIsCompactForSmallIntegers() { + val encoded = TupleCodec.encode(listOf(Value.IntegerValue(5))) + // 1 byte column count + 1 byte null bitmap + 1 byte tag + 1 byte zigzag payload + assertEquals(4, encoded.size) + } +} diff --git a/src/test/kotlin/kestrel/store/engine/LoggingPageStoreTest.kt b/src/test/kotlin/kestrel/store/engine/LoggingPageStoreTest.kt new file mode 100644 index 0000000..25cc45f --- /dev/null +++ b/src/test/kotlin/kestrel/store/engine/LoggingPageStoreTest.kt @@ -0,0 +1,74 @@ +package kestrel.store.engine + +import kestrel.store.heap.HeapFile +import kestrel.store.heap.HeapPageException +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import kestrel.store.txn.TransactionManager +import kestrel.store.wal.RecoveryManager +import kestrel.store.wal.WriteAheadLog +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class LoggingPageStoreTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private inner class Fixture { + val diskManager = DiskManager(tempFile("kestrel-logstore-data")) + val bufferPool = BufferPool(diskManager, 16) + val wal = WriteAheadLog(tempFile("kestrel-logstore-wal")) + val txnManager = TransactionManager(wal, diskManager) + } + + @Test + fun insertThroughTheLoggingStoreProducesAWalRecordForTheMutatedPage() { + val fx = Fixture() + val heapHeaderId = HeapFile.create(fx.bufferPool).headerPageId + + val txnId = fx.txnManager.begin() + val touched = mutableListOf() + val logging = LoggingPageStore(fx.bufferPool, fx.txnManager, txnId) { touched.add(it) } + val rid = HeapFile.open(logging, heapHeaderId).insert("hello".toByteArray()) + logging.finish() + + assertTrue(rid.pageId in touched, "the page the record landed on must be reported to the touch callback") + assertTrue(fx.wal.lastLsnOf(txnId) != WriteAheadLog.NO_LSN, "insert must produce at least one logged update") + } + + @Test + fun readOnlyAccessProducesNoWalRecord() { + val fx = Fixture() + val heap = HeapFile.create(fx.bufferPool) + val rid = heap.insert("already there".toByteArray()) + fx.bufferPool.flushAll() + + val txnId = fx.txnManager.begin() + val lsnAfterBegin = fx.wal.lastLsnOf(txnId) + val logging = LoggingPageStore(fx.bufferPool, fx.txnManager, txnId) + val bytes = HeapFile.open(logging, heap.headerPageId).get(rid) + logging.finish() + + assertEquals("already there", String(bytes)) + assertEquals(lsnAfterBegin, fx.wal.lastLsnOf(txnId), "a pure read must not append any update record") + } + + @Test + fun rollingBackAnUnfinishedInsertErasesItFromThePage() { + val fx = Fixture() + val heapHeaderId = HeapFile.create(fx.bufferPool).headerPageId + + val txnId = fx.txnManager.begin() + val logging = LoggingPageStore(fx.bufferPool, fx.txnManager, txnId) + val rid = HeapFile.open(logging, heapHeaderId).insert("temporary".toByteArray()) + logging.finish() + + RecoveryManager(fx.wal, fx.diskManager).rollback(txnId) + fx.bufferPool.invalidate(rid.pageId) + + assertFailsWith { HeapFile.open(fx.bufferPool, heapHeaderId).get(rid) } + } +} diff --git a/src/test/kotlin/kestrel/store/engine/StorageEngineTest.kt b/src/test/kotlin/kestrel/store/engine/StorageEngineTest.kt new file mode 100644 index 0000000..b1967e9 --- /dev/null +++ b/src/test/kotlin/kestrel/store/engine/StorageEngineTest.kt @@ -0,0 +1,122 @@ +package kestrel.store.engine + +import kestrel.store.codec.KeyCodec +import kestrel.store.codec.TupleCodec +import kestrel.store.heap.RecordId +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class StorageEngineTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + @Test + fun aCommittedInsertSurvivesACrashThatNeverFlushedTheBufferPool() { + val dataFile = tempFile("kestrel-engine-data") + val walFile = tempFile("kestrel-engine-wal") + + val engine1 = StorageEngine.open(dataFile, walFile) + val heap = engine1.createHeap() + val txn = engine1.beginTransaction() + val rid = heap.insert(txn, "durable across a crash".toByteArray()) + engine1.commit(txn) + // No engine1.close() / flush(): the committed page may still be sitting only in + // the buffer pool, never written to dataFile. Only the WAL commit record is durable. + + val engine2 = StorageEngine.open(dataFile, walFile) + val reopenedHeap = engine2.openHeap(heap.headerPageId) + val reader = engine2.beginTransaction() + assertEquals("durable across a crash", String(reopenedHeap.get(reader, rid))) + assertEquals(listOf(rid), reopenedHeap.scan(reader).map { it.first }) + } + + @Test + fun abortMakesAnInsertInvisibleWithoutRestartingTheEngine() { + val engine = StorageEngine.open(tempFile("kestrel-engine-data"), tempFile("kestrel-engine-wal")) + val heap = engine.createHeap() + + val writer = engine.beginTransaction() + heap.insert(writer, "temporary".toByteArray()) + engine.abort(writer) + + val reader = engine.beginTransaction() + assertTrue(heap.scan(reader).isEmpty()) + } + + @Test + fun aTableAndItsSecondaryIndexCommitAndAbortTogetherAsOneTransaction() { + val engine = StorageEngine.open(tempFile("kestrel-engine-data"), tempFile("kestrel-engine-wal")) + val table = engine.createHeap() + val index = engine.createIndex() + + fun rowFor(id: Long, name: String) = TupleCodec.encode(listOf(Value.IntegerValue(id), Value.TextValue(name))) + fun keyFor(id: Long) = KeyCodec.encode(Value.IntegerValue(id)) + + val setup = engine.beginTransaction() + val committedRid = table.insert(setup, rowFor(1, "alice")) + index.insert(setup, keyFor(1), committedRid.encode()) + engine.commit(setup) + + val doomed = engine.beginTransaction() + val doomedRid = table.insert(doomed, rowFor(2, "bob")) + index.insert(doomed, keyFor(2), doomedRid.encode()) + engine.abort(doomed) + + val reader = engine.beginTransaction() + assertNull(index.search(reader, keyFor(2)), "the aborted row's index entry must not survive") + val foundRid = RecordId.decode(index.search(reader, keyFor(1))!!) + assertEquals(committedRid, foundRid) + val row = TupleCodec.decode(table.get(reader, foundRid)) + assertEquals(listOf(Value.IntegerValue(1), Value.TextValue("alice")), row) + assertEquals(listOf(committedRid), table.scan(reader).map { it.first }) + } + + @Test + fun anAbortedInsertFromAPriorSessionStaysGoneAfterReopening() { + val dataFile = tempFile("kestrel-engine-data") + val walFile = tempFile("kestrel-engine-wal") + + val engine1 = StorageEngine.open(dataFile, walFile) + val heap = engine1.createHeap() + + val committed = engine1.beginTransaction() + heap.insert(committed, "kept".toByteArray()) + engine1.commit(committed) + + val doomed = engine1.beginTransaction() + heap.insert(doomed, "discarded".toByteArray()) + engine1.abort(doomed) + engine1.close() + + val engine2 = StorageEngine.open(dataFile, walFile) + val reopenedHeap = engine2.openHeap(heap.headerPageId) + val reader = engine2.beginTransaction() + assertEquals(listOf("kept"), reopenedHeap.scan(reader).map { String(it.second) }) + } + + @Test + fun closingAndReopeningPreservesBothTableAndIndexData() { + val dataFile = tempFile("kestrel-engine-data") + val walFile = tempFile("kestrel-engine-wal") + + val engine1 = StorageEngine.open(dataFile, walFile) + val table = engine1.createHeap() + val index = engine1.createIndex() + val txn = engine1.beginTransaction() + val rid = table.insert(txn, TupleCodec.encode(listOf(Value.IntegerValue(7)))) + index.insert(txn, KeyCodec.encode(Value.IntegerValue(7)), rid.encode()) + engine1.commit(txn) + engine1.close() + + val engine2 = StorageEngine.open(dataFile, walFile) + val reopenedTable = engine2.openHeap(table.headerPageId) + val reopenedIndex = engine2.openIndex(index.rootPageId) + val reader = engine2.beginTransaction() + val foundRid = RecordId.decode(reopenedIndex.search(reader, KeyCodec.encode(Value.IntegerValue(7)))!!) + assertEquals(listOf(Value.IntegerValue(7)), TupleCodec.decode(reopenedTable.get(reader, foundRid))) + } +} diff --git a/src/test/kotlin/kestrel/store/engine/TransactionalHeapTest.kt b/src/test/kotlin/kestrel/store/engine/TransactionalHeapTest.kt new file mode 100644 index 0000000..855b060 --- /dev/null +++ b/src/test/kotlin/kestrel/store/engine/TransactionalHeapTest.kt @@ -0,0 +1,118 @@ +package kestrel.store.engine + +import kestrel.store.heap.HeapFileException +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import kestrel.store.txn.TransactionManager +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class TransactionalHeapTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private inner class Fixture { + val diskManager = DiskManager(tempFile("kestrel-txheap-data")) + val bufferPool = BufferPool(diskManager, 32) + val wal = kestrel.store.wal.WriteAheadLog(tempFile("kestrel-txheap-wal")) + val txnManager = TransactionManager(wal, diskManager) + val tracker = TouchedPageTracker() + val heap = TransactionalHeap.create(bufferPool, txnManager, tracker) + + fun abort(txnId: Long) { + txnManager.abort(txnId) + for (pageId in tracker.forget(txnId)) bufferPool.invalidate(pageId) + } + } + + @Test + fun aCommittedInsertIsVisibleToLaterTransactions() { + val fx = Fixture() + val writer = fx.txnManager.begin() + val rid = fx.heap.insert(writer, "durable row".toByteArray()) + fx.txnManager.commit(writer) + + val reader = fx.txnManager.begin() + assertEquals("durable row", String(fx.heap.get(reader, rid))) + assertEquals(listOf(rid), fx.heap.scan(reader).map { it.first }) + } + + @Test + fun anAbortedInsertDisappearsImmediatelyEvenThoughItsPageIsStillCached() { + val fx = Fixture() + val writer = fx.txnManager.begin() + val rid = fx.heap.insert(writer, "should vanish".toByteArray()) + fx.abort(writer) + + val reader = fx.txnManager.begin() + assertTrue(fx.heap.scan(reader).isEmpty(), "an aborted insert must not be visible, cache or no cache") + assertFailsWith { fx.heap.get(reader, rid) } + } + + @Test + fun updateThenAbortRestoresThePriorValue() { + val fx = Fixture() + val setup = fx.txnManager.begin() + val rid = fx.heap.insert(setup, "original".toByteArray()) + fx.txnManager.commit(setup) + + val writer = fx.txnManager.begin() + fx.heap.update(writer, rid, "changed".toByteArray()) + fx.abort(writer) + + val reader = fx.txnManager.begin() + assertEquals("original", String(fx.heap.get(reader, rid))) + } + + @Test + fun deleteThenAbortRestoresTheRecord() { + val fx = Fixture() + val setup = fx.txnManager.begin() + val rid = fx.heap.insert(setup, "keep me".toByteArray()) + fx.txnManager.commit(setup) + + val writer = fx.txnManager.begin() + fx.heap.delete(writer, rid) + fx.abort(writer) + + val reader = fx.txnManager.begin() + assertEquals("keep me", String(fx.heap.get(reader, rid))) + } + + @Test + fun destroyFreesEveryPageSoALaterHeapReusesThemInsteadOfGrowingTheFile() { + val fx = Fixture() + val writer = fx.txnManager.begin() + for (i in 0 until 40) fx.heap.insert(writer, ByteArray(200) { i.toByte() }) + fx.txnManager.commit(writer) + val peak = fx.diskManager.allocatedPageCount() + + val destroyer = fx.txnManager.begin() + fx.heap.destroy(destroyer) + fx.txnManager.commit(destroyer) + + val fresh = TransactionalHeap.create(fx.bufferPool, fx.txnManager, fx.tracker) + val filler = fx.txnManager.begin() + for (i in 0 until 40) fresh.insert(filler, ByteArray(200) { i.toByte() }) + fx.txnManager.commit(filler) + + assertTrue( + fx.diskManager.allocatedPageCount() <= peak, + "expected the destroyed heap's pages to be reused by the new heap instead of growing the file " + + "further (peak=$peak, now=${fx.diskManager.allocatedPageCount()})" + ) + } + + @Test + fun oversizedRecordFailsWithoutLoggingAnything() { + val fx = Fixture() + val txnId = fx.txnManager.begin() + assertFailsWith { + fx.heap.insert(txnId, ByteArray(kestrel.store.heap.HeapPage.MAX_RECORD_SIZE + 1)) + } + fx.txnManager.commit(txnId) + } +} diff --git a/src/test/kotlin/kestrel/store/engine/TransactionalIndexTest.kt b/src/test/kotlin/kestrel/store/engine/TransactionalIndexTest.kt new file mode 100644 index 0000000..9fc385d --- /dev/null +++ b/src/test/kotlin/kestrel/store/engine/TransactionalIndexTest.kt @@ -0,0 +1,136 @@ +package kestrel.store.engine + +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import kestrel.store.txn.TransactionManager +import kestrel.store.wal.WriteAheadLog +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +class TransactionalIndexTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun key(i: Int): ByteArray = "key-%05d".format(i).toByteArray() + + private inner class Fixture { + val diskManager = DiskManager(tempFile("kestrel-txindex-data")) + val bufferPool = BufferPool(diskManager, 64) + val wal = WriteAheadLog(tempFile("kestrel-txindex-wal")) + val txnManager = TransactionManager(wal, diskManager) + val tracker = TouchedPageTracker() + val index = TransactionalIndex.create(bufferPool, txnManager, tracker, lockResourceId = -1L) + + fun abort(txnId: Long) { + txnManager.abort(txnId) + index.rollback(txnId) + for (pageId in tracker.forget(txnId)) bufferPool.invalidate(pageId) + } + + fun commit(txnId: Long) { + txnManager.commit(txnId) + index.forgetTransaction(txnId) + tracker.forget(txnId) + } + } + + @Test + fun aCommittedEntryIsVisibleToLaterTransactions() { + val fx = Fixture() + val writer = fx.txnManager.begin() + fx.index.insert(writer, key(1), 100L) + fx.commit(writer) + + val reader = fx.txnManager.begin() + assertEquals(100L, fx.index.search(reader, key(1))) + } + + @Test + fun deleteThenCommitRemovesTheEntry() { + val fx = Fixture() + val setup = fx.txnManager.begin() + fx.index.insert(setup, key(1), 1L) + fx.commit(setup) + + val writer = fx.txnManager.begin() + fx.index.delete(writer, key(1)) + fx.commit(writer) + + val reader = fx.txnManager.begin() + assertNull(fx.index.search(reader, key(1))) + } + + @Test + fun enoughInsertsForceARootSplitAndEveryKeyStaysSearchable() { + val fx = Fixture() + val originalRoot = fx.index.rootPageId + val writer = fx.txnManager.begin() + for (i in 0 until 400) fx.index.insert(writer, key(i), i.toLong()) + fx.commit(writer) + + assertNotEquals(originalRoot, fx.index.rootPageId, "400 entries must have split the original single-leaf root") + val reader = fx.txnManager.begin() + for (i in 0 until 400) assertEquals(i.toLong(), fx.index.search(reader, key(i))) + } + + @Test + fun abortingAWriteThatSplitTheRootRestoresTheOldRootAndKeepsPriorDataIntact() { + val fx = Fixture() + val setup = fx.txnManager.begin() + fx.index.insert(setup, key(0), 0L) + fx.commit(setup) + val rootBeforeWriter = fx.index.rootPageId + + val writer = fx.txnManager.begin() + for (i in 1 until 400) fx.index.insert(writer, key(i), i.toLong()) + assertNotEquals(rootBeforeWriter, fx.index.rootPageId, "the writer's own inserts must have split the root") + + fx.abort(writer) + + assertEquals(rootBeforeWriter, fx.index.rootPageId, "abort must restore the root pointer as it was before the writer began") + val reader = fx.txnManager.begin() + assertEquals(0L, fx.index.search(reader, key(0)), "data committed before the aborted writer must survive") + assertNull(fx.index.search(reader, key(1)), "data inserted by the aborted writer must not be visible") + } + + @Test + fun rangeScanReturnsKeysInOrderWithinBounds() { + val fx = Fixture() + val writer = fx.txnManager.begin() + for (i in 0 until 50) fx.index.insert(writer, key(i), i.toLong()) + fx.commit(writer) + + val reader = fx.txnManager.begin() + val scanned = fx.index.rangeScan(reader, from = key(10), to = key(15), toInclusive = true) + assertEquals((10..15).map { it.toLong() }, scanned.map { it.second }) + assertTrue(scanned.zipWithNext().all { (a, b) -> kestrel.store.codec.KeyCodec.compare(a.first, b.first) < 0 }) + } + + @Test + fun destroyFreesEveryPageSoALaterIndexReusesThemInsteadOfGrowingTheFile() { + val fx = Fixture() + val writer = fx.txnManager.begin() + for (i in 0 until 400) fx.index.insert(writer, key(i), i.toLong()) + fx.commit(writer) + val peak = fx.diskManager.allocatedPageCount() + + val destroyer = fx.txnManager.begin() + fx.index.destroy(destroyer) + fx.txnManager.commit(destroyer) + + val fresh = TransactionalIndex.create(fx.bufferPool, fx.txnManager, fx.tracker, lockResourceId = -2L) + val filler = fx.txnManager.begin() + for (i in 0 until 400) fresh.insert(filler, key(i), i.toLong()) + fx.txnManager.commit(filler) + + assertTrue( + fx.diskManager.allocatedPageCount() <= peak, + "expected the destroyed index's pages to be reused by the new index instead of growing the file " + + "further (peak=$peak, now=${fx.diskManager.allocatedPageCount()})" + ) + } +} diff --git a/src/test/kotlin/kestrel/store/heap/HeapFileTest.kt b/src/test/kotlin/kestrel/store/heap/HeapFileTest.kt new file mode 100644 index 0000000..25c5365 --- /dev/null +++ b/src/test/kotlin/kestrel/store/heap/HeapFileTest.kt @@ -0,0 +1,125 @@ +package kestrel.store.heap + +import kestrel.store.page.BufferPool +import kestrel.store.page.DiskManager +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class HeapFileTest { + private fun tempFile(): File = Files.createTempFile("kestrel-heap", ".db").toFile().also { it.deleteOnExit() } + + private fun newPool(file: File = tempFile(), capacity: Int = 64): BufferPool = BufferPool(DiskManager(file), capacity) + + @Test + fun insertAndGetRoundTrips() { + val heap = HeapFile.create(newPool()) + val rid = heap.insert("hello world".toByteArray()) + assertEquals("hello world", String(heap.get(rid))) + } + + @Test + fun insertManySmallRecordsAllReadableIndividually() { + val heap = HeapFile.create(newPool()) + val ids = (0 until 50).map { heap.insert("row-$it".toByteArray()) } + for ((index, rid) in ids.withIndex()) { + assertEquals("row-$index", String(heap.get(rid))) + } + } + + @Test + fun insertSpillsToANewPageWhenCurrentPageIsFull() { + val heap = HeapFile.create(newPool()) + val ids = (0 until 40).map { heap.insert(ByteArray(200) { it.toByte() }) } + val distinctPages = ids.map { it.pageId }.toSet() + assertTrue(distinctPages.size > 1, "expected records to spread across multiple pages") + for (rid in ids) assertEquals(200, heap.get(rid).size) + } + + @Test + fun scanReturnsEveryLiveRecordAcrossAllPages() { + val heap = HeapFile.create(newPool()) + val inserted = (0 until 30).map { heap.insert(ByteArray(300) { i -> i.toByte() }) }.toSet() + val scanned = heap.scan().map { it.first }.toSet() + assertEquals(inserted, scanned) + } + + @Test + fun deleteRemovesRecordFromScanAndGetFails() { + val heap = HeapFile.create(newPool()) + val keep = heap.insert("keep".toByteArray()) + val drop = heap.insert("drop".toByteArray()) + heap.delete(drop) + assertEquals(setOf(keep), heap.scan().map { it.first }.toSet()) + assertFailsWith { heap.get(drop) } + } + + @Test + fun updateThatFitsInPlaceKeepsSameRecordId() { + val heap = HeapFile.create(newPool()) + val rid = heap.insert("original text".toByteArray()) + val updated = heap.update(rid, "short".toByteArray()) + assertEquals(rid, updated) + assertEquals("short", String(heap.get(rid))) + } + + @Test + fun updateThatOutgrowsSlotReturnsNewRecordIdAndDropsOld() { + val heap = HeapFile.create(newPool()) + val rid = heap.insert("tiny".toByteArray()) + val updated = heap.update(rid, "a much longer replacement value".toByteArray()) + assertTrue(updated != rid) + assertEquals("a much longer replacement value", String(heap.get(updated))) + assertFailsWith { heap.get(rid) } + } + + @Test + fun deletedSpaceIsReusedByLaterInsertsBeforeGrowingTheFile() { + val heap = HeapFile.create(newPool()) + val filler = (0 until 15).map { heap.insert(ByteArray(250)) } + val pagesAfterFill = filler.map { it.pageId }.toSet() + for (rid in filler.take(10)) heap.delete(rid) + val reused = heap.insert(ByteArray(250)) + assertTrue(reused.pageId in pagesAfterFill, "expected reclaimed space to be reused instead of allocating a new page") + } + + @Test + fun insertRejectsRecordLargerThanAnEmptyPage() { + val heap = HeapFile.create(newPool()) + assertFailsWith { heap.insert(ByteArray(HeapPage.MAX_RECORD_SIZE + 1)) } + } + + @Test + fun allPageIdsOnAFreshHeapIsJustTheHeader() { + val heap = HeapFile.create(newPool()) + assertEquals(listOf(heap.headerPageId), heap.allPageIds()) + } + + @Test + fun allPageIdsCoversTheHeaderAndEveryDataPage() { + val heap = HeapFile.create(newPool()) + val ids = (0 until 40).map { heap.insert(ByteArray(200) { it.toByte() }) } + val dataPages = ids.map { it.pageId }.toSet() + + val allPageIds = heap.allPageIds() + assertEquals(allPageIds.size, allPageIds.toSet().size, "expected no page id to be visited twice") + assertTrue(allPageIds.contains(heap.headerPageId)) + assertEquals(dataPages + heap.headerPageId, allPageIds.toSet()) + } + + @Test + fun reopeningWithSameHeaderPageSeesPersistedData() { + val file = tempFile() + val diskManager = DiskManager(file) + val pool = BufferPool(diskManager, 64) + val heap = HeapFile.create(pool) + val rid = heap.insert("durable".toByteArray()) + pool.flushAll() + + val reopened = HeapFile.open(pool, heap.headerPageId) + assertEquals("durable", String(reopened.get(rid))) + } +} diff --git a/src/test/kotlin/kestrel/store/heap/HeapPageTest.kt b/src/test/kotlin/kestrel/store/heap/HeapPageTest.kt new file mode 100644 index 0000000..6e86ac8 --- /dev/null +++ b/src/test/kotlin/kestrel/store/heap/HeapPageTest.kt @@ -0,0 +1,139 @@ +package kestrel.store.heap + +import kestrel.store.page.Page +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class HeapPageTest { + private fun freshPage(): Page = Page(0).also { HeapPage.init(it) } + + @Test + fun freshPageHasNoSlotsAndFullFreeSpace() { + val page = freshPage() + assertEquals(0, HeapPage.slotCount(page)) + assertEquals(HeapPage.NO_PAGE, HeapPage.nextPageId(page)) + assertTrue(HeapPage.freeBytes(page) > HeapPage.MAX_RECORD_SIZE) + } + + @Test + fun insertAndGetRoundTrips() { + val page = freshPage() + val slot = HeapPage.insert(page, "hello".toByteArray()) + assertEquals(0, slot) + assertEquals("hello", String(HeapPage.get(page, slot!!))) + } + + @Test + fun multipleSlotsPreserveIndependentContent() { + val page = freshPage() + val a = HeapPage.insert(page, "alpha".toByteArray())!! + val b = HeapPage.insert(page, "beta".toByteArray())!! + val c = HeapPage.insert(page, "gamma".toByteArray())!! + assertEquals("alpha", String(HeapPage.get(page, a))) + assertEquals("beta", String(HeapPage.get(page, b))) + assertEquals("gamma", String(HeapPage.get(page, c))) + } + + @Test + fun freeBytesShrinksAsRecordsAreInserted() { + val page = freshPage() + val before = HeapPage.freeBytes(page) + HeapPage.insert(page, ByteArray(100)) + val after = HeapPage.freeBytes(page) + assertEquals(100 + 4, before - after) + } + + @Test + fun insertReturnsNullWhenPageIsFull() { + val page = freshPage() + var last: Int? = 0 + while (last != null) { + last = HeapPage.insert(page, ByteArray(200)) + } + assertNull(HeapPage.insert(page, ByteArray(200))) + } + + @Test + fun deleteTombstonesSlotAndGetThrows() { + val page = freshPage() + val slot = HeapPage.insert(page, "doomed".toByteArray())!! + HeapPage.delete(page, slot) + assertTrue(HeapPage.isDeleted(page, slot)) + assertFailsWith { HeapPage.get(page, slot) } + } + + @Test + fun deletingAlreadyDeletedSlotFails() { + val page = freshPage() + val slot = HeapPage.insert(page, "x".toByteArray())!! + HeapPage.delete(page, slot) + assertFailsWith { HeapPage.delete(page, slot) } + } + + @Test + fun compactReclaimsSpaceFromDeletedSlots() { + val page = freshPage() + val slots = ArrayList() + while (true) { + val slot = HeapPage.insert(page, ByteArray(250)) ?: break + slots.add(slot) + } + for (slot in slots.take(slots.size / 2)) HeapPage.delete(page, slot) + // Freed space is fragmented (tombstoned), not contiguous, until compact runs from within insert. + assertTrue(HeapPage.insert(page, ByteArray(250)) != null) + } + + @Test + fun compactPreservesLiveSlotIdsAndContent() { + val page = freshPage() + val a = HeapPage.insert(page, "keep-a".toByteArray())!! + val b = HeapPage.insert(page, "gone".toByteArray())!! + val c = HeapPage.insert(page, "keep-c".toByteArray())!! + HeapPage.delete(page, b) + HeapPage.compact(page) + assertEquals("keep-a", String(HeapPage.get(page, a))) + assertEquals("keep-c", String(HeapPage.get(page, c))) + assertTrue(HeapPage.isDeleted(page, b)) + } + + @Test + fun overwriteInPlaceShrinksLength() { + val page = freshPage() + val slot = HeapPage.insert(page, "original-value".toByteArray())!! + HeapPage.overwrite(page, slot, "short".toByteArray()) + assertEquals("short", String(HeapPage.get(page, slot))) + assertEquals(5, HeapPage.lengthOf(page, slot)) + } + + @Test + fun overwriteGrowingBeyondCapacityFails() { + val page = freshPage() + val slot = HeapPage.insert(page, "short".toByteArray())!! + assertFailsWith { HeapPage.overwrite(page, slot, "much-longer-value".toByteArray()) } + } + + @Test + fun accessingOutOfRangeSlotFails() { + val page = freshPage() + assertFailsWith { HeapPage.get(page, 0) } + assertFailsWith { HeapPage.get(page, -1) } + } + + @Test + fun insertRejectsRecordLargerThanPage() { + val page = freshPage() + assertFailsWith { HeapPage.insert(page, ByteArray(HeapPage.MAX_RECORD_SIZE + 1)) } + } + + @Test + fun setNextPageIdRoundTripsAndMarksPageDirty() { + val page = freshPage() + page.isDirty = false + HeapPage.setNextPageId(page, 42L) + assertEquals(42L, HeapPage.nextPageId(page)) + assertTrue(page.isDirty) + } +} diff --git a/src/test/kotlin/kestrel/store/page/BufferPoolTest.kt b/src/test/kotlin/kestrel/store/page/BufferPoolTest.kt new file mode 100644 index 0000000..f228ff4 --- /dev/null +++ b/src/test/kotlin/kestrel/store/page/BufferPoolTest.kt @@ -0,0 +1,154 @@ +package kestrel.store.page + +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class BufferPoolTest { + private fun newManager() = DiskManager(Files.createTempFile("kestrel-pool", ".db").toFile().also { it.deleteOnExit() }) + + @Test + fun fetchingTheSamePageTwiceReturnsTheSameCachedInstance() { + val pool = BufferPool(newManager(), capacity = 4) + val page = pool.newPage() + page.data[0] = 1 + pool.unpinPage(page.id, true) + val first = pool.fetchPage(page.id) + val second = pool.fetchPage(page.id) + assertTrue(first === second) + pool.unpinPage(page.id, false) + pool.unpinPage(page.id, false) + } + + @Test + fun dirtyPageIsPersistedOnEviction() { + val manager = newManager() + val pool = BufferPool(manager, capacity = 1) + val page = pool.newPage() + page.data[0] = 99 + pool.unpinPage(page.id, true) + + // Force eviction of the only frame by fetching a second page. + val other = pool.newPage() + pool.unpinPage(other.id, false) + + val reloaded = manager.readPage(page.id) + assertEquals(99, reloaded.data[0]) + } + + @Test + fun cleanPageIsNotRewrittenOnEviction() { + val manager = newManager() + val pool = BufferPool(manager, capacity = 1) + val page = pool.newPage() + pool.unpinPage(page.id, true) + pool.flushAll() + + val fetched = pool.fetchPage(page.id) + fetched.data[5] = 55 // mutate without marking dirty + pool.unpinPage(page.id, markDirty = false) + + pool.newPage() // evicts the clean page without writeback + val reloaded = manager.readPage(page.id) + assertEquals(0, reloaded.data[5]) + } + + @Test + fun pinnedPagesAreNeverEvicted() { + val pool = BufferPool(newManager(), capacity = 1) + val pinned = pool.newPage() // pinCount == 1, never unpinned + assertFailsWith { pool.newPage() } + pool.unpinPage(pinned.id, false) + } + + @Test + fun clockGivesRecentlyTouchedPagesASecondChanceOverStalePages() { + val pool = BufferPool(newManager(), capacity = 3) + val a = pool.newPage(); pool.unpinPage(a.id, true) + val b = pool.newPage(); pool.unpinPage(b.id, true) + val c = pool.newPage(); pool.unpinPage(c.id, true) + + // One full clock sweep: evicts `a` (first frame revisited) and clears b/c's reference bits. + val d = pool.newPage(); pool.unpinPage(d.id, true) + + // Refresh `c`'s reference bit by touching it again; `b` is left stale. + val cTouch = pool.fetchPage(c.id) + pool.unpinPage(c.id, false) + + // Next eviction must take the stale page (`b`), sparing the freshly touched `c`. + val e = pool.newPage(); pool.unpinPage(e.id, true) + + assertTrue(pool.fetchPage(c.id) === cTouch, "c should still be cached, not re-read from disk") + pool.unpinPage(c.id, false) + assertTrue(pool.fetchPage(b.id) !== b, "b should have been evicted and re-read as a fresh instance") + pool.unpinPage(b.id, false) + } + + @Test + fun deletingAPageFreesItInTheDiskManager() { + val manager = newManager() + val pool = BufferPool(manager, capacity = 4) + val page = pool.newPage() + val id = page.id + pool.unpinPage(id, true) + pool.deletePage(id) + val recycled = pool.newPage() + assertEquals(id, recycled.id) + pool.unpinPage(recycled.id, true) + } + + @Test + fun deletingAPinnedPageIsRejected() { + val pool = BufferPool(newManager(), capacity = 4) + val page = pool.newPage() + assertFailsWith { pool.deletePage(page.id) } + pool.unpinPage(page.id, false) + } + + @Test + fun flushPageWritesBackImmediatelyWithoutEviction() { + val manager = newManager() + val pool = BufferPool(manager, capacity = 4) + val page = pool.newPage() + page.data[0] = 77 + pool.unpinPage(page.id, true) + pool.flushPage(page.id) + assertEquals(77, manager.readPage(page.id).data[0]) + } + + @Test + fun invalidateDropsADirtyPageWithoutWritingItBack() { + val manager = newManager() + val pool = BufferPool(manager, capacity = 4) + val page = pool.newPage() + pool.unpinPage(page.id, true) + pool.flushAll() // establishes a known, all-zero baseline on disk + + val fetched = pool.fetchPage(page.id) + fetched.data[0] = 42 + pool.unpinPage(page.id, true) + + pool.invalidate(page.id) + + assertEquals(0, manager.readPage(page.id).data[0], "invalidate must not flush the dirty page it drops") + val refetched = pool.fetchPage(page.id) + assertTrue(refetched !== fetched, "a page dropped by invalidate must be re-read fresh from disk") + pool.unpinPage(page.id, false) + } + + @Test + fun invalidatingAPinnedPageIsRejected() { + val pool = BufferPool(newManager(), capacity = 4) + val page = pool.newPage() + assertFailsWith { pool.invalidate(page.id) } + pool.unpinPage(page.id, false) + } + + @Test + fun invalidatingAnUncachedPageIsANoOp() { + val pool = BufferPool(newManager(), capacity = 4) + pool.invalidate(999L) + } +} diff --git a/src/test/kotlin/kestrel/store/page/DiskManagerTest.kt b/src/test/kotlin/kestrel/store/page/DiskManagerTest.kt new file mode 100644 index 0000000..e67d8cc --- /dev/null +++ b/src/test/kotlin/kestrel/store/page/DiskManagerTest.kt @@ -0,0 +1,116 @@ +package kestrel.store.page + +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class DiskManagerTest { + private fun tempFile() = Files.createTempFile("kestrel-disk", ".db").toFile().also { it.deleteOnExit() } + + @Test + fun allocatedPagesStartZeroedAndGrowSequentially() { + val manager = DiskManager(tempFile()) + val first = manager.allocatePage() + val second = manager.allocatePage() + assertEquals(0L, first) + assertEquals(1L, second) + assertEquals(2L, manager.allocatedPageCount()) + manager.close() + } + + @Test + fun writtenPageContentSurvivesReadBack() { + val manager = DiskManager(tempFile()) + val id = manager.allocatePage() + val page = Page(id) + page.data[0] = 0x42 + page.data[Page.PAGE_SIZE - 1] = 0x24 + manager.writePage(page) + val reloaded = manager.readPage(id) + assertEquals(0x42, reloaded.data[0]) + assertEquals(0x24, reloaded.data[Page.PAGE_SIZE - 1]) + manager.close() + } + + @Test + fun stateSurvivesCloseAndReopen() { + val file = tempFile() + val manager = DiskManager(file) + val id = manager.allocatePage() + val page = Page(id) + page.data.fill(7) + manager.writePage(page) + manager.close() + + val reopened = DiskManager(file) + assertEquals(1L, reopened.allocatedPageCount()) + val reloaded = reopened.readPage(id) + assertTrue(reloaded.data.all { it == 7.toByte() }) + reopened.close() + } + + @Test + fun freedPagesAreRecycledBeforeGrowingTheFile() { + val manager = DiskManager(tempFile()) + val a = manager.allocatePage() + val b = manager.allocatePage() + manager.freePage(a) + val reused = manager.allocatePage() + assertEquals(a, reused) + assertEquals(2L, manager.allocatedPageCount()) + val fresh = manager.allocatePage() + assertEquals(2L, fresh) + assertTrue(b != reused) + manager.close() + } + + @Test + fun freeListSurvivesReopenAndPopsMostRecentlyFreedFirst() { + val file = tempFile() + val manager = DiskManager(file) + val a = manager.allocatePage() + val b = manager.allocatePage() + manager.freePage(a) + manager.freePage(b) + manager.close() + + val reopened = DiskManager(file) + assertEquals(b, reopened.allocatePage()) + assertEquals(a, reopened.allocatePage()) + reopened.close() + } + + @Test + fun corruptedPageBytesAreDetectedOnRead() { + val file = tempFile() + val manager = DiskManager(file) + val id = manager.allocatePage() + val page = Page(id) + page.data.fill(9) + manager.writePage(page) + manager.close() + + // Flip a byte inside the payload region of the physical slot for `id`, after the header slot and checksum. + val slotSize = 4 + Page.PAGE_SIZE + val offset = (id + 1) * slotSize + 4 + 10 + val raf = java.io.RandomAccessFile(file, "rw") + raf.seek(offset) + val original = raf.read() + raf.seek(offset) + raf.writeByte((original xor 0xFF) and 0xFF) + raf.close() + + val reopened = DiskManager(file) + assertFailsWith { reopened.readPage(id) } + reopened.close() + } + + @Test + fun readingUnallocatedPageIsRejected() { + val manager = DiskManager(tempFile()) + assertFailsWith { manager.readPage(0) } + manager.close() + } +} diff --git a/src/test/kotlin/kestrel/store/page/PageTest.kt b/src/test/kotlin/kestrel/store/page/PageTest.kt new file mode 100644 index 0000000..f20d1e7 --- /dev/null +++ b/src/test/kotlin/kestrel/store/page/PageTest.kt @@ -0,0 +1,29 @@ +package kestrel.store.page + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class PageTest { + @Test + fun freshPageIsZeroedAndClean() { + val page = Page(3) + assertEquals(3, page.id) + assertEquals(Page.PAGE_SIZE, page.data.size) + assertTrue(page.data.all { it == 0.toByte() }) + assertFalse(page.isDirty) + assertEquals(0, page.pinCount) + } + + @Test + fun resetClearsDataAndDirtyFlagButKeepsId() { + val page = Page(1) + page.data[0] = 5 + page.isDirty = true + page.reset() + assertTrue(page.data.all { it == 0.toByte() }) + assertFalse(page.isDirty) + assertEquals(1, page.id) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DirectoryEntryCodecTest.kt b/src/test/kotlin/kestrel/store/sql/DirectoryEntryCodecTest.kt new file mode 100644 index 0000000..3a7a3f3 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DirectoryEntryCodecTest.kt @@ -0,0 +1,167 @@ +package kestrel.store.sql + +import kestrel.catalog.Column +import kestrel.catalog.UniqueConstraint +import kestrel.storage.ColumnType +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class DirectoryEntryCodecTest { + @Test + fun roundTripsATableWithNoIndexes() { + val entry = TableDirectoryEntry( + tableName = "widgets", + columns = listOf(Column("id", ColumnType.Integer, notNull = true, primaryKey = false)), + dataHeaderPageId = 3L, + primaryKeyIndexRootPageId = null, + secondaryIndexes = emptyList() + ) + assertEquals(entry, DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry))) + } + + @Test + fun roundTripsATableWithAPrimaryKeyAndSecondaryIndexes() { + val entry = TableDirectoryEntry( + tableName = "orders", + columns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("customer", ColumnType.Text, notNull = false, primaryKey = false), + Column("total", ColumnType.Real, notNull = true, primaryKey = false), + Column("shipped", ColumnType.Boolean, notNull = false, primaryKey = false) + ), + dataHeaderPageId = 10L, + primaryKeyIndexRootPageId = 11L, + secondaryIndexes = listOf( + SecondaryIndexEntry("idx_customer", listOf("customer"), 12L), + SecondaryIndexEntry("idx_total", listOf("total"), 13L) + ) + ) + assertEquals(entry, DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry))) + } + + @Test + fun roundTripsDecimalColumnsWithAndWithoutExplicitPrecisionAndDateColumns() { + val entry = TableDirectoryEntry( + tableName = "priced", + columns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("bare", ColumnType.Decimal, notNull = false, primaryKey = false), + Column("scaled", ColumnType.Decimal, notNull = false, primaryKey = false, precision = 12, scale = 4), + Column("placed", ColumnType.Date, notNull = false, primaryKey = false) + ), + dataHeaderPageId = 4L, + primaryKeyIndexRootPageId = 5L, + secondaryIndexes = emptyList() + ) + val decoded = DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry)) + assertEquals(entry, decoded) + assertEquals(null, decoded.columns[1].precision) + assertEquals(12, decoded.columns[2].precision) + assertEquals(4, decoded.columns[2].scale) + } + + @Test + fun roundTripsATableWithACompositePrimaryKeyAndEmptyOnesDoNotAddAnyBytes() { + val withComposite = TableDirectoryEntry( + tableName = "order_lines", + columns = listOf( + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("line_no", ColumnType.Integer, notNull = true, primaryKey = false), + Column("sku", ColumnType.Text, notNull = false, primaryKey = false) + ), + dataHeaderPageId = 7L, + primaryKeyIndexRootPageId = 8L, + secondaryIndexes = emptyList(), + compositePrimaryKey = listOf("order_id", "line_no") + ) + val decoded = DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(withComposite)) + assertEquals(withComposite, decoded) + assertEquals(listOf("order_id", "line_no"), decoded.compositePrimaryKey) + + val withoutComposite = withComposite.copy(compositePrimaryKey = emptyList()) + assertEquals(withoutComposite, DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(withoutComposite))) + } + + @Test + fun roundTripsAUniqueSecondaryIndexFlagAndDeclarativeUniqueConstraints() { + val entry = TableDirectoryEntry( + tableName = "users", + columns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("email", ColumnType.Text, notNull = false, primaryKey = false), + Column("event_id", ColumnType.Integer, notNull = false, primaryKey = false), + Column("user_id", ColumnType.Integer, notNull = false, primaryKey = false) + ), + dataHeaderPageId = 20L, + primaryKeyIndexRootPageId = 21L, + secondaryIndexes = listOf(SecondaryIndexEntry("idx_email", listOf("email"), 22L, unique = true)), + uniqueConstraints = listOf( + UniqueConstraint("uq_email_named", listOf("email")), + UniqueConstraint(null, listOf("event_id", "user_id")) + ) + ) + val decoded = DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry)) + assertEquals(entry, decoded) + assertTrue(decoded.secondaryIndexes.single().unique) + assertEquals(listOf("event_id", "user_id"), decoded.uniqueConstraints[1].columns) + assertEquals(null, decoded.uniqueConstraints[1].name) + } + + @Test + fun aNonUniqueSecondaryIndexAndEmptyUniqueConstraintsDoNotAddSpuriousFlagsOnDecode() { + val entry = TableDirectoryEntry( + tableName = "widgets", + columns = listOf(Column("id", ColumnType.Integer, notNull = true, primaryKey = false)), + dataHeaderPageId = 1L, + primaryKeyIndexRootPageId = null, + secondaryIndexes = listOf(SecondaryIndexEntry("idx_id", listOf("id"), 2L)) + ) + val decoded = DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry)) + assertEquals(false, decoded.secondaryIndexes.single().unique) + assertTrue(decoded.uniqueConstraints.isEmpty()) + } + + @Test + fun roundTripsACompositeMultiColumnSecondaryIndex() { + val entry = TableDirectoryEntry( + tableName = "order_lines", + columns = listOf( + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("sku", ColumnType.Text, notNull = false, primaryKey = false) + ), + dataHeaderPageId = 5L, + primaryKeyIndexRootPageId = null, + secondaryIndexes = listOf(SecondaryIndexEntry("idx_order_sku", listOf("order_id", "sku"), 6L, unique = true)) + ) + val decoded = DirectoryEntryCodec.decode(DirectoryEntryCodec.encode(entry)) + assertEquals(entry, decoded) + assertEquals(listOf("order_id", "sku"), decoded.secondaryIndexes.single().columnNames) + } + + @Test + fun decodingAnUnknownColumnTypeFails() { + val entry = TableDirectoryEntry("t", listOf(Column("c", ColumnType.Integer, false, false)), 0L, null, emptyList()) + val bytes = DirectoryEntryCodec.encode(entry).copyOf() + // Corrupt the type-keyword length byte to force an unrecognised keyword. + var offset = 0 + // skip table name: length-prefix varint then bytes + val nameLength = bytes[offset].toInt() and 0x7F + offset += 1 + nameLength + offset += 1 // column count varint (small) + val colNameLength = bytes[offset].toInt() and 0x7F + offset += 1 + colNameLength + val typeLength = bytes[offset].toInt() and 0x7F + for (i in 0 until typeLength) bytes[offset + 1 + i] = 'Z'.code.toByte() + assertThrowsCodecException { DirectoryEntryCodec.decode(bytes) } + } + + private fun assertThrowsCodecException(block: () -> Unit) { + try { + block() + throw AssertionError("expected decode to fail") + } catch (e: kestrel.store.codec.CodecException) { + // expected + } + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskCatalogTest.kt b/src/test/kotlin/kestrel/store/sql/DiskCatalogTest.kt new file mode 100644 index 0000000..1238a4c --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskCatalogTest.kt @@ -0,0 +1,353 @@ +package kestrel.store.sql + +import kestrel.catalog.Column +import kestrel.catalog.SequenceDefinition +import kestrel.catalog.UniqueConstraint +import kestrel.storage.ColumnType +import kestrel.storage.Row +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class DiskCatalogTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private val usersColumns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("name", ColumnType.Text, notNull = true, primaryKey = false) + ) + + @Test + fun creatingTheSameTableTwiceFails() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + catalog.createTable("users", usersColumns) + assertFailsWith { catalog.createTable("users", usersColumns) } + } + + @Test + fun droppedTableIsNoLongerListedOrQueryable() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + catalog.createTable("users", usersColumns) + catalog.dropTable("users") + + assertTrue(catalog.tableNames.isEmpty()) + assertFailsWith { catalog.table("users") } + } + + @Test + fun reopeningAFreshProcessSeesTablesAndRowsThroughNewObjectsEntirely() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", usersColumns) + val txn1 = catalog1.beginTransaction() + table1.insert(txn1, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice")))) + table1.insert(txn1, Row(listOf(Value.IntegerValue(2), Value.TextValue("bob")))) + catalog1.commit(txn1) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + assertEquals(listOf("users"), catalog2.tableNames) + val table2 = catalog2.table("users") + val reader = catalog2.beginTransaction() + assertEquals(setOf(1L, 2L), table2.scan(reader).map { (it.second.values[0] as Value.IntegerValue).value }.toSet()) + val (_, aliceRow) = table2.findByPrimaryKey(reader, Value.IntegerValue(1))!! + assertEquals("alice", (aliceRow.values[1] as Value.TextValue).value) + catalog2.commit(reader) + } + + @Test + fun aCommittedWriteThatNeverFlushedTheBufferPoolSurvivesAReopen() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", usersColumns) + val txn = catalog1.beginTransaction() + table1.insert(txn, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice")))) + catalog1.commit(txn) + // No catalog1.close(): the dirty pages may still only live in the buffer pool. + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("users") + val reader = catalog2.beginTransaction() + assertEquals("alice", table2.findByPrimaryKey(reader, Value.IntegerValue(1))!!.second.values[1].let { (it as Value.TextValue).value }) + catalog2.commit(reader) + } + + @Test + fun aPrimaryKeyIndexThatSplitsSurvivesAReopenWithTheCorrectRoot() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", usersColumns) + val txn = catalog1.beginTransaction() + for (id in 1..300L) table1.insert(txn, Row(listOf(Value.IntegerValue(id), Value.TextValue("user$id")))) + catalog1.commit(txn) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("users") + val reader = catalog2.beginTransaction() + assertEquals(300, table2.scan(reader).size) + for (id in listOf(1L, 150L, 300L)) { + assertEquals("user$id", table2.findByPrimaryKey(reader, Value.IntegerValue(id))!!.second.values[1].let { (it as Value.TextValue).value }) + } + val range = table2.rangeByPrimaryKey(reader, Value.IntegerValue(100), true, Value.IntegerValue(105), true) + assertEquals((100..105L).toList(), range.map { (it.second.values[0] as Value.IntegerValue).value }) + catalog2.commit(reader) + } + + @Test + fun aSecondaryIndexCreatedBeforeCloseSurvivesAReopen() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + val columns = usersColumns + Column("dept", ColumnType.Text, notNull = false, primaryKey = false) + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", columns) + catalog1.createIndex("users", "idx_dept", "dept") + val txn = catalog1.beginTransaction() + table1.insert(txn, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice"), Value.TextValue("eng")))) + table1.insert(txn, Row(listOf(Value.IntegerValue(2), Value.TextValue("bob"), Value.TextValue("eng")))) + catalog1.commit(txn) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("users") + val reader = catalog2.beginTransaction() + val engineers = table2.findBySecondary(reader, "idx_dept", Value.TextValue("eng")) + assertEquals(setOf("alice", "bob"), engineers.map { (it.second.values[1] as Value.TextValue).value }.toSet()) + catalog2.commit(reader) + } + + private val orderLineColumns = listOf( + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("line_no", ColumnType.Integer, notNull = true, primaryKey = false), + Column("sku", ColumnType.Text, notNull = false, primaryKey = false) + ) + + @Test + fun aTableCannotDeclareBothAnInlinePrimaryKeyColumnAndACompositePrimaryKey() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + assertFailsWith { + catalog.createTable("users", usersColumns, compositePrimaryKey = listOf("name")) + } + } + + @Test + fun aCompositePrimaryKeyReferencingAnUnknownColumnIsRejected() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + assertFailsWith { + catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "missing")) + } + } + + @Test + fun aCompositePrimaryKeyListingTheSameColumnTwiceIsRejected() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + assertFailsWith { + catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "order_id")) + } + } + + @Test + fun aCompositePrimaryKeyThatSplitsSurvivesAReopenWithTheCorrectRootAndPointLookupsStillWork() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "line_no")) + val txn = catalog1.beginTransaction() + for (orderId in 1..50L) { + for (lineNo in 1..6L) { + table1.insert(txn, Row(listOf(Value.IntegerValue(orderId), Value.IntegerValue(lineNo), Value.TextValue("sku-$orderId-$lineNo")))) + } + } + catalog1.commit(txn) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("order_lines") + val reader = catalog2.beginTransaction() + assertEquals(300, table2.scan(reader).size) + val found = table2.findByPrimaryKey(reader, listOf(Value.IntegerValue(25), Value.IntegerValue(4)))!! + assertEquals("sku-25-4", (found.second.values[2] as Value.TextValue).value) + assertEquals(null, table2.findByPrimaryKey(reader, listOf(Value.IntegerValue(25), Value.IntegerValue(99)))) + catalog2.commit(reader) + } + + @Test + fun anAbortedTransactionsWritesAreInvisibleAfterAReopen() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", usersColumns) + val committed = catalog1.beginTransaction() + table1.insert(committed, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice")))) + catalog1.commit(committed) + + val doomed = catalog1.beginTransaction() + table1.insert(doomed, Row(listOf(Value.IntegerValue(2), Value.TextValue("bob")))) + catalog1.abort(doomed) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("users") + val reader = catalog2.beginTransaction() + assertEquals(listOf(1L), table2.scan(reader).map { (it.second.values[0] as Value.IntegerValue).value }) + catalog2.commit(reader) + } + + @Test + fun aUniqueConstraintReferencingAnUnknownColumnIsRejected() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + assertFailsWith { + catalog.createTable("users", usersColumns, uniqueConstraints = listOf(UniqueConstraint(null, listOf("missing")))) + } + } + + @Test + fun aUniqueIndexAndItsEnforcementSurviveAReopen() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + val columns = usersColumns + Column("email", ColumnType.Text, notNull = false, primaryKey = false) + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("users", columns) + catalog1.createIndex("users", "idx_email", "email", unique = true) + val txn = catalog1.beginTransaction() + table1.insert(txn, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice"), Value.TextValue("a@x.com")))) + catalog1.commit(txn) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("users") + assertTrue(table2.secondaryIndexes.single().unique) + val reader = catalog2.beginTransaction() + assertFailsWith { + table2.insert(reader, Row(listOf(Value.IntegerValue(2), Value.TextValue("bob"), Value.TextValue("a@x.com")))) + } + catalog2.abort(reader) + } + + @Test + fun aUniqueIndexCreatedOverPreExistingDuplicatesFailsWithoutLeavingAPartialDirectoryEntry() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + val columns = usersColumns + Column("email", ColumnType.Text, notNull = false, primaryKey = false) + + val catalog = DiskCatalog.open(dataFile, walFile) + val table = catalog.createTable("users", columns) + val txn = catalog.beginTransaction() + table.insert(txn, Row(listOf(Value.IntegerValue(1), Value.TextValue("alice"), Value.TextValue("dup@x.com")))) + table.insert(txn, Row(listOf(Value.IntegerValue(2), Value.TextValue("bob"), Value.TextValue("dup@x.com")))) + catalog.commit(txn) + + assertFailsWith { catalog.createIndex("users", "idx_email", "email", unique = true) } + assertTrue(table.secondaryIndexes.isEmpty()) + + catalog.close() + val reopened = DiskCatalog.open(dataFile, walFile) + assertTrue(reopened.table("users").secondaryIndexes.isEmpty()) + } + + @Test + fun aCompositeUniqueIndexAndItsEnforcementSurviveAReopenWithPointLookupsStillWorking() { + val dataFile = tempFile("kestrel-cat-data") + val walFile = tempFile("kestrel-cat-wal") + + val catalog1 = DiskCatalog.open(dataFile, walFile) + val table1 = catalog1.createTable("order_lines", orderLineColumns) + catalog1.createIndex("order_lines", "idx_order_sku", listOf("order_id", "sku"), unique = true) + val txn = catalog1.beginTransaction() + table1.insert(txn, Row(listOf(Value.IntegerValue(1), Value.IntegerValue(1), Value.TextValue("widget")))) + catalog1.commit(txn) + catalog1.close() + + val catalog2 = DiskCatalog.open(dataFile, walFile) + val table2 = catalog2.table("order_lines") + val handle = table2.secondaryIndexes.single() + assertEquals(listOf("order_id", "sku"), handle.columnNames) + assertTrue(handle.unique) + + val reader = catalog2.beginTransaction() + assertFailsWith { + table2.insert(reader, Row(listOf(Value.IntegerValue(1), Value.IntegerValue(2), Value.TextValue("widget")))) + } + val found = table2.findBySecondary(reader, "idx_order_sku", listOf(Value.IntegerValue(1), Value.TextValue("widget"))) + assertEquals(1, found.size) + catalog2.abort(reader) + } + + private val wideTextPadding = "x".repeat(200) + + private fun DiskCatalog.buildWideTable(name: String): DiskTable { + val table = createTable(name, usersColumns) + createIndex(name, "idx_${name}_name", "name") + val txn = beginTransaction() + for (id in 1..60L) table.insert(txn, Row(listOf(Value.IntegerValue(id), Value.TextValue("$wideTextPadding-$id")))) + commit(txn) + return table + } + + @Test + fun droppingATableReclaimsItsPagesForANewTableInsteadOfGrowingTheFile() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + catalog.buildWideTable("wide") + val peak = catalog.allocatedPageCount() + + catalog.dropTable("wide") + catalog.buildWideTable("wide2") + + assertTrue( + catalog.allocatedPageCount() <= peak, + "expected the dropped table's reclaimed heap/index pages to be reused by the new table instead of " + + "growing the file further (peak=$peak, now=${catalog.allocatedPageCount()})" + ) + } + + @Test + fun droppingASecondaryIndexReclaimsItsPagesForANewIndexOnTheSameTable() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + val table = catalog.createTable("wide", usersColumns) + val txn = catalog.beginTransaction() + for (id in 1..60L) table.insert(txn, Row(listOf(Value.IntegerValue(id), Value.TextValue("$wideTextPadding-$id")))) + catalog.commit(txn) + catalog.createIndex("wide", "idx_name", "name") + val peak = catalog.allocatedPageCount() + + catalog.dropIndex("idx_name") + catalog.createIndex("wide", "idx_name_2", "name") + + assertTrue( + catalog.allocatedPageCount() <= peak, + "expected the dropped index's reclaimed pages to be reused by the new index instead of growing the " + + "file further (peak=$peak, now=${catalog.allocatedPageCount()})" + ) + assertEquals("idx_name_2", table.secondaryIndexes.single().name) + } + + @Test + fun droppingASequenceReclaimsItsCounterPageForANewSequence() { + val catalog = DiskCatalog.open(tempFile("kestrel-cat-data"), tempFile("kestrel-cat-wal")) + catalog.createSequence(SequenceDefinition("seq1", incrementBy = 1, minValue = 1, maxValue = Long.MAX_VALUE, startValue = 1, cycle = false)) + val peak = catalog.allocatedPageCount() + + catalog.dropSequence("seq1") + catalog.createSequence(SequenceDefinition("seq2", incrementBy = 1, minValue = 1, maxValue = Long.MAX_VALUE, startValue = 1, cycle = false)) + + assertTrue( + catalog.allocatedPageCount() <= peak, + "expected the dropped sequence's counter page to be reused instead of growing the file further " + + "(peak=$peak, now=${catalog.allocatedPageCount()})" + ) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorChangeColumnTypeTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorChangeColumnTypeTest.kt new file mode 100644 index 0000000..ca170d8 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorChangeColumnTypeTest.kt @@ -0,0 +1,246 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.AlterTableResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.math.BigDecimal +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `ALTER TABLE t ALTER COLUMN col TYPE ...` on [DiskExecutor] - mirrors + * [kestrel.AlterTableTest]'s in-memory `ChangeColumnType` cases. Built on [DiskTable.rewriteRows] + * exactly like `ADD COLUMN` (see [DiskCatalog.changeColumnType]'s own doc): every row's stored + * bytes are rewritten in place under a single WAL transaction, so a NOT NULL/CHECK failure + * discovered partway through a multi-row rewrite aborts the whole transaction rather than leaving + * some rows converted and others not - the disk-specific angle [kestrel.AlterTableTest] cannot + * exercise at all, since the in-memory engine has no WAL to roll back. + */ +class DiskExecutorChangeColumnTypeTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private fun real(value: Double) = Value.RealValue(value) + private fun decimal(value: String) = Value.DecimalValue(BigDecimal(value)) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + + /** Closes and reopens the catalog against the same files, simulating a fresh process. */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-alter-type-data"), tempFile("kestrel-disk-alter-type-wal")) + + @Test + fun changeColumnTypeWideningIntegerToRealCoercesExistingRowsWithNoExplicitUsing() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, score INTEGER)") + db.run("INSERT INTO t VALUES (1, 10), (2, 20)") + val result = db.exec("ALTER TABLE t ALTER COLUMN score TYPE REAL") + assertEquals(AlterTableResult("t"), result) + assertEquals(listOf(listOf(real(10.0)), listOf(real(20.0))), db.select("SELECT score FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeWideningIntegerToDecimalCoercesExistingRows() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("INSERT INTO t VALUES (1, 42)") + db.exec("ALTER TABLE t ALTER COLUMN amount TYPE DECIMAL(10, 2)") + assertEquals(listOf(listOf(decimal("42.00"))), db.select("SELECT amount FROM t").rows) + } + + @Test + fun changeColumnTypeWithUsingConvertsTextToIntegerViaCast() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT)") + db.run("INSERT INTO t VALUES (1, '100'), (2, '200')") + db.exec("ALTER TABLE t ALTER COLUMN code TYPE INTEGER USING CAST(code AS INTEGER)") + assertEquals(listOf(listOf(int(100)), listOf(int(200))), db.select("SELECT code FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeWithUsingCanReferenceTheRowsOtherColumns() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, price INTEGER, quantity INTEGER)") + db.run("INSERT INTO t VALUES (1, 10, 3)") + db.exec("ALTER TABLE t ALTER COLUMN price TYPE INTEGER USING price * quantity") + assertEquals(listOf(listOf(int(30))), db.select("SELECT price FROM t").rows) + } + + @Test + fun changeColumnTypeRejectsImplicitCoercionWhenTypesAreIncompatibleAndNoUsingIsGiven() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT)") + db.run("INSERT INTO t VALUES (1, 'abc')") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN code TYPE INTEGER") } + assertTrue(error.message!!.contains("type mismatch"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsUnknownColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN missing TYPE TEXT") } + assertTrue(error.message!!.contains("no such column"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, price INTEGER, doubled INTEGER GENERATED ALWAYS AS (price * 2) STORED)") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN doubled TYPE REAL") } + assertTrue(error.message!!.contains("GENERATED"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsPrimaryKeyColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN id TYPE TEXT") } + assertTrue(error.message!!.contains("PRIMARY KEY"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnUsedByItsOwnForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, FOREIGN KEY (dept_id) REFERENCES departments (id))") + val error = assertFailsWith { db.run("ALTER TABLE employees ALTER COLUMN dept_id TYPE TEXT") } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnReferencedByAnotherTablesForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT, CONSTRAINT uq_code UNIQUE (code))") + db.run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT, " + + "FOREIGN KEY (dept_code) REFERENCES departments (code))" + ) + val error = assertFailsWith { db.run("ALTER TABLE departments ALTER COLUMN code TYPE INTEGER") } + assertTrue(error.message!!.contains("employees"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnUsedByAUniqueConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT, CONSTRAINT uq_email UNIQUE (email))") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN email TYPE TEXT") } + assertTrue(error.message!!.contains("UNIQUE"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsColumnWithASecondaryIndex() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("CREATE INDEX idx_name ON t (name)") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN name TYPE TEXT") } + assertTrue(error.message!!.contains("idx_name"), error.message!!) + } + + @Test + fun changeColumnTypeRejectsWhenUsingProducesNullForANotNullColumnAndLeavesTheTableUnchanged() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, code TEXT NOT NULL)") + db.run("INSERT INTO t VALUES (1, '100'), (2, '')") + val error = assertFailsWith { + db.run("ALTER TABLE t ALTER COLUMN code TYPE INTEGER USING CASE WHEN code = '' THEN NULL ELSE CAST(code AS INTEGER) END") + } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + assertEquals(listOf(listOf(text("100")), listOf(text(""))), db.select("SELECT code FROM t ORDER BY id ASC").rows) + } + + /** + * The disk-specific proof [kestrel.AlterTableTest] cannot give: row 1's converted value (20) + * would already have been written to its heap page by [DiskTable.rewriteRows] before row 2's + * CHECK violation is discovered and thrown - if the surrounding transaction were not aborted, + * row 1 would be left silently converted while row 2 was not. Asserting BOTH rows still hold + * their ORIGINAL, unconverted values after the failure is the proof that [DiskCatalog.changeColumnType]'s + * `catch { engine.abort(txn) }` really unwinds a partial per-row rewrite already applied to disk. + */ + @Test + fun changeColumnTypeRevalidatesACheckConstraintAgainstTheNewValuesAndLeavesTheTableUnchangedOnFailure() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER, CHECK (amount <= 100))") + db.run("INSERT INTO t VALUES (1, 10), (2, 60)") + val error = assertFailsWith { + db.run("ALTER TABLE t ALTER COLUMN amount TYPE INTEGER USING amount * 2") + } + assertTrue(error.message!!.contains("CHECK constraint"), error.message!!) + assertEquals( + listOf(listOf(int(1), int(10)), listOf(int(2), int(60))), + db.select("SELECT id, amount FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun changeColumnTypeSucceedsAndAppliesUsingToEveryRowWhenAllSatisfyAnExistingCheckConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER, CHECK (amount <= 100))") + db.run("INSERT INTO t VALUES (1, 10), (2, 20)") + db.exec("ALTER TABLE t ALTER COLUMN amount TYPE INTEGER USING amount * 2") + assertEquals(listOf(listOf(int(20)), listOf(int(40))), db.select("SELECT amount FROM t ORDER BY id ASC").rows) + } + + @Test + fun changeColumnTypeRejectsAnIncompatibleDefaultExpressionUpFront() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT DEFAULT 'pending')") + val error = assertFailsWith { db.run("ALTER TABLE t ALTER COLUMN status TYPE INTEGER USING 0") } + assertTrue(error.message!!.contains("DEFAULT"), error.message!!) + } + + @Test + fun changeColumnTypeAcceptsAnOptionalColumnKeyword() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("INSERT INTO t VALUES (1, 5)") + db.exec("ALTER TABLE t ALTER amount TYPE REAL") + assertEquals(listOf(listOf(real(5.0))), db.select("SELECT amount FROM t").rows) + } + + /** The type change - and the converted data - must survive a full process close/reopen, not just live in the still-open transaction's page cache. */ + @Test + fun changeColumnTypeSurvivesReopen() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("INSERT INTO t VALUES (1, 42)") + db.exec("ALTER TABLE t ALTER COLUMN amount TYPE DECIMAL(10, 2)") + db.reopen() + assertEquals(listOf(listOf(decimal("42.00"))), db.select("SELECT amount FROM t").rows) + val error = assertFailsWith { db.run("INSERT INTO t VALUES (2, 'not-a-number')") } + assertTrue(error.message!!.contains("type mismatch"), error.message!!) + } + + @Test + fun changeColumnTypePreservesNotNullAndStillEnforcesItGoingForward() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER NOT NULL)") + db.run("INSERT INTO t VALUES (1, 10)") + db.exec("ALTER TABLE t ALTER COLUMN amount TYPE REAL") + val error = assertFailsWith { db.run("INSERT INTO t VALUES (2, NULL)") } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorCollateTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorCollateTest.kt new file mode 100644 index 0000000..5d302e4 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorCollateTest.kt @@ -0,0 +1,105 @@ +package kestrel.store.sql + +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * `expr COLLATE BINARY|NOCASE|RTRIM` against the disk engine, mirroring a subset of + * [kestrel.CollateTest]'s in-memory scenarios (equality, RTRIM, ORDER BY, IN) to prove the shared + * [kestrel.engine.Evaluator]/[kestrel.engine.compareOrderKeyValues] code [kestrel.store.sql. + * DiskExecutor] reuses directly behaves identically here, plus one disk-only scenario: a + * COLLATE-bearing WHERE clause typed straight into the disk engine (never routed through + * [kestrel.persistence.SqlWriter] - the disk engine has no text WAL to round-trip through) keeps + * comparing the same way after a real file close/reopen. + */ +class DiskExecutorCollateTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + private fun bool(value: Boolean) = Value.BooleanValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + var executor = DiskExecutor(catalog) + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-collate-data"), tempFile("kestrel-disk-collate-wal")) + + private fun Database.seedNames() { + run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + run("INSERT INTO people VALUES (1, 'Bob'), (2, 'bob'), (3, 'Alice'), (4, 'alice '), (5, 'Zoe')") + } + + @Test + fun equalityWithNocaseIgnoresAsciiCaseButPlainEqualityDoesNot() { + val db = newDatabase() + db.seedNames() + assertEquals(bool(true), db.select("SELECT name = 'BOB' COLLATE NOCASE FROM people WHERE id = 1").rows.single().single()) + assertEquals(bool(false), db.select("SELECT name = 'BOB' FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun rtrimIgnoresOnlyTrailingSpaces() { + val db = newDatabase() + db.seedNames() + assertEquals(bool(true), db.select("SELECT name = 'alice' COLLATE RTRIM FROM people WHERE id = 4").rows.single().single()) + assertEquals(bool(false), db.select("SELECT name = 'alice' FROM people WHERE id = 4").rows.single().single()) + } + + @Test + fun orderByCollateNocaseInterleavesUpperAndLowerCaseNames() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people ORDER BY name COLLATE NOCASE, id") + assertEquals(listOf("Alice", "alice ", "Bob", "bob", "Zoe").map { listOf(text(it)) }, result.rows) + } + + @Test + fun inListHonorsTheOperandsExplicitCollate() { + val db = newDatabase() + db.seedNames() + assertEquals( + bool(true), + db.select("SELECT name COLLATE NOCASE IN ('BOB', 'other') FROM people WHERE id = 1").rows.single().single() + ) + assertEquals( + bool(false), + db.select("SELECT name IN ('BOB', 'other') FROM people WHERE id = 1").rows.single().single() + ) + } + + @Test + fun aCollateComparisonKeepsComparingTheSameWayAfterARealFileCloseAndReopen() { + val db = newDatabase() + db.seedNames() + assertEquals( + listOf(int(1), int(2)), + db.select("SELECT id FROM people WHERE name COLLATE NOCASE = 'BOB' ORDER BY id").rows.map { it.single() } + ) + db.reopen() + assertEquals( + listOf(int(1), int(2)), + db.select("SELECT id FROM people WHERE name COLLATE NOCASE = 'BOB' ORDER BY id").rows.map { it.single() } + ) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorColumnCollationTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorColumnCollationTest.kt new file mode 100644 index 0000000..a89a8c6 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorColumnCollationTest.kt @@ -0,0 +1,558 @@ +package kestrel.store.sql + +import kestrel.engine.ExplainResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * A column's own declared `COLLATE` (see [kestrel.catalog.Column.collation]) against the disk + * engine, mirroring a subset of [kestrel.ColumnCollationTest]'s in-memory scenarios to prove the + * same [kestrel.engine.Evaluator]/[kestrel.engine.ColumnResolver.declaredCollation] machinery + * [DiskExecutor] reuses behaves identically here, plus the one thing that only matters for this + * engine: the declared collation is stored as a byte in [DirectoryEntryCodec]'s own binary schema + * encoding (not this engine's text dump, which does not exist here), so it must survive a real + * file close/reopen, not just staying alive in the same [DiskCatalog] instance. + */ +class DiskExecutorColumnCollationTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun bool(value: Boolean) = Value.BooleanValue(value) + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + var executor = DiskExecutor(catalog) + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-column-collation-data"), tempFile("kestrel-disk-column-collation-wal")) + + private fun Database.seedNames() { + run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + run("INSERT INTO people VALUES (1, 'Bob'), (2, 'bob'), (3, 'Alice'), (4, 'Zoe')") + } + + @Test + fun bareEqualityHonorsTheColumnsDeclaredCollationWithoutAnExplicitClause() { + val db = newDatabase() + db.seedNames() + assertEquals( + listOf(int(1), int(2)), + db.select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows.map { it.single() } + ) + } + + @Test + fun explicitCollateInTheQueryOverridesTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + assertEquals(bool(false), db.select("SELECT name COLLATE BINARY = 'BOB' FROM people WHERE id = 1").rows.single().single()) + } + + @Test + fun columnWithNoDeclaredCollationStaysBinaryByDefault() { + val db = newDatabase() + db.run("CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO plain VALUES (1, 'Bob')") + assertEquals(bool(false), db.select("SELECT name = 'BOB' FROM plain WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInsideAnOnConflictDoUpdateAssignment() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, was_bob BOOLEAN)") + db.run("INSERT INTO t VALUES (1, 'Bob', FALSE)") + db.run("INSERT INTO t (id, name) VALUES (1, 'x') ON CONFLICT (id) DO UPDATE SET was_bob = (name = 'BOB')") + assertEquals(bool(true), db.select("SELECT was_bob FROM t WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInAMergeOnCondition() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, hits INTEGER)") + db.run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO accounts VALUES (1, 'Bob', 0)") + db.run("INSERT INTO staging VALUES (2, 'BOB')") + db.run("MERGE INTO accounts USING staging ON accounts.name = staging.name WHEN MATCHED THEN UPDATE SET hits = hits + 1") + assertEquals(int(1), db.select("SELECT hits FROM accounts WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesInATriggerWhenClauseOnNew() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + db.run("CREATE TABLE hits (id INTEGER PRIMARY KEY, count INTEGER)") + db.run("INSERT INTO t VALUES (1, 'Bob')") + db.run("INSERT INTO hits VALUES (1, 0)") + db.run( + "CREATE TRIGGER bump AFTER UPDATE ON t FOR EACH ROW WHEN NEW.name = 'BOB' " + + "BEGIN UPDATE hits SET count = count + 100 WHERE id = NEW.id; END" + ) + db.run("UPDATE t SET name = 'bob' WHERE id = 1") + assertEquals(int(100), db.select("SELECT count FROM hits WHERE id = 1").rows.single().single()) + } + + @Test + fun declaredCollationAppliesToDefaultOrderBySortOrder() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people ORDER BY name") + assertEquals(listOf("Alice", "Bob", "bob", "Zoe").map { listOf(Value.TextValue(it)) }, result.rows) + } + + @Test + fun explicitCollateInOrderByOverridesTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people ORDER BY name COLLATE BINARY") + assertEquals(listOf("Alice", "Bob", "Zoe", "bob").map { listOf(Value.TextValue(it)) }, result.rows) + } + + @Test + fun declaredCollationAppliesInAWindowFunctionsOrderBy() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name, ROW_NUMBER() OVER (ORDER BY name) AS rn FROM people ORDER BY rn") + assertEquals( + listOf("Alice" to 1L, "Bob" to 2L, "bob" to 3L, "Zoe" to 4L).map { (name, rn) -> listOf(Value.TextValue(name), int(rn)) }, + result.rows + ) + } + + /** Mirrors [kestrel.ColumnCollationTest]'s set-operation ORDER BY cases against the disk engine. */ + @Test + fun declaredCollationAppliesToASetOperationsOrderByThroughAStarProjection() { + val db = newDatabase() + db.seedNames() + val result = db.select( + "SELECT * FROM people WHERE id <= 2 UNION ALL SELECT * FROM people WHERE id > 2 ORDER BY name" + ) + assertEquals( + listOf(int(3) to "Alice", int(1) to "Bob", int(2) to "bob", int(4) to "Zoe") + .map { (id, name) -> listOf(id, Value.TextValue(name)) }, + result.rows + ) + } + + @Test + fun explicitCollateInASetOperationsOrderByOverridesTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select( + "SELECT * FROM people WHERE id <= 2 UNION ALL SELECT * FROM people WHERE id > 2 ORDER BY name COLLATE BINARY" + ) + assertEquals(listOf("Alice", "Bob", "Zoe", "bob"), result.rows.map { (it[1] as Value.TextValue).value }) + } + + @Test + fun declaredCollationDoesNotReachThroughADerivedTableInASetOperationsLeftBranch() { + val db = newDatabase() + db.seedNames() + val result = db.select( + "SELECT * FROM (SELECT * FROM people) sub WHERE id <= 2 " + + "UNION ALL SELECT * FROM (SELECT * FROM people) sub2 WHERE id > 2 " + + "ORDER BY name" + ) + assertEquals(listOf("Alice", "Bob", "Zoe", "bob"), result.rows.map { (it[1] as Value.TextValue).value }) + } + + /** Mirrors [kestrel.ColumnCollationTest.groupByDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun groupByDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name, COUNT(*) FROM people GROUP BY name ORDER BY name") + assertEquals( + listOf("Alice" to 1L, "Bob" to 2L, "Zoe" to 1L).map { (name, count) -> listOf(Value.TextValue(name), int(count)) }, + result.rows + ) + } + + /** Mirrors [kestrel.ColumnCollationTest.selectDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun selectDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT DISTINCT name FROM people ORDER BY name") + assertEquals(listOf("Alice", "Bob", "Zoe").map { listOf(Value.TextValue(it)) }, result.rows) + } + + /** Mirrors [kestrel.ColumnCollationTest.selectDistinctStarDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun selectDistinctStarDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE tags (label TEXT COLLATE NOCASE)") + db.run("INSERT INTO tags VALUES ('Bob'), ('bob'), ('Alice')") + val result = db.select("SELECT DISTINCT * FROM tags ORDER BY label") + assertEquals(listOf("Alice", "Bob").map { listOf(Value.TextValue(it)) }, result.rows) + } + + /** Mirrors [kestrel.ColumnCollationTest.selectDistinctOnDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun selectDistinctOnDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE, hits INTEGER)") + db.run("INSERT INTO people VALUES (1, 'Bob', 10), (2, 'bob', 20), (3, 'Alice', 5)") + val result = db.select("SELECT DISTINCT ON (name) name, hits FROM people ORDER BY name, id") + assertEquals( + listOf("Alice" to 5L, "Bob" to 10L).map { (name, hits) -> listOf(Value.TextValue(name), int(hits)) }, + result.rows + ) + } + + /** Mirrors [kestrel.ColumnCollationTest.countDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun countDistinctDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + assertEquals(int(3), db.select("SELECT COUNT(DISTINCT name) FROM people").rows.single().single()) + } + + /** Mirrors [kestrel.ColumnCollationTest.plainUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun plainUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people WHERE id = 1 UNION SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(Value.TextValue("Bob"))), result.rows) + } + + /** Mirrors [kestrel.ColumnCollationTest.intersectMatchesCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun intersectMatchesCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people WHERE id = 1 INTERSECT SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(Value.TextValue("Bob"))), result.rows) + } + + /** Mirrors [kestrel.ColumnCollationTest.exceptExcludesCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun exceptExcludesCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val result = db.select("SELECT name FROM people WHERE id IN (1, 4) EXCEPT SELECT name FROM people WHERE id = 2") + assertEquals(listOf(listOf(Value.TextValue("Zoe"))), result.rows) + } + + /** Mirrors [kestrel.ColumnCollationTest.recursiveCteWithUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation]. */ + @Test + fun recursiveCteWithUnionDedupsCaseVariantsThroughTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE origin (name TEXT COLLATE NOCASE)") + db.run("INSERT INTO origin VALUES ('Bob')") + db.run("CREATE TABLE steps (name TEXT COLLATE NOCASE, next_name TEXT COLLATE NOCASE)") + db.run("INSERT INTO steps VALUES ('Bob', 'Carol'), ('Carol', 'bob')") + val result = db.select( + "WITH RECURSIVE walk(name) AS (" + + "SELECT name FROM origin " + + "UNION " + + "SELECT steps.next_name FROM steps JOIN walk ON steps.name = walk.name" + + ") SELECT name FROM walk ORDER BY name" + ) + assertEquals(listOf(listOf(Value.TextValue("Bob")), listOf(Value.TextValue("Carol"))), result.rows) + } + + /** + * [DiskTable.rejectDuplicateUniqueConstraints] (declarative `UNIQUE`, heap-scanned on every + * write, no backing index) folded through [kestrel.engine.collationFoldedUniquenessKey] - + * mirrors [kestrel.ColumnCollationTest.uniqueConstraintHonorsTheColumnsDeclaredCollation]. + */ + @Test + fun declarativeUniqueConstraintHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE u1 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE UNIQUE)") + db.run("INSERT INTO u1 VALUES (1, 'Bob')") + assertFailsWith { db.run("INSERT INTO u1 VALUES (2, 'bob')") } + } + + /** Control negative: a `UNIQUE` column with no declared `COLLATE` behaves exactly as before this session. */ + @Test + fun declarativeUniqueConstraintWithNoDeclaredCollationStaysBinary() { + val db = newDatabase() + db.run("CREATE TABLE u3 (id INTEGER PRIMARY KEY, name TEXT UNIQUE)") + db.run("INSERT INTO u3 VALUES (1, 'Bob')") + db.run("INSERT INTO u3 VALUES (2, 'bob')") + assertEquals(2, db.select("SELECT id FROM u3").rows.size) + } + + /** + * [DiskTable.backfillSecondaryIndex]'s pre-existing-duplicate check, also folded through + * [kestrel.engine.collationFoldedUniquenessKey] - two rows already differing only by case + * under a `NOCASE`-declared column must now fail `CREATE UNIQUE INDEX` cleanly instead of + * silently building an index that lies about uniqueness. + */ + @Test + fun createUniqueIndexHonorsTheColumnsDeclaredCollationAgainstPreexistingRows() { + val db = newDatabase() + db.run("CREATE TABLE u2 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + db.run("INSERT INTO u2 VALUES (1, 'Bob'), (2, 'bob')") + assertFailsWith { db.run("CREATE UNIQUE INDEX idxU2 ON u2 (name)") } + } + + @Test + fun aDeclaredColumnCollationKeepsComparingTheSameWayAfterARealFileCloseAndReopen() { + val db = newDatabase() + db.seedNames() + assertEquals( + listOf(int(1), int(2)), + db.select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows.map { it.single() } + ) + db.reopen() + assertEquals( + listOf(int(1), int(2)), + db.select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows.map { it.single() } + ) + } + + /** + * [DiskIndexAccessPlanner]'s equivalent of [kestrel.ColumnCollationTest.indexedEqualityHonorsTheColumnsDeclaredCollation]: + * [DiskTable.findBySecondary] used to encode both the inserted row's key AND the search + * literal straight to bytes via [kestrel.store.codec.KeyCodec], with no regard for the + * column's declared `COLLATE` - so once `CREATE INDEX` made the planner choose an index scan + * over the full scan the un-indexed [bareEqualityHonorsTheColumnsDeclaredCollationWithoutAnExplicitClause] + * exercises, the case-variant row silently dropped out of the result instead of raising an + * error, since [DiskExecutor] only re-checks WHERE against candidates the chosen access plan + * already returned. + */ + @Test + fun indexedEqualityHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.seedNames() + val scanned = db.select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows + db.run("CREATE INDEX idxName ON people (name)") + val plan = db.explain("EXPLAIN SELECT id FROM people WHERE name = 'BOB'") + assertTrue(plan.any { it.contains("Index Scan using idxName") }, "expected index scan, got $plan") + val indexed = db.select("SELECT id FROM people WHERE name = 'BOB' ORDER BY id").rows + assertEquals(scanned, indexed) + assertEquals(listOf(listOf(int(1)), listOf(int(2))), indexed) + } + + /** Mirrors [indexedEqualityHonorsTheColumnsDeclaredCollation] for a trailing-space collation instead of case. */ + @Test + fun indexedEqualityHonorsAnRtrimDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE tags (id INTEGER PRIMARY KEY, tag TEXT COLLATE RTRIM)") + db.run("INSERT INTO tags VALUES (1, 'a'), (2, 'a '), (3, 'b')") + db.run("CREATE INDEX idxTag ON tags (tag)") + val plan = db.explain("EXPLAIN SELECT id FROM tags WHERE tag = 'a'") + assertTrue(plan.any { it.contains("Index Scan using idxTag") }, "expected index scan, got $plan") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), db.select("SELECT id FROM tags WHERE tag = 'a' ORDER BY id").rows) + } + + /** + * Mirrors [kestrel.ColumnCollationTest.compositeIndexEqualityHonorsPerColumnDeclaredCollationIndependently]: + * a composite secondary index's key is `encodeKey(first) ++ encodeKey(last) ++ rid` - each + * column position must fold through its OWN declared collation independently (`first` is + * NOCASE, `last` stays BINARY) rather than, say, folding the whole composite key uniformly or + * not at all. + */ + @Test + fun compositeIndexEqualityHonorsPerColumnDeclaredCollationIndependently() { + val db = newDatabase() + db.run("CREATE TABLE contacts (id INTEGER PRIMARY KEY, first TEXT COLLATE NOCASE, last TEXT)") + db.run("INSERT INTO contacts VALUES (1, 'Bob', 'Smith'), (2, 'bob', 'SMITH')") + db.run("CREATE INDEX idxContact ON contacts (first, last)") + val plan = db.explain("EXPLAIN SELECT id FROM contacts WHERE first = 'BOB' AND last = 'Smith'") + assertTrue(plan.any { it.contains("Index Scan using idxContact") }, "got $plan") + assertEquals( + listOf(listOf(int(1))), + db.select("SELECT id FROM contacts WHERE first = 'BOB' AND last = 'Smith'").rows + ) + } + + /** + * Safety net mirroring [kestrel.ColumnCollationTest.explicitCollateOverrideOnAnIndexedColumnFallsBackToASeqScanInsteadOfMisusingTheIndex]: + * [DiskIndexAccessPlanner.asColumnPredicate] only recognizes a bare [kestrel.sql.ast.ColumnExpression], + * so an explicit `COLLATE`-wrapped operand is excluded from pushdown entirely instead of being + * (wrongly) served by an index whose physical bytes are folded for the column's OWN collation. + */ + @Test + fun explicitCollateOverrideOnAnIndexedColumnFallsBackToASeqScanInsteadOfMisusingTheIndex() { + val db = newDatabase() + db.seedNames() + db.run("CREATE INDEX idxName ON people (name)") + val plan = db.explain("EXPLAIN SELECT id FROM people WHERE name COLLATE BINARY = 'BOB'") + assertTrue(plan.any { it.contains("Seq Scan") }, "expected seq scan fallback, got $plan") + assertEquals(emptyList(), db.select("SELECT id FROM people WHERE name COLLATE BINARY = 'BOB'").rows) + } + + /** Control negative: an index over a column with no declared `COLLATE` behaves exactly as before this session. */ + @Test + fun indexedEqualityWithNoDeclaredCollationStaysBinary() { + val db = newDatabase() + db.run("CREATE TABLE plain3 (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO plain3 VALUES (1, 'Bob'), (2, 'bob')") + db.run("CREATE INDEX idxPlain3 ON plain3 (name)") + val plan = db.explain("EXPLAIN SELECT id FROM plain3 WHERE name = 'Bob'") + assertTrue(plan.any { it.contains("Index Scan using idxPlain3") }, "got $plan") + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM plain3 WHERE name = 'Bob'").rows) + } + + /** + * [DiskTable.findByPrimaryKey]'s point-lookup counterpart of [indexedEqualityHonorsTheColumnsDeclaredCollation]: + * a single-column, inline `PRIMARY KEY` is backed by its own B+Tree exactly like a secondary + * index, so `WHERE name = 'BOB'` against a `NOCASE`-declared `PRIMARY KEY` column must be + * planned as [DiskAccessPlan.PrimaryKeyPoint] and still find the case-variant row. + */ + @Test + fun primaryKeyPointLookupHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE pk1 (name TEXT COLLATE NOCASE PRIMARY KEY, hits INTEGER)") + db.run("INSERT INTO pk1 VALUES ('Bob', 10)") + val plan = db.explain("EXPLAIN SELECT hits FROM pk1 WHERE name = 'BOB'") + assertTrue(plan.any { it.contains("Primary Key Scan") }, "expected primary key point lookup, got $plan") + assertEquals(listOf(listOf(int(10))), db.select("SELECT hits FROM pk1 WHERE name = 'BOB'").rows) + } + + /** + * [encodePrimaryKey]'s composite counterpart of [primaryKeyPointLookupHonorsTheColumnsDeclaredCollation]: + * a table-level composite `PRIMARY KEY (region, code)` folds each position through ITS OWN + * declared collation independently ([DiskAccessPlan.CompositePrimaryKeyPoint] mirrors + * [compositeIndexEqualityHonorsPerColumnDeclaredCollationIndependently] but for the PK's own + * B+Tree rather than a secondary index) - `region` is NOCASE (matches both rows) but `code` + * stays plain BINARY (only an exact-case match counts), so only row 1 survives both conjuncts. + */ + @Test + fun compositePrimaryKeyPointLookupHonorsPerColumnDeclaredCollationIndependently() { + val db = newDatabase() + db.run("CREATE TABLE zones (region TEXT COLLATE NOCASE, code TEXT, hits INTEGER, PRIMARY KEY (region, code))") + db.run("INSERT INTO zones VALUES ('EU', 'west', 1), ('eu', 'WEST', 2)") + val plan = db.explain("EXPLAIN SELECT hits FROM zones WHERE region = 'eu' AND code = 'west'") + assertTrue(plan.any { it.contains("Primary Key Scan") }, "expected composite primary key point lookup, got $plan") + assertEquals(listOf(listOf(int(1))), db.select("SELECT hits FROM zones WHERE region = 'eu' AND code = 'west'").rows) + } + + /** + * [DiskTable.rangeByPrimaryKey]'s composite counterpart of [rangeQueryOnAPrimaryKeyHonorsTheColumnsDeclaredCollation]: + * a table-level composite `PRIMARY KEY (order_id, sku)` ranged on its LAST column (`sku`, the + * only shape [DiskAccessPlan.CompositePrimaryKeyRange] plans - see its own doc) after an + * equality prefix on `order_id` (plain BINARY, irrelevant here) still folds the bound through + * `sku`'s own declared `NOCASE` collation, same as the single-column case - `'Mango'` only + * survives `sku > 'lemon'` once both sides are canonicalized to `'MANGO' > 'LEMON'`. + */ + @Test + fun rangeQueryOnTheLastColumnOfACompositePrimaryKeyAfterAnEqualityPrefixHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE order_lines4 (order_id INTEGER, sku TEXT COLLATE NOCASE, PRIMARY KEY (order_id, sku))") + db.run("INSERT INTO order_lines4 VALUES (1, 'Apple'), (1, 'Mango'), (1, 'kiwi'), (2, 'Zebra')") + val plan = db.explain("EXPLAIN SELECT sku FROM order_lines4 WHERE order_id = 1 AND sku > 'lemon'") + assertTrue(plan.any { it.contains("Primary Key Range Scan") }, "expected primary key range scan, got $plan") + assertEquals( + listOf(listOf(Value.TextValue("Mango"))), + db.select("SELECT sku FROM order_lines4 WHERE order_id = 1 AND sku > 'lemon'").rows + ) + } + + /** + * [DiskTable.rangeByPrimaryKey]'s counterpart of [primaryKeyPointLookupHonorsTheColumnsDeclaredCollation] + * for a range predicate instead of equality: [kestrel.engine.collationFoldedUniquenessKey] folds + * both the query's bound and every stored key through the same declared collation before the + * B+Tree ever compares bytes, so the ordering the index range-scans over agrees with what + * `Evaluator`'s own collation-aware `>`/`<` would decide row-by-row. The literal bound + * (`'ALICE'`, all-caps) is deliberately a different case than the stored `'Alice'` row it must + * exclude, proving the bound itself is folded too, not just the stored keys. `NoCase.canonicalize` + * uppercases, so the bound is deliberately given in lowercase (`'alice'`, not `'ALICE'`) - an + * unfolded lowercase bound would compare as bytewise *greater* than every uppercase-folded + * stored key (lowercase ASCII sorts after uppercase), wrongly excluding every row instead of + * just `'Alice'` itself. + */ + @Test + fun rangeQueryOnAPrimaryKeyHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE pk2 (name TEXT COLLATE NOCASE PRIMARY KEY, hits INTEGER)") + db.run("INSERT INTO pk2 VALUES ('Alice', 1), ('Bob', 2), ('Zoe', 3)") + val plan = db.explain("EXPLAIN SELECT name FROM pk2 WHERE name > 'alice'") + assertTrue(plan.any { it.contains("Primary Key Range Scan") }, "expected primary key range scan, got $plan") + assertEquals( + listOf(listOf(Value.TextValue("Bob")), listOf(Value.TextValue("Zoe"))), + db.select("SELECT name FROM pk2 WHERE name > 'alice' ORDER BY name").rows + ) + } + + /** + * [DiskTable.rangeBySecondary]'s counterpart of [indexedEqualityHonorsTheColumnsDeclaredCollation] + * for a range predicate: a secondary index entry carries a RID suffix a primary key entry does + * not, so the fold has to reuse [kestrel.engine.collationFoldedUniquenessKey]'s + * (all-zero-suffix, all-one-suffix) bracketing trick rather than [DiskTable.rangeByPrimaryKey]'s + * bare bound - see [DiskTable.rangeBySecondary]'s own doc for why a bare bound would be wrong + * here. `>=` (inclusive) deliberately lands exactly on the folded value of an existing row + * (`'bob'` folds the same as the query's `'BOB'`) to prove the inclusive boundary itself is + * collation-aware, not just strictly-inside values. The bound is deliberately lowercase + * (`'bob'`), same reasoning as [rangeQueryOnAPrimaryKeyHonorsTheColumnsDeclaredCollation]: an + * unfolded lowercase bound would sort bytewise after every uppercase-folded stored key. + */ + @Test + fun rangeQueryOnASecondaryIndexHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + db.run("INSERT INTO items VALUES (1, 'Alice'), (2, 'Bob'), (3, 'ZOE')") + db.run("CREATE INDEX idxItemName ON items (name)") + val plan = db.explain("EXPLAIN SELECT id FROM items WHERE name >= 'bob'") + assertTrue(plan.any { it.contains("Index Range Scan using idxItemName") }, "expected index range scan, got $plan") + assertEquals( + listOf(listOf(int(2)), listOf(int(3))), + db.select("SELECT id FROM items WHERE name >= 'bob' ORDER BY id").rows + ) + } + + /** + * [DiskTable.rangeBySecondary]'s prefix+range counterpart of [rangeQueryOnASecondaryIndexHonorsTheColumnsDeclaredCollation] + * and [compositeIndexEqualityHonorsPerColumnDeclaredCollationIndependently]: an equality-prefix + * (`order_id`, plain BINARY, irrelevant here) plus a range on the composite index's second column + * (`sku`, NOCASE) must fold BOTH the stored `sku` values AND the query's own bound through + * `sku`'s declared collation before the B+Tree ever compares bytes - `'Mango'` only survives + * `sku > 'lemon'` once both sides are canonicalized to `'MANGO' > 'LEMON'`; an unfolded lowercase + * bound would sort bytewise after every uppercase-folded stored value (lowercase ASCII sorts + * after uppercase), wrongly excluding every row instead of just `'Apple'`/`'kiwi'`. + */ + @Test + fun rangeQueryOnASecondaryIndexAfterAnEqualityPrefixHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE order_lines2 (order_id INTEGER, sku TEXT COLLATE NOCASE)") + db.run("INSERT INTO order_lines2 VALUES (1, 'Apple'), (1, 'Mango'), (1, 'kiwi'), (2, 'Zebra')") + db.run("CREATE INDEX idxOrderSku2 ON order_lines2 (order_id, sku)") + val plan = db.explain("EXPLAIN SELECT sku FROM order_lines2 WHERE order_id = 1 AND sku > 'lemon'") + assertTrue(plan.any { it.contains("Index Range Scan using idxOrderSku2") }, "expected index range scan, got $plan") + assertEquals( + listOf(listOf(Value.TextValue("Mango"))), + db.select("SELECT sku FROM order_lines2 WHERE order_id = 1 AND sku > 'lemon'").rows + ) + } + + /** + * Now that [DiskTable]'s secondary-index B+Tree keys are folded through the column's declared + * `COLLATE`, ONGOING enforcement of an already-created `CREATE UNIQUE INDEX` catches a + * case-variant duplicate too - this test used to assert the opposite (see git history) as a + * documented scope-cut; [kestrel.engine.collationFoldedUniquenessKey]'s own doc comment + * previously named this exact gap as "the harder, still-open half" of the disk engine's + * COLLATE support. + */ + @Test + fun uniqueIndexEnforcementAfterCreationHonorsTheColumnsDeclaredCollation() { + val db = newDatabase() + db.run("CREATE TABLE u4 (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + db.run("INSERT INTO u4 VALUES (1, 'Bob')") + db.run("CREATE UNIQUE INDEX idxU4 ON u4 (name)") + assertFailsWith { db.run("INSERT INTO u4 VALUES (2, 'bob')") } + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM u4").rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorDistinctTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorDistinctTest.kt new file mode 100644 index 0000000..7a2a495 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorDistinctTest.kt @@ -0,0 +1,167 @@ +package kestrel.store.sql + +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +/** + * `SELECT DISTINCT`/`SELECT DISTINCT ON (...)` through [DiskExecutor] against a real [DiskCatalog] + * backed by temp files - ports [kestrel.DistinctTest]'s in-memory scenarios 1:1, since `runSelect` + * shares the same dedup pipeline shape as [kestrel.engine.SelectRunner]'s `Plan.projectRows`. + * [distinctSurvivesClosingAndReopeningTheCatalog] has no in-memory equivalent - it exercises the one + * thing only the disk-backed engine's persistence layer can demonstrate. + */ +class DiskExecutorDistinctTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-distinct-data"), tempFile("kestrel-disk-distinct-wal")) + + private fun Database.seedSales() { + run("CREATE TABLE sales (dept TEXT, region TEXT, amount INTEGER)") + run( + "INSERT INTO sales VALUES " + + "('eng', 'eu', 100), ('eng', 'us', 200), ('eng', 'eu', 100), " + + "('ops', 'eu', 50), ('ops', 'eu', 70), ('ops', 'eu', 50)" + ) + } + + @Test + fun distinctRemovesDuplicateOutputRows() { + val db = newDatabase() + db.seedSales() + val result = db.select("SELECT DISTINCT dept FROM sales ORDER BY dept") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops"))), result.rows) + } + + @Test + fun distinctStarDedupsOnEveryColumn() { + val db = newDatabase() + db.seedSales() + val result = db.select("SELECT DISTINCT * FROM sales ORDER BY dept, region, amount") + assertEquals(4, result.rows.size) + } + + @Test + fun distinctWithLimitCountsDistinctRowsNotSourceRows() { + val db = newDatabase() + db.seedSales() + val result = db.select("SELECT DISTINCT dept FROM sales ORDER BY dept LIMIT 1") + assertEquals(listOf(listOf(text("eng"))), result.rows) + } + + @Test + fun distinctWithOffsetSkipsDistinctRowsNotSourceRows() { + val db = newDatabase() + db.seedSales() + val result = db.select("SELECT DISTINCT dept FROM sales ORDER BY dept LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun distinctCombinesWithGroupBy() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER)") + db.run("INSERT INTO t VALUES (1, 10), (1, 20), (2, 10)") + val result = db.select("SELECT DISTINCT a FROM t GROUP BY a, b ORDER BY a") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), result.rows) + } + + @Test + fun distinctOnKeepsFirstRowPerGroupInOrderByOrder() { + val db = newDatabase() + db.seedSales() + val result = db.select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, amount DESC") + assertEquals(listOf(listOf(text("eng"), int(200)), listOf(text("ops"), int(70))), result.rows) + } + + @Test + fun distinctOnWithoutOrderByUsesScanOrder() { + val db = newDatabase() + db.run("CREATE TABLE t (k TEXT, v INTEGER)") + db.run("INSERT INTO t VALUES ('a', 1), ('a', 2), ('b', 3)") + val result = db.select("SELECT DISTINCT ON (k) k, v FROM t") + assertEquals(listOf(listOf(text("a"), int(1)), listOf(text("b"), int(3))), result.rows) + } + + @Test + fun distinctOnRequiresOrderByToStartWithItsOwnExpressions() { + val db = newDatabase() + db.seedSales() + assertFailsWith { + db.select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY amount") + } + } + + @Test + fun distinctOnRejectsGroupBy() { + val db = newDatabase() + db.seedSales() + assertFailsWith { + db.select("SELECT DISTINCT ON (dept) dept, SUM(amount) FROM sales GROUP BY dept") + } + } + + @Test + fun distinctOnRejectsAWindowFunction() { + val db = newDatabase() + db.seedSales() + assertFailsWith { + db.select("SELECT DISTINCT ON (ROW_NUMBER() OVER (ORDER BY amount)) dept FROM sales") + } + } + + @Test + fun distinctOnRejectsAnUnknownColumn() { + val db = newDatabase() + db.seedSales() + assertFailsWith { + db.select("SELECT DISTINCT ON (nope) dept FROM sales") + } + } + + @Test + fun distinctSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedSales() + db.reopen() + val result = db.select("SELECT DISTINCT dept FROM sales ORDER BY dept") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops"))), result.rows) + } + + @Test + fun distinctOnSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedSales() + db.reopen() + val result = db.select("SELECT DISTINCT ON (dept) dept, amount FROM sales ORDER BY dept, amount DESC") + assertEquals(listOf(listOf(text("eng"), int(200)), listOf(text("ops"), int(70))), result.rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorExplainTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorExplainTest.kt new file mode 100644 index 0000000..3cd048c --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorExplainTest.kt @@ -0,0 +1,398 @@ +package kestrel.store.sql + +import kestrel.engine.ExplainResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * Closes the "this executor has no `explain`" scope cut documented on [DiskExecutor.executeCreateView]: + * proves EXPLAIN on the disk-backed executor both describes the right plan shape (Seq Scan/Primary Key + * Scan/Index Scan/Nested Loop/GroupAggregate/Sort/Limit/CTE/UNION) AND - the part a plan-text assertion + * alone cannot show - never touches a single row of real table data to produce it, verified against the + * same [DiskTable] access counters [DiskExecutorTest] itself uses to prove index pushdown really happens. + */ +class DiskExecutorExplainTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + + private inner class Database(dataFile: File, walFile: File) { + val catalog = DiskCatalog.open(dataFile, walFile) + val executor = DiskExecutor(catalog) + + fun run(sql: String): List = parse(sql).map { executor.execute(it) } + fun explain(sql: String): List = (run(sql).single() as ExplainResult).plan + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-explain-data"), tempFile("kestrel-disk-explain-wal")) + + private fun Database.seedUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER)") + run("INSERT INTO users VALUES (1, 'ada', 36), (2, 'linus', 54), (3, 'grace', 85)") + } + + private fun Database.seedOrders() { + run("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("INSERT INTO customers VALUES (1, 'ada'), (2, 'linus'), (3, 'grace')") + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER NOT NULL, amount INTEGER NOT NULL)") + run("INSERT INTO orders VALUES (1, 1, 100), (2, 1, 50), (3, 2, 200)") + } + + @Test + fun explainOnAnUnindexedWhereReportsASeqScanAndTouchesNoRowData() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE name = 'linus'") + assertTrue(plan.any { it == "Seq Scan on users" }) + assertEquals(0, table.scanCount) + } + + @Test + fun explainOnAPrimaryKeyEqualityReportsAPrimaryKeyScanAndTouchesNoRowData() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE id = 2") + assertTrue(plan.any { it == "Primary Key Scan on users (id = 2)" }) + assertEquals(0, table.pointLookupCount) + assertEquals(0, table.scanCount) + } + + @Test + fun explainOnAPrimaryKeyRangeReportsAPrimaryKeyRangeScanAndTouchesNoRowData() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE id > 1 AND id <= 3") + assertTrue(plan.any { it == "Primary Key Range Scan on users (id > 1 AND id <= 3)" }) + assertEquals(0, table.rangeScanCount) + assertEquals(0, table.scanCount) + } + + @Test + fun explainOnACompositePrimaryKeyEqualityReportsAPrimaryKeyScan() { + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT, PRIMARY KEY (order_id, line_no))") + db.run("INSERT INTO order_lines VALUES (1, 1, 'widget'), (1, 2, 'gadget')") + val table = db.catalog.table("order_lines") + val plan = db.explain("EXPLAIN SELECT sku FROM order_lines WHERE order_id = 1 AND line_no = 2") + assertTrue(plan.any { it == "Primary Key Scan on order_lines (order_id = 1 AND line_no = 2)" }) + assertEquals(0, table.pointLookupCount) + } + + @Test + fun explainOnACompositePrimaryKeyRangeReportsAPrimaryKeyRangeScan() { + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT, PRIMARY KEY (order_id, line_no))") + db.run("INSERT INTO order_lines VALUES (1, 1, 'widget'), (1, 2, 'gadget')") + val table = db.catalog.table("order_lines") + val plan = db.explain("EXPLAIN SELECT sku FROM order_lines WHERE order_id = 1 AND line_no > 1") + assertTrue(plan.any { it == "Primary Key Range Scan on order_lines (order_id = 1 AND line_no > 1)" }) + assertEquals(0, table.rangeScanCount) + assertEquals(0, table.scanCount) + } + + @Test + fun explainOnASecondaryIndexEqualityNamesTheIndexAndTouchesNoRowData() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + val lookupsBeforeExplain = table.secondaryLookupCount + val scansBeforeExplain = table.scanCount + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE age = 54") + assertTrue(plan.any { it == "Index Scan using idx_age on users (age = 54)" }) + assertEquals(lookupsBeforeExplain, table.secondaryLookupCount) + assertEquals(scansBeforeExplain, table.scanCount) + } + + @Test + fun explainOnASecondaryIndexRangeNamesTheIndexAndTouchesNoRowData() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + val rangeScansBeforeExplain = table.rangeScanCount + val scansBeforeExplain = table.scanCount + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE age > 40 AND age <= 85") + assertTrue(plan.any { it == "Index Range Scan using idx_age on users (age > 40 AND age <= 85)" }) + assertEquals(rangeScansBeforeExplain, table.rangeScanCount) + assertEquals(scansBeforeExplain, table.scanCount) + } + + @Test + fun explainOnACompositeSecondaryIndexPrefixRangeNamesBothThePrefixEqualityAndTheRange() { + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT)") + db.run("INSERT INTO order_lines VALUES (1, 1, 'apple'), (1, 2, 'mango'), (2, 1, 'kiwi')") + db.run("CREATE INDEX idx_order_sku ON order_lines(order_id, sku)") + val table = db.catalog.table("order_lines") + val rangeScansBeforeExplain = table.rangeScanCount + val scansBeforeExplain = table.scanCount + val plan = db.explain("EXPLAIN SELECT line_no FROM order_lines WHERE order_id = 1 AND sku > 'apple'") + assertTrue(plan.any { it == "Index Range Scan using idx_order_sku on order_lines (order_id = 1 AND sku > 'apple')" }) + assertEquals(rangeScansBeforeExplain, table.rangeScanCount) + assertEquals(scansBeforeExplain, table.scanCount) + } + + @Test + fun explainRevertsToASeqScanAfterTheIndexItWasUsingIsDropped() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + db.run("DROP INDEX idx_age") + val plan = db.explain("EXPLAIN SELECT name FROM users WHERE age = 54") + assertTrue(plan.any { it == "Seq Scan on users" }) + } + + @Test + fun explainOnACompositeSecondaryIndexEqualityNamesEveryColumn() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + db.run("INSERT INTO users VALUES (1, 36, 'ada'), (2, 36, 'grace')") + db.run("CREATE INDEX idx_both ON users(age, name)") + val plan = db.explain("EXPLAIN SELECT id FROM users WHERE age = 36 AND name = 'grace'") + assertTrue(plan.any { it == "Index Scan using idx_both on users (age = 36 AND name = 'grace')" }) + } + + @Test + fun explainOnOnlyOneColumnOfACompositeIndexFallsBackToASeqScan() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + db.run("INSERT INTO users VALUES (1, 36, 'ada')") + db.run("CREATE INDEX idx_both ON users(age, name)") + val plan = db.explain("EXPLAIN SELECT id FROM users WHERE age = 36") + assertTrue(plan.any { it == "Seq Scan on users" }) + } + + @Test + fun explainWithAJoinNeverPushesDownEvenWhenWhereMatchesThePrimaryKey() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain( + "EXPLAIN SELECT customers.name, orders.amount FROM customers JOIN orders " + + "ON customers.id = orders.customer_id WHERE customers.id = 1" + ) + assertTrue(plan.any { it.trim() == "-> Seq Scan on customers" }) + assertTrue(plan.none { it.contains("Primary Key") }) + } + + @Test + fun explainNeverPrintsHashJoinEvenForAnEqualityCondition() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain( + "EXPLAIN SELECT customers.name, orders.amount FROM customers " + + "JOIN orders ON customers.id = orders.customer_id" + ) + assertTrue(plan.any { it == "Nested Loop Inner" }) + assertTrue(plan.none { it.contains("Hash Join") }) + assertTrue(plan.any { it.trim() == "-> Seq Scan on orders" }) + } + + @Test + fun explainAnnotatesRightFullAndCrossJoinsWithTheirLabel() { + val db = newDatabase() + db.seedOrders() + val right = db.explain("EXPLAIN SELECT customers.name FROM customers RIGHT JOIN orders ON customers.id = orders.customer_id") + assertTrue(right.any { it == "Nested Loop Right" }) + val full = db.explain("EXPLAIN SELECT customers.name FROM customers FULL JOIN orders ON customers.id = orders.customer_id") + assertTrue(full.any { it == "Nested Loop Full" }) + val cross = db.explain("EXPLAIN SELECT customers.name FROM customers CROSS JOIN orders") + assertTrue(cross.any { it == "Nested Loop Cross" }) + } + + @Test + fun explainAnnotatesALateralJoinAndNamesItsRightSideALateralSubqueryScan() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + db.run("INSERT INTO departments VALUES (1, 'eng')") + db.run("INSERT INTO employees VALUES (1, 1, 'ada', 300)") + val plan = db.explain( + "EXPLAIN SELECT d.name, top.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE" + ) + assertTrue(plan.any { it == "Nested Loop Inner Lateral" }) + assertTrue(plan.any { it.trim() == "-> Lateral Subquery Scan on top" }) + } + + @Test + fun explainOnAGroupByReportsGroupAggregateWithItsKeys() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain("EXPLAIN SELECT customer_id, SUM(amount) FROM orders GROUP BY customer_id") + assertTrue(plan.any { it.trim() == "GroupAggregate keys: customer_id" }) + } + + @Test + fun explainOnAPlainAggregateWithNoGroupByReportsAggregate() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain("EXPLAIN SELECT COUNT(*) FROM orders") + assertTrue(plan.any { it.trim() == "Aggregate" }) + } + + @Test + fun explainOnARollupReportsTheNumberOfGroupingSets() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain("EXPLAIN SELECT customer_id, SUM(amount) FROM orders GROUP BY ROLLUP (customer_id)") + assertTrue(plan.any { it.contains("(2 grouping sets)") }) + } + + @Test + fun explainOnAPlainGroupByDoesNotMentionGroupingSets() { + val db = newDatabase() + db.seedOrders() + val plan = db.explain("EXPLAIN SELECT customer_id, SUM(amount) FROM orders GROUP BY customer_id") + assertTrue(plan.none { it.contains("grouping sets") }) + } + + @Test + fun explainWithOrderByWrapsSortAroundThePlan() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT name FROM users ORDER BY age") + assertEquals("Sort", plan.first()) + } + + @Test + fun explainOnDistinctReportsUnique() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT DISTINCT age FROM users") + assertEquals("Unique", plan.first()) + } + + @Test + fun explainOnDistinctOnReportsUniqueDistinctOn() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT DISTINCT ON (age) name FROM users ORDER BY age") + assertEquals("Unique (DISTINCT ON)", plan.first()) + } + + @Test + fun explainWithLimitWrapsLimitAroundThePlan() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT name FROM users LIMIT 5") + assertEquals("Limit 5", plan.first()) + } + + @Test + fun explainWithLimitAndOffsetWrapsLimitOutermostAndOffsetJustInside() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT name FROM users ORDER BY name LIMIT 5 OFFSET 2") + assertEquals("Limit 5", plan[0]) + assertEquals(" Offset 2", plan[1]) + } + + @Test + fun explainOverACompoundQueryWithOffsetWrapsOffsetJustInsideLimit() { + val db = newDatabase() + db.run("CREATE TABLE t1 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("CREATE TABLE t2 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("INSERT INTO t1 VALUES (1, 'ada')") + db.run("INSERT INTO t2 VALUES (2, 'linus')") + val plan = db.explain("EXPLAIN SELECT name FROM t1 UNION SELECT name FROM t2 ORDER BY name LIMIT 1 OFFSET 1") + assertEquals("Limit 1", plan[0]) + assertEquals(" Offset 1", plan[1]) + } + + @Test + fun explainOverACompoundQueryDescribesTheSetOperatorAndBothSides() { + val db = newDatabase() + db.run("CREATE TABLE t1 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("CREATE TABLE t2 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("INSERT INTO t1 VALUES (1, 'ada')") + db.run("INSERT INTO t2 VALUES (2, 'linus')") + val plan = db.explain("EXPLAIN SELECT name FROM t1 UNION SELECT name FROM t2") + assertEquals("Union", plan.first()) + assertTrue(plan.any { it.trim() == "-> Seq Scan on t1" }) + assertTrue(plan.any { it.trim() == "-> Seq Scan on t2" }) + } + + @Test + fun explainOverAWithRecursiveQueryDescribesTheCteAndItsBody() { + val db = newDatabase() + db.run("CREATE TABLE dual (x INTEGER)") + db.run("INSERT INTO dual VALUES (1)") + val plan = db.explain( + "EXPLAIN WITH RECURSIVE counter(n) AS (" + + "SELECT 1 FROM dual UNION ALL SELECT n + 1 FROM counter WHERE n < 5" + + ") SELECT n FROM counter" + ) + assertTrue(plan.any { it == "CTE counter" }) + assertTrue(plan.any { it == "Seq Scan on counter" }) + } + + @Test + fun explainOverAViewDescribesTheOuterQueryAgainstTheViewsMaterializedSchema() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40") + val plan = db.explain("EXPLAIN SELECT * FROM adults") + assertTrue(plan.any { it == "Seq Scan on adults" }) + } + + @Test + fun explainOverADerivedTableDescribesItByItsAlias() { + val db = newDatabase() + db.seedUsers() + val plan = db.explain("EXPLAIN SELECT s.name FROM (SELECT id, name FROM users) AS s") + assertTrue(plan.any { it == "Seq Scan on s" }) + } + + @Test + fun explainRejectsAWindowFunctionInWhereStructurallyWithoutRunningTheQuery() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val failure = assertFailsWith { + db.explain("EXPLAIN SELECT name FROM users WHERE ROW_NUMBER() OVER (ORDER BY id) > 1") + } + assertTrue(failure.message!!.contains("window functions are not allowed in WHERE")) + assertEquals(0, table.scanCount) + } + + @Test + fun explainRejectsAColumnMissingFromGroupByJustLikeRunningTheQueryWould() { + val db = newDatabase() + db.seedOrders() + val failure = assertFailsWith { + db.explain("EXPLAIN SELECT customer_id, amount FROM orders GROUP BY customer_id") + } + assertTrue(failure.message!!.contains("must appear in GROUP BY")) + } + + @Test + fun explainRejectsADuplicateTableAliasJustLikeRunningTheQueryWould() { + val db = newDatabase() + db.seedUsers() + val failure = assertFailsWith { + db.explain("EXPLAIN SELECT * FROM users u JOIN users u ON u.id = u.id") + } + assertTrue(failure.message!!.contains("duplicate table alias")) + } + + @Test + fun explainOnAMissingTableStillReportsNoSuchTable() { + val db = newDatabase() + assertFailsWith { db.explain("EXPLAIN SELECT * FROM missing") } + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorGeneratedColumnTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorGeneratedColumnTest.kt new file mode 100644 index 0000000..9e5bf3b --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorGeneratedColumnTest.kt @@ -0,0 +1,241 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.MergeResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `GENERATED ALWAYS AS (expr) STORED` through [DiskExecutor] against a real [DiskCatalog] backed by + * temp files - scenarios ported 1:1 from [kestrel.GeneratedColumnTest] (the in-memory engine's + * equivalent) wherever they translate, plus [generatedColumnSurvivesAReopen] proving + * [DirectoryEntryCodec] round-trips [kestrel.catalog.Column.generated] the same way it already did + * [kestrel.catalog.Column.default]. + */ +class DiskExecutorGeneratedColumnTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-generated-data"), tempFile("kestrel-disk-generated-wal")) + + @Test + fun generatedColumnIsComputedWhenOmittedFromInsert() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("INSERT INTO t (a) VALUES (5)") + assertEquals(listOf(listOf(int(5), int(10))), db.select("SELECT a, b FROM t").rows) + } + + @Test + fun insertRejectsAnExplicitValueForAGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { db.run("INSERT INTO t (a, b) VALUES (5, 99)") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun updatingADependencyColumnRecomputesTheGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("INSERT INTO t (a) VALUES (5)") + db.run("UPDATE t SET a = 9") + assertEquals(listOf(listOf(int(18))), db.select("SELECT b FROM t").rows) + } + + @Test + fun updateRejectsAnExplicitValueForAGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("INSERT INTO t (a) VALUES (5)") + assertFailsWith { db.run("UPDATE t SET b = 1") } + } + + @Test + fun onConflictDoUpdateRecomputesTheGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("INSERT INTO t (id, a) VALUES (1, 5)") + db.run("INSERT INTO t (id, a) VALUES (1, 20) ON CONFLICT (id) DO UPDATE SET a = EXCLUDED.a") + assertEquals(listOf(listOf(int(40))), db.select("SELECT b FROM t").rows) + } + + @Test + fun mergeInsertComputesTheGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("CREATE TABLE s (id INTEGER, a INTEGER)") + db.run("INSERT INTO s VALUES (1, 5)") + val result = db.exec( + "MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, a) VALUES (s.id, s.a)" + ) as MergeResult + assertEquals(MergeResult(1, 0, 0), result) + assertEquals(listOf(listOf(int(10))), db.select("SELECT b FROM t").rows) + } + + @Test + fun mergeUpdateRecomputesTheGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("CREATE TABLE s (id INTEGER, a INTEGER)") + db.run("INSERT INTO t (id, a) VALUES (1, 5)") + db.run("INSERT INTO s VALUES (1, 9)") + db.exec("MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET a = s.a") + assertEquals(listOf(listOf(int(18))), db.select("SELECT b FROM t").rows) + } + + @Test + fun rejectsGeneratedExpressionReferencingANonexistentColumn() { + val db = newDatabase() + val error = assertFailsWith { + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (missing) STORED)") + } + assertTrue(error.message!!.contains("no such column")) + } + + @Test + fun rejectsAGeneratedColumnReferencingAnotherGeneratedColumn() { + val db = newDatabase() + val error = assertFailsWith { + db.run( + "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, " + + "c INTEGER GENERATED ALWAYS AS (b + 1) STORED)" + ) + } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun renameColumnPropagatesIntoADependentGeneratedExpression() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("ALTER TABLE t RENAME COLUMN a TO a2") + db.run("INSERT INTO t (a2) VALUES (5)") + assertEquals(listOf(listOf(int(10))), db.select("SELECT b FROM t").rows) + } + + @Test + fun dropColumnRestrictsWhenAGeneratedColumnDependsOnIt() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN a") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun dropColumnCascadeRemovesTheDependentGeneratedColumnToo() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, c TEXT)") + db.run("INSERT INTO t (a, c) VALUES (5, 'x')") + db.run("ALTER TABLE t DROP COLUMN a CASCADE") + val result = db.select("SELECT * FROM t") + assertEquals(listOf("c"), result.columns) + assertEquals(listOf(listOf(Value.TextValue("x"))), result.rows) + } + + @Test + fun generatedColumnSurvivesAReopen() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + db.run("INSERT INTO t (a) VALUES (5)") + db.reopen() + assertEquals(listOf(listOf(int(5), int(10))), db.select("SELECT a, b FROM t").rows) + // The GENERATED expression itself (not just data already computed before the reopen) must + // have round-tripped through DirectoryEntryCodec - proven by inserting a NEW row after the + // reopen and getting a freshly computed value, not a stale/missing one. + db.run("INSERT INTO t (a) VALUES (7)") + assertEquals(listOf(listOf(int(14))), db.select("SELECT b FROM t WHERE a = 7").rows) + } + + // ---- ALTER TABLE ADD COLUMN GENERATED ALWAYS AS ---- + + @Test + fun alterTableAddColumnGeneratedBackfillsEachExistingRowFromItsOwnOtherColumns() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER)") + db.run("INSERT INTO t (a) VALUES (1), (2), (3)") + db.run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + assertEquals( + listOf(listOf(int(1), int(2)), listOf(int(2), int(4)), listOf(int(3), int(6))), + db.select("SELECT a, b FROM t ORDER BY a ASC").rows + ) + } + + @Test + fun afterAlterTableAddColumnGeneratedAnInsertStillRejectsAnExplicitValue() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER)") + db.run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + val error = assertFailsWith { db.run("INSERT INTO t (a, b) VALUES (5, 99)") } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun alterTableAddColumnGeneratedNotNullSkipsTheDefaultRequirementOnANonEmptyTable() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER)") + db.run("INSERT INTO t (a) VALUES (1), (2)") + db.run("ALTER TABLE t ADD COLUMN b INTEGER NOT NULL GENERATED ALWAYS AS (a * 2) STORED") + assertEquals(listOf(listOf(int(2)), listOf(int(4))), db.select("SELECT b FROM t ORDER BY a ASC").rows) + } + + @Test + fun alterTableAddColumnGeneratedNotNullRejectsARowWhoseBackfilledValueIsNull() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER)") + db.run("INSERT INTO t (a) VALUES (NULL)") + val error = assertFailsWith { + db.run("ALTER TABLE t ADD COLUMN b INTEGER NOT NULL GENERATED ALWAYS AS (a) STORED") + } + assertTrue(error.message!!.contains("NOT NULL")) + } + + @Test + fun alterTableAddColumnGeneratedRejectsReferencingAnAlreadyExistingGeneratedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED)") + val error = assertFailsWith { + db.run("ALTER TABLE t ADD COLUMN c INTEGER GENERATED ALWAYS AS (b + 1) STORED") + } + assertTrue(error.message!!.contains("GENERATED")) + } + + @Test + fun alterTableAddColumnGeneratedSurvivesAReopenAndBackfilledRowsKeepTheirValue() { + val db = newDatabase() + db.run("CREATE TABLE t (a INTEGER)") + db.run("INSERT INTO t (a) VALUES (5)") + db.run("ALTER TABLE t ADD COLUMN b INTEGER GENERATED ALWAYS AS (a * 2) STORED") + db.reopen() + assertEquals(listOf(listOf(int(5), int(10))), db.select("SELECT a, b FROM t").rows) + db.run("INSERT INTO t (a) VALUES (7)") + assertEquals(listOf(listOf(int(14))), db.select("SELECT b FROM t WHERE a = 7").rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorMergeTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorMergeTest.kt new file mode 100644 index 0000000..b8ef6f5 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorMergeTest.kt @@ -0,0 +1,537 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.MergeResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `MERGE INTO ... USING ... ON ... WHEN [NOT] MATCHED ...` through [DiskExecutor] against a real + * [DiskCatalog] backed by temp files - see [DiskExecutorTest] for the general end-to-end pattern this + * reuses, and [kestrel.MergeTest] for the in-memory engine's equivalent scenarios, ported 1:1 wherever + * they translate (every one of them does; the disk-backed engine has no scope cut left for MERGE). + * [mergeSurvivesClosingAndReopeningTheCatalog] has no in-memory equivalent - it exercises the one thing + * only the disk-backed engine's persistence layer can demonstrate. + */ +class DiskExecutorMergeTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-merge-data"), tempFile("kestrel-disk-merge-wal")) + + private fun Database.setUpAccountsAndStaging() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER NOT NULL)") + run("CREATE TABLE staging (id INTEGER, amount INTEGER)") + } + + @Test + fun matchedRowIsUpdated() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 50)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount" + ) as MergeResult + assertEquals(MergeResult(0, 1, 0), result) + assertEquals(listOf(listOf(int(150))), db.select("SELECT balance FROM accounts").rows) + } + + @Test + fun unmatchedRowIsInserted() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO staging VALUES (7, 30)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(1, 0, 0), result) + assertEquals(listOf(listOf(int(7), int(30))), db.select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun matchedRowIsDeleted() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 0)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id WHEN MATCHED THEN DELETE" + ) as MergeResult + assertEquals(MergeResult(0, 0, 1), result) + assertEquals(emptyList(), db.select("SELECT * FROM accounts").rows) + } + + @Test + fun aMultiRowMergeInsertsUpdatesAndDeletesInOneStatement() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100), (2, 200)") + db.run("INSERT INTO staging VALUES (1, 10), (2, -999), (3, 5)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND accounts.balance + staging.amount < 0 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(1, 1, 1), result) + assertEquals( + listOf(listOf(int(1), int(110)), listOf(int(3), int(5))), + db.select("SELECT id, balance FROM accounts ORDER BY id").rows + ) + } + + @Test + fun firstSatisfiedWhenMatchedClauseWinsInWrittenOrder() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 5)") + db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 100 THEN DELETE " + + "WHEN MATCHED AND staging.amount > 0 THEN UPDATE SET balance = 1 " + + "WHEN MATCHED THEN UPDATE SET balance = 2" + ) + assertEquals(listOf(listOf(int(1))), db.select("SELECT balance FROM accounts").rows) + } + + @Test + fun firstSatisfiedWhenNotMatchedClauseWinsInWrittenOrder() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO staging VALUES (1, 500)") + db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND staging.amount > 1000 THEN INSERT (id, balance) VALUES (staging.id, 1) " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, 2)" + ) + assertEquals(listOf(listOf(int(1), int(2))), db.select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun noSatisfiedWhenClauseLeavesTheRowUntouched() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 5)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 1000 THEN UPDATE SET balance = 0" + ) as MergeResult + assertEquals(MergeResult(0, 0, 0), result) + assertEquals(listOf(listOf(int(100))), db.select("SELECT balance FROM accounts").rows) + } + + @Test + fun anUnmatchedSourceRowWithNoSatisfiedNotMatchedClauseInsertsNothing() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO staging VALUES (1, 5)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND staging.amount > 1000 THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) as MergeResult + assertEquals(MergeResult(0, 0, 0), result) + assertEquals(emptyList(), db.select("SELECT * FROM accounts").rows) + } + + @Test + fun aTargetRowMatchedByTwoSourceRowsCannotBeAffectedTwice() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 5), (1, 10)") + val error = assertFailsWith { + db.run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount" + ) + } + assertTrue(error.message!!.contains("a second time"), error.message) + } + + @Test + fun aTargetRowMatchedByTwoSourceRowsIsFineWhenOnlyOneClauseActuallyFires() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 5), (1, 999)") + db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 100 THEN UPDATE SET balance = accounts.balance + staging.amount" + ) + assertEquals(listOf(listOf(int(1099))), db.select("SELECT balance FROM accounts").rows) + } + + @Test + fun anUnqualifiedColumnPresentOnBothSidesIsAmbiguousInOnCondition() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER)") + db.run("CREATE TABLE s (id INTEGER, v INTEGER)") + val error = assertFailsWith { + db.run("MERGE INTO t USING s ON id = s.id WHEN MATCHED THEN UPDATE SET v = 1") + } + assertTrue(error.message!!.contains("ambiguous"), error.message) + } + + @Test + fun aQualifierMatchingNeitherSideFails() { + val db = newDatabase() + db.setUpAccountsAndStaging() + val error = assertFailsWith { + db.run("MERGE INTO accounts USING staging ON bogus.id = staging.id WHEN MATCHED THEN DELETE") + } + assertTrue(error.message!!.contains("no such table or alias"), error.message) + } + + @Test + fun aWhenNotMatchedClauseCannotReferenceTheTargetTable() { + val db = newDatabase() + db.setUpAccountsAndStaging() + val error = assertFailsWith { + db.run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED AND accounts.balance > 0 THEN INSERT (id, balance) VALUES (staging.id, 1)" + ) + } + assertTrue(error.message!!.contains("WHEN NOT MATCHED"), error.message) + } + + @Test + fun columnValidationRunsUpFrontEvenWhenNoSourceRowEverReachesTheBadClause() { + val db = newDatabase() + db.setUpAccountsAndStaging() + val error = assertFailsWith { + db.run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN NOT MATCHED THEN INSERT (id, nope) VALUES (staging.id, 1)" + ) + } + assertTrue(error.message!!.contains("no such column"), error.message) + } + + @Test + fun theTargetAndSourceCannotShareTheSameNameOrAlias() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE other (id INTEGER)") + val error = assertFailsWith { + db.run("MERGE INTO t USING other AS t ON t.id = t.id WHEN MATCHED THEN DELETE") + } + assertTrue(error.message!!.contains("target and source"), error.message) + } + + @Test + fun sourceCanBeADerivedTableSubquery() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO staging VALUES (1, 10), (1, 20)") + db.exec( + "MERGE INTO accounts USING (SELECT id, SUM(amount) AS total FROM staging GROUP BY id) AS agg " + + "ON accounts.id = agg.id " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (agg.id, agg.total)" + ) + assertEquals(listOf(listOf(int(1), int(30))), db.select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun aDefaultKeywordInTheInsertValuesUsesTheColumnDefault() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT DEFAULT 'unlabeled')") + db.run("CREATE TABLE s (id INTEGER)") + db.run("INSERT INTO s VALUES (1)") + db.exec("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, DEFAULT)") + assertEquals(listOf(listOf(int(1), text("unlabeled"))), db.select("SELECT id, label FROM t").rows) + } + + @Test + fun anOmittedColumnListInsertsIntoEveryColumnInSchemaOrder() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + db.run("CREATE TABLE s (id INTEGER, label TEXT)") + db.run("INSERT INTO s VALUES (1, 'a')") + db.exec("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT VALUES (s.id, s.label)") + assertEquals(listOf(listOf(int(1), text("a"))), db.select("SELECT id, label FROM t").rows) + } + + @Test + fun aNextvalDefaultStillAdvancesThroughAMergeInsert() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + db.run("CREATE TABLE s (label TEXT)") + db.run("INSERT INTO s VALUES ('a'), ('b')") + db.exec("MERGE INTO t USING s ON t.label = s.label WHEN NOT MATCHED THEN INSERT (label) VALUES (s.label)") + assertEquals( + listOf(listOf(int(1), text("a")), listOf(int(2), text("b"))), + db.select("SELECT id, label FROM t ORDER BY id").rows + ) + } + + @Test + fun notNullViolationOnAMergeInsertFailsAndInsertsNothing() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT NOT NULL)") + db.run("CREATE TABLE s (id INTEGER)") + db.run("INSERT INTO s VALUES (1)") + assertFailsWith { + db.run("MERGE INTO t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id)") + } + assertEquals(emptyList(), db.select("SELECT * FROM t").rows) + } + + @Test + fun aForeignKeyViolationOnAMergeUpdateIsRejected() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + db.run("CREATE TABLE moves (id INTEGER, new_dept INTEGER)") + db.run("INSERT INTO departments VALUES (1)") + db.run("INSERT INTO employees VALUES (1, 1)") + db.run("INSERT INTO moves VALUES (1, 999)") + val error = assertFailsWith { + db.run("MERGE INTO employees USING moves ON employees.id = moves.id WHEN MATCHED THEN UPDATE SET dept_id = moves.new_dept") + } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message) + assertEquals(listOf(listOf(int(1))), db.select("SELECT dept_id FROM employees").rows) + } + + @Test + fun rowTriggersFireForEachMergeDrivenAction() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + db.run("CREATE TABLE s (id INTEGER, label TEXT)") + db.run("CREATE TABLE log (event TEXT)") + db.run("INSERT INTO t VALUES (1, 'old'), (2, 'gone')") + db.run("INSERT INTO s VALUES (1, 'new'), (2, 'new'), (3, 'fresh')") + db.run("CREATE TRIGGER log_insert AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('insert:', NEW.label)); END") + db.run("CREATE TRIGGER log_update AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('update:', NEW.label)); END") + db.run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN INSERT INTO log VALUES (CONCAT('delete:', OLD.label)); END") + db.exec( + "MERGE INTO t USING s ON t.id = s.id " + + "WHEN MATCHED AND s.id = 2 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET label = s.label " + + "WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, s.label)" + ) + assertEquals( + setOf(text("update:new"), text("delete:gone"), text("insert:fresh")), + db.select("SELECT event FROM log").rows.map { it.single() }.toSet() + ) + } + + @Test + fun rollingBackATransactionUndoesEveryEffectOfAMerge() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 50), (2, 20)") + db.run("BEGIN") + db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) + db.run("ROLLBACK") + assertEquals(listOf(listOf(int(1), int(100))), db.select("SELECT id, balance FROM accounts").rows) + } + + @Test + fun anAliasedTargetAndSourceCanBeReferencedByTheirAlias() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 25)") + db.exec( + "MERGE INTO accounts AS acc USING staging AS s ON acc.id = s.id " + + "WHEN MATCHED THEN UPDATE SET balance = acc.balance + s.amount" + ) + assertEquals(listOf(listOf(int(125))), db.select("SELECT balance FROM accounts").rows) + } + + @Test + fun mergeRequiresAtLeastOneWhenClause() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE s (id INTEGER)") + assertFailsWith { db.run("MERGE INTO t USING s ON t.id = s.id") } + } + + @Test + fun mergingIntoAnUnknownTableFails() { + val db = newDatabase() + db.run("CREATE TABLE s (id INTEGER)") + assertFailsWith { + db.run("MERGE INTO nope USING s ON nope.id = s.id WHEN MATCHED THEN DELETE") + } + } + + @Test + fun mergeSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 50), (2, 20)") + db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount)" + ) + db.reopen() + assertEquals( + listOf(listOf(int(1), int(150)), listOf(int(2), int(20))), + db.select("SELECT id, balance FROM accounts ORDER BY id").rows + ) + } + + @Test + fun returningStarProjectsEveryColumnOfEveryActionedRow() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 10), (2, 5)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "WHEN NOT MATCHED THEN INSERT (id, balance) VALUES (staging.id, staging.amount) " + + "RETURNING *" + ) as MergeResult + assertEquals(listOf("id", "balance"), result.returning!!.columns) + assertEquals( + listOf(listOf(int(1), int(110)), listOf(int(2), int(5))), + result.returning!!.rows + ) + } + + @Test + fun returningEvaluatesExpressionsAgainstTheAffectedRow() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 10)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "RETURNING balance * 2 AS doubled" + ) as MergeResult + assertEquals(listOf("doubled"), result.returning!!.columns) + assertEquals(listOf(listOf(int(220))), result.returning!!.rows) + } + + @Test + fun returningIncludesThePreDeletionImageOfADeletedRow() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 0)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id WHEN MATCHED THEN DELETE RETURNING id, balance" + ) as MergeResult + assertEquals(listOf(listOf(int(1), int(100))), result.returning!!.rows) + assertEquals(emptyList(), db.select("SELECT * FROM accounts").rows) + } + + @Test + fun mergeActionLabelsEachReturnedRowByWhichClauseFired() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + db.run("CREATE TABLE s (id INTEGER, label TEXT)") + db.run("INSERT INTO t VALUES (1, 'old'), (2, 'gone')") + db.run("INSERT INTO s VALUES (1, 'new'), (2, 'new'), (3, 'fresh')") + val result = db.exec( + "MERGE INTO t USING s ON t.id = s.id " + + "WHEN MATCHED AND s.id = 2 THEN DELETE " + + "WHEN MATCHED THEN UPDATE SET label = s.label " + + "WHEN NOT MATCHED THEN INSERT (id, label) VALUES (s.id, s.label) " + + "RETURNING id, MERGE_ACTION() AS kind" + ) as MergeResult + assertEquals( + setOf(listOf(int(1), text("UPDATE")), listOf(int(2), text("DELETE")), listOf(int(3), text("INSERT"))), + result.returning!!.rows.toSet() + ) + } + + @Test + fun noRowMatchingAnyClauseReturnsAnEmptyReturningResultNotANullOne() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 5)") + val result = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED AND staging.amount > 1000 THEN UPDATE SET balance = 0 " + + "RETURNING *" + ) as MergeResult + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun mergeActionIsRejectedOutsideOfAMergesOwnReturningClause() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1)") + val error = assertFailsWith { db.select("SELECT MERGE_ACTION() FROM t") } + assertTrue(error.message!!.contains("MERGE_ACTION"), error.message) + } + + @Test + fun returningColumnValidationRunsUpFrontEvenWhenNoSourceRowEverReachesAMatchingClause() { + val db = newDatabase() + db.setUpAccountsAndStaging() + val error = assertFailsWith { + db.run( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN DELETE RETURNING nope" + ) + } + assertTrue(error.message!!.contains("no such column"), error.message) + } + + @Test + fun returningSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.setUpAccountsAndStaging() + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("INSERT INTO staging VALUES (1, 50)") + val beforeReopen = db.exec( + "MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = accounts.balance + staging.amount " + + "RETURNING id, balance" + ) as MergeResult + assertEquals(listOf(listOf(int(1), int(150))), beforeReopen.returning!!.rows) + db.reopen() + assertEquals(listOf(listOf(int(1), int(150))), db.select("SELECT id, balance FROM accounts").rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorOnConflictTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorOnConflictTest.kt new file mode 100644 index 0000000..400df78 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorOnConflictTest.kt @@ -0,0 +1,288 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `INSERT ... ON CONFLICT (...) DO NOTHING|DO UPDATE` through [DiskExecutor] against a real + * [DiskCatalog] backed by temp files - see [DiskExecutorTest] for the general end-to-end pattern + * this reuses. Scenarios are ported 1:1 from [kestrel.UpsertTest] (the in-memory engine's + * equivalent) wherever they translate; [conflictResolutionSurvivesAReopen] replaces that suite's + * dump/replay test (this executor has no [kestrel.persistence.SqlWriter] equivalent - durability + * here means surviving a real close/reopen of the backing files) and is new to this file. + */ +class DiskExecutorOnConflictTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + fun insertCount(sql: String): Int = (exec(sql) as InsertResult).count + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-onconflict-data"), tempFile("kestrel-disk-onconflict-wal")) + + private fun Database.seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER NOT NULL)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + } + + @Test + fun noConflictInsertsNormallyAndReportsOneRow() { + val db = newDatabase() + db.seed() + val count = db.insertCount("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (id) DO NOTHING") + assertEquals(1, count) + assertEquals(2, db.select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictOnPrimaryKeyDoesNothingAndReportsZero() { + val db = newDatabase() + db.seed() + val count = db.insertCount("INSERT INTO accounts VALUES (1, 'ignored', 999) ON CONFLICT (id) DO NOTHING") + assertEquals(0, count) + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals("ada", (db.select("SELECT name FROM accounts WHERE id = 1").rows[0][0] as Value.TextValue).value) + } + + @Test + fun conflictWithBareOnConflictWithNoTargetCatchesAnyUniqueViolation() { + val db = newDatabase() + db.seed() + db.run("CREATE UNIQUE INDEX idxName ON accounts (name)") + val count = db.insertCount("INSERT INTO accounts VALUES (2, 'ada', 300) ON CONFLICT DO NOTHING") + assertEquals(0, count) + assertEquals(1, db.select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictDoUpdateAppliesTheAssignments() { + val db = newDatabase() + db.seed() + db.insertCount("INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999") + assertEquals(int(999), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(1, db.select("SELECT * FROM accounts").rows.size) + } + + @Test + fun conflictDoUpdateCanReferenceExcludedForTheProposedRow() { + val db = newDatabase() + db.seed() + db.insertCount("INSERT INTO accounts VALUES (1, 'ada', 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance") + assertEquals(int(150), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictDoUpdateWithFailingWhereBehavesLikeDoNothingForThatRow() { + val db = newDatabase() + db.seed() + val count = db.insertCount( + "INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999 WHERE excluded.balance < 0" + ) + assertEquals(0, count) + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictDoUpdateWithPassingWhereApplies() { + val db = newDatabase() + db.seed() + val count = db.insertCount( + "INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = 999 WHERE excluded.balance > 0" + ) + assertEquals(1, count) + assertEquals(int(999), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + @Test + fun multiRowInsertHandlesEachRowsConflictIndependently() { + val db = newDatabase() + db.seed() + val count = db.insertCount( + """ + INSERT INTO accounts VALUES (1, 'ada', 5), (2, 'linus', 200), (3, 'grace', 300) + ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance + """.trimIndent() + ) + assertEquals(3, count) + assertEquals(int(105), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(int(200), db.select("SELECT balance FROM accounts WHERE id = 2").rows[0][0]) + assertEquals(int(300), db.select("SELECT balance FROM accounts WHERE id = 3").rows[0][0]) + } + + @Test + fun laterRowInTheSameStatementSeesAnEarlierRowsInsert() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, hits INTEGER NOT NULL)") + val count = db.insertCount( + """ + INSERT INTO t VALUES (1, 1), (1, 1) + ON CONFLICT (id) DO UPDATE SET hits = t.hits + excluded.hits + """.trimIndent() + ) + assertEquals(2, count) + assertEquals(int(2), db.select("SELECT hits FROM t WHERE id = 1").rows[0][0]) + } + + @Test + fun conflictTargetMustMatchAnExistingUniqueOrPrimaryKey() { + val db = newDatabase() + db.seed() + val error = assertFailsWith { + db.run("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (name) DO NOTHING") + } + assertTrue(error.message!!.contains("no unique or primary key constraint"), error.message!!) + } + + @Test + fun conflictTargetMatchesADeclaredUniqueConstraintNotJustThePrimaryKey() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE, name TEXT)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com', 'ada')") + val count = db.insertCount( + "INSERT INTO users VALUES (2, 'ada@example.com', 'ada2') ON CONFLICT (email) DO UPDATE SET name = excluded.name" + ) + assertEquals(1, count) + assertEquals("ada2", (db.select("SELECT name FROM users WHERE email = 'ada@example.com'").rows[0][0] as Value.TextValue).value) + assertEquals(1, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun conflictOnADifferentUnrelatedUniqueColumnStillThrows() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE, tag TEXT UNIQUE)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com', 'x')") + val error = assertFailsWith { + db.run("INSERT INTO users VALUES (2, 'new@example.com', 'x') ON CONFLICT (email) DO NOTHING") + } + assertTrue(error.message!!.contains("UNIQUE constraint"), error.message!!) + } + + @Test + fun conflictDoUpdateEnforcesNotNullOnTheResultingRow() { + val db = newDatabase() + db.seed() + val error = assertFailsWith { + db.run("INSERT INTO accounts VALUES (1, 'ada', 100) ON CONFLICT (id) DO UPDATE SET name = NULL") + } + assertTrue(error.message!!.contains("NOT NULL"), error.message!!) + } + + @Test + fun conflictDoUpdateEnforcesCheckConstraintsOnTheResultingRow() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER CHECK (balance >= 0))") + db.run("INSERT INTO accounts VALUES (1, 100)") + val error = assertFailsWith { + db.run("INSERT INTO accounts VALUES (1, 100) ON CONFLICT (id) DO UPDATE SET balance = -5") + } + assertTrue(error.message!!.contains("CHECK constraint"), error.message!!) + } + + @Test + fun conflictDoUpdateRunsForeignKeyValidationOnTheUpdatedRow() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO departments VALUES (1)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept INTEGER REFERENCES departments(id))") + db.run("INSERT INTO employees VALUES (1, 1)") + val error = assertFailsWith { + db.run("INSERT INTO employees VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET dept = 999") + } + assertTrue(error.message!!.contains("FOREIGN KEY"), error.message!!) + } + + @Test + fun conflictDoUpdateCascadesToChildTablesWhenTheReferencedKeyChanges() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO departments VALUES (1)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept INTEGER REFERENCES departments(id) ON UPDATE CASCADE)") + db.run("INSERT INTO employees VALUES (1, 1)") + db.insertCount("INSERT INTO departments VALUES (1) ON CONFLICT (id) DO UPDATE SET id = 2") + assertEquals(int(2), db.select("SELECT dept FROM employees WHERE id = 1").rows[0][0]) + } + + @Test + fun compositePrimaryKeyConflictTargetIsRecognised() { + val db = newDatabase() + db.run("CREATE TABLE grades (student INTEGER, course INTEGER, score INTEGER, PRIMARY KEY (student, course))") + db.run("INSERT INTO grades VALUES (1, 10, 50)") + val count = db.insertCount( + "INSERT INTO grades VALUES (1, 10, 90) ON CONFLICT (student, course) DO UPDATE SET score = excluded.score" + ) + assertEquals(1, count) + assertEquals(int(90), db.select("SELECT score FROM grades WHERE student = 1 AND course = 10").rows[0][0]) + } + + @Test + fun conflictDoUpdateDefaultKeywordFallsBackToColumnDefault() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, note TEXT DEFAULT 'fresh')") + db.run("INSERT INTO t VALUES (1, 'stale')") + db.insertCount("INSERT INTO t VALUES (1, 'stale') ON CONFLICT (id) DO UPDATE SET note = DEFAULT") + assertEquals("fresh", (db.select("SELECT note FROM t WHERE id = 1").rows[0][0] as Value.TextValue).value) + } + + @Test + fun onConflictInsideAnExplicitTransactionParticipatesInItsCommit() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.insertCount("INSERT INTO accounts VALUES (1, 'ada', 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + db.insertCount("INSERT INTO accounts VALUES (2, 'linus', 200) ON CONFLICT (id) DO NOTHING") + db.run("ROLLBACK") + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(1, db.select("SELECT * FROM accounts").rows.size) + } + + /** + * Durability proof unique to the disk-backed executor: a resolved conflict (one row inserted + * fresh, one row updated in place) must be readable after a real close/reopen of the backing + * files, not just correct in the live process's buffer pool - mirrors the reopen tests already + * established for SEQUENCE ([DiskExecutorSequenceTest.sequencePositionSurvivesAReopen]) and + * SAVEPOINT ([DiskExecutorSavepointTest.dataWrittenAfterASavepointRollbackSurvivesAReopenAfterCommit]). + */ + @Test + fun conflictResolutionSurvivesAReopen() { + val db = newDatabase() + db.seed() + db.insertCount( + """ + INSERT INTO accounts VALUES (1, 'ada', 999), (2, 'linus', 200) + ON CONFLICT (id) DO UPDATE SET balance = excluded.balance + """.trimIndent() + ) + db.reopen() + assertEquals(int(999), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(int(200), db.select("SELECT balance FROM accounts WHERE id = 2").rows[0][0]) + assertEquals(2, db.select("SELECT * FROM accounts").rows.size) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorOrderByNullsTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorOrderByNullsTest.kt new file mode 100644 index 0000000..0d63ab1 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorOrderByNullsTest.kt @@ -0,0 +1,85 @@ +package kestrel.store.sql + +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * `ORDER BY expr [ASC|DESC] [NULLS FIRST|NULLS LAST]` against the disk engine, mirroring + * [kestrel.OrderByNullsTest]'s in-memory scenarios to prove [kestrel.engine.compareOrderKeyValues] + * is genuinely shared rather than reimplemented per engine with divergent NULL handling. + */ +class DiskExecutorOrderByNullsTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + + private inner class Database(dataFile: File, walFile: File) { + val catalog = DiskCatalog.open(dataFile, walFile) + val executor = DiskExecutor(catalog) + + fun run(sql: String): List = parse(sql).map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-nulls-data"), tempFile("kestrel-disk-nulls-wal")) + + /** ids 2 and 4 are NULL; the rest ascend 10/20/30 out of id order so ASC/DESC actually reorder them. */ + private fun Database.seedScored() { + run("CREATE TABLE scored (id INTEGER PRIMARY KEY, score INTEGER)") + run("INSERT INTO scored VALUES (1, 10), (2, NULL), (3, 30), (4, NULL), (5, 20)") + } + + @Test + fun ascendingOrderByDefaultsToNullsFirstButNullsLastOverridesIt() { + val db = newDatabase() + db.seedScored() + assertEquals( + listOf(2, 4, 1, 5, 3).map { listOf(int(it.toLong())) }, + db.select("SELECT id FROM scored ORDER BY score").rows + ) + assertEquals( + listOf(1, 5, 3, 2, 4).map { listOf(int(it.toLong())) }, + db.select("SELECT id FROM scored ORDER BY score NULLS LAST").rows + ) + } + + @Test + fun descendingOrderByDefaultsToNullsLastButNullsFirstOverridesIt() { + val db = newDatabase() + db.seedScored() + assertEquals( + listOf(3, 5, 1, 2, 4).map { listOf(int(it.toLong())) }, + db.select("SELECT id FROM scored ORDER BY score DESC").rows + ) + assertEquals( + listOf(2, 4, 3, 5, 1).map { listOf(int(it.toLong())) }, + db.select("SELECT id FROM scored ORDER BY score DESC NULLS FIRST").rows + ) + } + + @Test + fun windowFunctionOrderByHonorsAnExplicitNullsLastClause() { + val db = newDatabase() + db.seedScored() + val result = db.select( + "SELECT id, ROW_NUMBER() OVER (ORDER BY score NULLS LAST) AS rn FROM scored ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(4)), listOf(int(3), int(3)), + listOf(int(4), int(5)), listOf(int(5), int(2)) + ), + result.rows + ) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorReturningTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorReturningTest.kt new file mode 100644 index 0000000..ae36ae1 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorReturningTest.kt @@ -0,0 +1,205 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.DeleteResult +import kestrel.engine.InsertResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +/** + * `RETURNING` on INSERT/UPDATE/DELETE through [DiskExecutor] against a real [DiskCatalog] backed by + * temp files - see [DiskExecutorTest] for the general end-to-end pattern this reuses. Scenarios are + * ported 1:1 from [kestrel.ReturningTest] (the in-memory engine's equivalent) wherever they + * translate; [returningSurvivesAReopen] has no in-memory equivalent since [DiskExecutor] has no + * [kestrel.persistence.SqlWriter] counterpart - durability here means surviving a real close/reopen. + */ +class DiskExecutorReturningTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): SelectResult = exec(sql) as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-returning-data"), tempFile("kestrel-disk-returning-wal")) + + private fun Database.seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER NOT NULL)") + run("INSERT INTO accounts VALUES (1, 'ada', 100)") + } + + @Test + fun insertReturningStarReportsTheFullInsertedRow() { + val db = newDatabase() + db.seed() + val result = db.exec("INSERT INTO accounts VALUES (2, 'linus', 200) RETURNING *") as InsertResult + assertEquals(1, result.count) + assertEquals(listOf("id", "name", "balance"), result.returning!!.columns) + assertEquals(listOf(listOf(int(2), text("linus"), int(200))), result.returning!!.rows) + } + + @Test + fun insertReturningExpressionListWithAliasProjectsAndLabels() { + val db = newDatabase() + db.seed() + val result = db.exec("INSERT INTO accounts VALUES (2, 'linus', 200) RETURNING id, balance * 2 AS doubled") as InsertResult + assertEquals(listOf("id", "doubled"), result.returning!!.columns) + assertEquals(listOf(listOf(int(2), int(400))), result.returning!!.rows) + } + + @Test + fun insertWithoutReturningLeavesReturningNull() { + val db = newDatabase() + db.seed() + val result = db.exec("INSERT INTO accounts VALUES (2, 'linus', 200)") as InsertResult + assertNull(result.returning) + } + + @Test + fun onConflictDoNothingContributesNoRowToReturning() { + val db = newDatabase() + db.seed() + val result = db.exec( + "INSERT INTO accounts VALUES (1, 'ignored', 999) ON CONFLICT (id) DO NOTHING RETURNING *" + ) as InsertResult + assertEquals(0, result.count) + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun onConflictDoUpdateReturnsThePostUpdateRowNotTheProposedOne() { + val db = newDatabase() + db.seed() + val result = db.exec( + "INSERT INTO accounts VALUES (1, 'ada', 50) ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance RETURNING balance" + ) as InsertResult + assertEquals(1, result.count) + assertEquals(listOf(listOf(int(150))), result.returning!!.rows) + } + + @Test + fun mixedConflictAndFreshRowsInOneStatementReturnBothKinds() { + val db = newDatabase() + db.seed() + val result = db.exec( + "INSERT INTO accounts VALUES (1, 'ada', 5), (2, 'linus', 200) " + + "ON CONFLICT (id) DO UPDATE SET balance = accounts.balance + excluded.balance RETURNING id, balance" + ) as InsertResult + assertEquals( + listOf(listOf(int(1), int(105)), listOf(int(2), int(200))), + result.returning!!.rows + ) + } + + @Test + fun updateReturningReportsThePostUpdateValuesForEveryMatchedRow() { + val db = newDatabase() + db.seed() + db.run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = db.exec("UPDATE accounts SET balance = balance + 10 RETURNING id, balance") as UpdateResult + assertEquals(2, result.count) + assertEquals(listOf(listOf(int(1), int(110)), listOf(int(2), int(210))), result.returning!!.rows) + } + + @Test + fun updateReturningWithNoMatchesReturnsAnEmptyResultNotNull() { + val db = newDatabase() + db.seed() + val result = db.exec("UPDATE accounts SET balance = 0 WHERE id = 999 RETURNING id") as UpdateResult + assertEquals(0, result.count) + assertEquals(emptyList(), result.returning!!.rows) + } + + @Test + fun deleteReturningReportsThePreDeletionRowValues() { + val db = newDatabase() + db.seed() + db.run("INSERT INTO accounts VALUES (2, 'linus', 200)") + val result = db.exec("DELETE FROM accounts WHERE id = 1 RETURNING *") as DeleteResult + assertEquals(1, result.count) + assertEquals(listOf(listOf(int(1), text("ada"), int(100))), result.returning!!.rows) + assertEquals(1, db.select("SELECT * FROM accounts").rows.size) + } + + @Test + fun returningRejectsAnUnknownColumnEvenWhenZeroRowsMatch() { + val db = newDatabase() + db.seed() + assertFailsWith { + db.exec("UPDATE accounts SET balance = 0 WHERE id = 999 RETURNING nope") + } + } + + @Test + fun returningRejectsAnAggregateFunction() { + val db = newDatabase() + db.seed() + assertFailsWith { + db.exec("DELETE FROM accounts WHERE id = 1 RETURNING COUNT(*)") + } + } + + @Test + fun returningIsRejectedInsideATriggerBodyStatement() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("CREATE TABLE audit (order_id INTEGER)") + assertFailsWith { + db.run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id) RETURNING order_id; END" + ) + } + } + + @Test + fun aTableWithATriggerStillHonoursReturningOnTheOutermostStatement() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("CREATE TABLE audit (order_id INTEGER)") + db.run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + val result = db.exec("INSERT INTO orders VALUES (1, 100) RETURNING id, amount") as InsertResult + assertEquals(listOf(listOf(int(1), int(100))), result.returning!!.rows) + assertEquals(listOf(listOf(int(1))), db.select("SELECT order_id FROM audit").rows) + } + + @Test + fun returningSurvivesAReopen() { + val db = newDatabase() + db.seed() + val inserted = db.exec("INSERT INTO accounts VALUES (2, 'linus', 200) RETURNING *") as InsertResult + assertEquals(listOf(listOf(int(2), text("linus"), int(200))), inserted.returning!!.rows) + db.reopen() + val deleted = db.exec("DELETE FROM accounts WHERE id = 2 RETURNING id") as DeleteResult + assertEquals(listOf(listOf(int(2))), deleted.returning!!.rows) + assertEquals(1, db.select("SELECT * FROM accounts").rows.size) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorSavepointTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorSavepointTest.kt new file mode 100644 index 0000000..b14bb04 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorSavepointTest.kt @@ -0,0 +1,332 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.BeginResult +import kestrel.engine.CommitResult +import kestrel.engine.ReleaseSavepointResult +import kestrel.engine.RollbackResult +import kestrel.engine.RollbackToSavepointResult +import kestrel.engine.SavepointResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +/** + * Exercises `SAVEPOINT`/`RELEASE SAVEPOINT`/`ROLLBACK TO SAVEPOINT` through [DiskExecutor] against a + * real [DiskCatalog] backed by temp files - see [DiskExecutorTest] for the general end-to-end pattern + * this reuses. Most scenarios here are ported 1:1 from [kestrel.SavepointTest] (the in-memory engine's + * equivalent), since the stack semantics ([DiskExecutor.indexOfLastSavepoint] et al.) are identical - + * the exception is `rollbackToSavepointUndoesCreateTable`, which does not translate: DDL is rejected + * inside an explicit transaction on this executor entirely (see [DiskExecutorTest]), so there is + * nothing for a savepoint to roll back there. The tests unique to this file + * ([fullRollbackAfterASavepointRollbackUndoesEverythingIncludingThePreSavepointWrite], + * [rollbackToSavepointSurvivesACrashBeforeTheTransactionResolves], + * [dataWrittenAfterASavepointRollbackSurvivesAReopenAfterCommit]) cover what is actually new on the + * disk-backed path: unlike [kestrel.engine.Executor]'s whole-catalog snapshot/restore, a savepoint here + * partially rolls back the storage engine's own WAL transaction via ARIES-style compensation log + * records (see [kestrel.store.wal.LogRecord.CompensationUpdate]), so only an end-to-end reopen (or + * simulated crash) test proves the result is actually durable and not just correct in the live + * process's buffer pool cache. + */ +class DiskExecutorSavepointTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + fun rowCount(sql: String): Int = select(sql).rows.size + + /** + * Closes and reopens the catalog against the same files, simulating a fresh process. Called + * with an explicit transaction still open (no preceding COMMIT/ROLLBACK), this simulates a + * crash: the storage engine's own recovery pass on [DiskCatalog.open] is what must resolve it. + */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-savepoint-data"), tempFile("kestrel-disk-savepoint-wal")) + + private fun Database.seed() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100), (2, 200)") + } + + @Test + fun rollbackToSavepointUndoesOnlyTheInsertsAfterIt() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + assertEquals(SavepointResult("beforeFour"), db.run("SAVEPOINT beforeFour").single()) + db.run("INSERT INTO accounts VALUES (4, 400)") + assertEquals(4, db.rowCount("SELECT * FROM accounts")) + assertEquals(RollbackToSavepointResult("beforeFour"), db.run("ROLLBACK TO SAVEPOINT beforeFour").single()) + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + assertEquals(CommitResult, db.run("COMMIT").single()) + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + } + + @Test + fun rollbackToSavepointUndoesUpdatesAndDeletes() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("UPDATE accounts SET balance = 0 WHERE id = 1") + db.run("DELETE FROM accounts WHERE id = 2") + assertEquals(int(0), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + db.run("ROLLBACK TO SAVEPOINT s1") + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(2, db.rowCount("SELECT * FROM accounts")) + db.run("ROLLBACK") + } + + @Test + fun theSavepointItselfStaysUsableAfterRollingBackToIt() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("ROLLBACK TO SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (4, 400)") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + db.run("ROLLBACK TO SAVEPOINT s1") + assertEquals(2, db.rowCount("SELECT * FROM accounts")) + db.run("COMMIT") + } + + @Test + fun releaseSavepointForgetsItWithoutUndoingAnything() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 300)") + assertEquals(ReleaseSavepointResult("s1"), db.run("RELEASE SAVEPOINT s1").single()) + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT s1") } + db.run("COMMIT") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + } + + @Test + fun releaseSavepointAlsoForgetsLaterNestedSavepoints() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT spOuter") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("SAVEPOINT spInner") + db.run("INSERT INTO accounts VALUES (4, 400)") + db.run("RELEASE SAVEPOINT spOuter") + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT spInner") } + assertEquals(4, db.rowCount("SELECT * FROM accounts")) + db.run("COMMIT") + } + + @Test + fun reusingASavepointNameShadowsTheOlderOneUntilItIsConsumed() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("SAVEPOINT s") + db.run("INSERT INTO accounts VALUES (4, 400)") + db.run("ROLLBACK TO SAVEPOINT s") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + db.run("ROLLBACK TO SAVEPOINT s") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + db.run("COMMIT") + } + + @Test + fun rollbackRestoresIndexStateEvenAcrossASavepoint() { + val db = newDatabase() + db.seed() + db.run("CREATE INDEX idxBalance ON accounts (balance)") + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 100)") + assertEquals(2, db.rowCount("SELECT * FROM accounts WHERE balance = 100")) + db.run("ROLLBACK TO SAVEPOINT s1") + assertEquals(1, db.rowCount("SELECT * FROM accounts WHERE balance = 100")) + db.run("ROLLBACK") + } + + @Test + fun savepointOutsideATransactionIsRejected() { + val db = newDatabase() + db.seed() + assertFailsWith { db.run("SAVEPOINT s1") } + } + + @Test + fun releaseSavepointOutsideATransactionIsRejected() { + val db = newDatabase() + db.seed() + assertFailsWith { db.run("RELEASE SAVEPOINT s1") } + } + + @Test + fun rollbackToSavepointOutsideATransactionIsRejected() { + val db = newDatabase() + db.seed() + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT s1") } + } + + @Test + fun rollbackToAnUnknownSavepointNameIsRejected() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT nope") } + db.run("ROLLBACK") + } + + @Test + fun releaseAnUnknownSavepointNameIsRejected() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + assertFailsWith { db.run("RELEASE SAVEPOINT nope") } + db.run("ROLLBACK") + } + + @Test + fun fullRollbackClearsEveryOutstandingSavepoint() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("ROLLBACK") + db.run("BEGIN") + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT s1") } + db.run("ROLLBACK") + } + + @Test + fun commitClearsEveryOutstandingSavepoint() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("COMMIT") + db.run("BEGIN") + assertFailsWith { db.run("ROLLBACK TO SAVEPOINT s1") } + db.run("ROLLBACK") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + } + + @Test + fun releaseKeywordAloneWithoutSavepointKeywordIsAccepted() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("RELEASE s1") + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + db.run("COMMIT") + } + + @Test + fun beginResultTypeIsReturnedForTheEnclosingTransaction() { + val db = newDatabase() + db.seed() + assertEquals(BeginResult, db.run("BEGIN").single()) + db.run("ROLLBACK") + } + + /** + * The SQL-level twin of `RecoveryManagerTest.fullRollbackAfterAPartialRollbackDoesNotMisapplyAnAlreadyUndoneBeforeImage` - + * proves the whole stack (DiskExecutor -> DiskCatalog -> StorageEngine -> TransactionManager -> + * RecoveryManager) wires the compensation-record undo-of-undo correctly end to end, not just at the + * page level. + */ + @Test + fun fullRollbackAfterASavepointRollbackUndoesEverythingIncludingThePreSavepointWrite() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (4, 400)") + db.run("ROLLBACK TO SAVEPOINT s1") + assertEquals(3, db.rowCount("SELECT * FROM accounts"), "the pre-savepoint insert must still be visible in-session") + assertEquals(RollbackResult, db.run("ROLLBACK").single()) + assertEquals(2, db.rowCount("SELECT * FROM accounts"), "a full rollback must also undo the pre-savepoint insert") + } + + /** + * A crash that lands right after `ROLLBACK TO SAVEPOINT` but before the enclosing transaction ever + * commits or rolls back must still leave the database as if the whole transaction never happened: + * recovery resolves any still-open transaction with a full rollback regardless of what savepoints + * it touched along the way. This is the reopen-based proof that + * [kestrel.store.wal.RecoveryManager.rollbackTo]'s compensation records are genuinely durable, not + * just correct in the live process's buffer pool. + */ + @Test + fun rollbackToSavepointSurvivesACrashBeforeTheTransactionResolves() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (4, 400)") + db.run("ROLLBACK TO SAVEPOINT s1") + // No COMMIT, no ROLLBACK: crash right here, mid-transaction. + db.reopen() + + assertEquals(2, db.rowCount("SELECT * FROM accounts"), "the entire unresolved transaction must be undone by crash recovery") + } + + /** + * The durable-happy-path counterpart to [rollbackToSavepointSurvivesACrashBeforeTheTransactionResolves]: + * a transaction that rolls back to a savepoint and then commits must persist exactly the rows that + * survived the rollback, including a write made *after* the savepoint rollback - proving the + * transaction's WAL chain still threads correctly (new writes point back at the compensation + * record, not at the undone update it replaced) once it resumes. + */ + @Test + fun dataWrittenAfterASavepointRollbackSurvivesAReopenAfterCommit() { + val db = newDatabase() + db.seed() + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (4, 400)") + db.run("ROLLBACK TO SAVEPOINT s1") + db.run("INSERT INTO accounts VALUES (5, 500)") + db.run("COMMIT") + db.reopen() + + assertEquals( + listOf(int(1), int(2), int(3), int(5)), + db.select("SELECT id FROM accounts ORDER BY id").rows.map { it[0] } + ) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorSequenceTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorSequenceTest.kt new file mode 100644 index 0000000..88196fe --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorSequenceTest.kt @@ -0,0 +1,385 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.AlterSequenceResult +import kestrel.engine.CreateSequenceResult +import kestrel.engine.DropSequenceResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +/** + * Exercises `CREATE`/`DROP`/`ALTER SEQUENCE` and `NEXTVAL`/`CURRVAL` through [DiskExecutor] against a + * real [DiskCatalog] backed by temp files - see [DiskExecutorTest] for that general end-to-end + * pattern. Most scenarios here are ported 1:1 from [kestrel.SequenceTest] (the in-memory engine's + * equivalent), since [DiskExecutor] reuses the exact same [kestrel.catalog.Sequence]/ + * [kestrel.catalog.SequenceDefinition] arithmetic - the three exceptions are the "ROLLBACK undoes + * CREATE/DROP/ALTER SEQUENCE" tests, which do not translate: `CREATE`/`DROP`/`ALTER SEQUENCE` are DDL, + * and DDL is rejected inside an explicit disk transaction entirely (see + * [rollbackCannotWrapCreateSequenceEitherBecauseItIsRejectedInsideAnExplicitTransaction] and its two + * siblings) rather than being undoable by it, unlike the in-memory engine's whole-catalog + * snapshot/restore. [sequencePositionSurvivesAReopen] and + * [alterSequenceRestartPositionSurvivesAReopen] instead cover what is actually new on the disk-backed + * path: [DiskCatalog.nextValue]'s counter write bypasses the WAL entirely (see its own doc), so + * nothing but this kind of end-to-end reopen test proves it is still durable. + */ +class DiskExecutorSequenceTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + /** Closes and reopens the catalog against the same files, simulating a fresh process. */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-seq-data"), tempFile("kestrel-disk-seq-wal")) + + @Test + fun createSequenceReturnsItsNameAndNextvalStartsAtOneByDefault() { + val db = newDatabase() + assertEquals(CreateSequenceResult("seq"), db.run("CREATE SEQUENCE seq").single()) + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(3))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun createSequenceOptionsAreHonored() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq START WITH 100 INCREMENT BY 10") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(100)), listOf(int(110))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun descendingSequenceDefaultsToMaxvalueAndCountsDown() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq INCREMENT BY -1") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(-2)), listOf(int(-1))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun exhaustedAscendingSequenceWithoutCycleThrows() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq MAXVALUE 2") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertFailsWith { db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") } + } + + @Test + fun cyclingAscendingSequenceWrapsToMinvalue() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq MAXVALUE 2 CYCLE") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(1))), db.select("SELECT id FROM t").rows) + } + + @Test + fun dropSequenceReturnsItsNameAndFurtherNextvalFails() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + assertEquals(DropSequenceResult("seq"), db.run("DROP SEQUENCE seq").single()) + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") } + } + + @Test + fun duplicateCreateSequenceFails() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + assertFailsWith { db.run("CREATE SEQUENCE seq") } + } + + @Test + fun droppingAnUnknownSequenceFails() { + val db = newDatabase() + assertFailsWith { db.run("DROP SEQUENCE nope") } + } + + @Test + fun alterSequenceReturnsItsNameAndRestartWithResetsPosition() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(AlterSequenceResult("seq"), db.run("ALTER SEQUENCE seq RESTART WITH 50").single()) + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(50))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun bareRestartResetsToOriginalStartValue() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq START WITH 5") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + db.run("ALTER SEQUENCE seq RESTART") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(5)), listOf(int(5)), listOf(int(6))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun alterSequenceIncrementByWithoutRestartPreservesCurrentPosition() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + db.run("ALTER SEQUENCE seq INCREMENT BY 100") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(101))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + @Test + fun alterSequenceRestartValueOutOfNewBoundsFails() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq MAXVALUE 10") + assertFailsWith { db.run("ALTER SEQUENCE seq RESTART WITH 20") } + } + + @Test + fun alteringAnUnknownSequenceFails() { + val db = newDatabase() + assertFailsWith { db.run("ALTER SEQUENCE nope INCREMENT BY 2") } + } + + @Test + fun currvalReturnsLastNextvalInThisSession() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER, again INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'), CURRVAL('seq'))") + assertEquals(listOf(listOf(int(1), int(1))), db.select("SELECT id, again FROM t").rows) + } + + @Test + fun currvalBeforeAnyNextvalInThisSessionFails() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { db.run("INSERT INTO t VALUES (CURRVAL('seq'))") } + } + + @Test + fun currvalIsScopedPerExecutorInstanceNotPerCatalog() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + val secondExecutor = DiskExecutor(db.catalog) + assertFailsWith { + parse("INSERT INTO t VALUES (CURRVAL('seq'))").map { secondExecutor.execute(it) } + } + } + + @Test + fun alterSequenceInvalidatesRememberedCurrvalForThatSession() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + db.run("ALTER SEQUENCE seq INCREMENT BY 1") + assertFailsWith { db.run("INSERT INTO t VALUES (CURRVAL('seq'))") } + } + + @Test + fun rollbackDoesNotUndoANextvalAdvance() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("BEGIN") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + db.run("ROLLBACK") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + // The rolled-back row never landed, but NEXTVAL's advance to 1 survives - the surviving + // insert gets 2, not a reused 1. + assertEquals(listOf(listOf(int(2))), db.select("SELECT id FROM t").rows) + } + + @Test + fun createSequenceIsRejectedInsideAnExplicitTransaction() { + val db = newDatabase() + db.run("BEGIN") + assertFailsWith { db.run("CREATE SEQUENCE seq") } + } + + @Test + fun dropSequenceIsRejectedInsideAnExplicitTransaction() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("BEGIN") + assertFailsWith { db.run("DROP SEQUENCE seq") } + } + + @Test + fun alterSequenceIsRejectedInsideAnExplicitTransaction() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("BEGIN") + assertFailsWith { db.run("ALTER SEQUENCE seq INCREMENT BY 2") } + } + + @Test + fun nextvalAsColumnDefaultActsLikeAnAutoIncrement() { + val db = newDatabase() + db.run("CREATE SEQUENCE order_ids") + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('order_ids'), item TEXT)") + db.run("INSERT INTO orders (item) VALUES ('a'), ('b'), ('c')") + assertEquals( + listOf(listOf(int(1), Value.TextValue("a")), listOf(int(2), Value.TextValue("b")), listOf(int(3), Value.TextValue("c"))), + db.select("SELECT id, item FROM orders ORDER BY id").rows + ) + } + + @Test + fun createTableWithNextvalDefaultRequiresTheSequenceToAlreadyExist() { + val db = newDatabase() + assertFailsWith { db.run("CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('nope'))") } + } + + @Test + fun createTableDoesNotConsumeASequenceValueJustToValidateTheDefault() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE orders (id INTEGER DEFAULT NEXTVAL('seq'))") + db.run("INSERT INTO orders (id) VALUES (DEFAULT)") + // If CREATE TABLE's own validation had already called NEXTVAL once, this would be 2. + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM orders").rows) + } + + @Test + fun nextvalDefaultOnANonIntegerColumnFails() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + assertFailsWith { db.run("CREATE TABLE orders (id TEXT DEFAULT NEXTVAL('seq'))") } + } + + @Test + fun currvalAsColumnDefaultIsRejected() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + assertFailsWith { db.run("CREATE TABLE orders (id INTEGER DEFAULT CURRVAL('seq'))") } + } + + @Test + fun addColumnWithNextvalDefaultIsRejected() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { db.run("ALTER TABLE t ADD COLUMN seq_id INTEGER DEFAULT NEXTVAL('seq')") } + } + + @Test + fun nextvalIsUsableInUpdateSet() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NULL)") + db.run("UPDATE t SET id = NEXTVAL('seq')") + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM t").rows) + } + + @Test + fun nextvalInAWhereClauseIsRejected() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (1)") + // Thrown by the shared Evaluator (no SequenceAccess given for a WHERE clause), not this + // executor's own code - see kestrel.engine.Evaluator.evaluateNextValue. + assertFailsWith { db.run("UPDATE t SET id = 2 WHERE id = NEXTVAL('seq')") } + } + + @Test + fun nextvalInACheckConstraintIsRejectedAtCreateTableTime() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + assertFailsWith { db.run("CREATE TABLE t (id INTEGER CHECK (id = NEXTVAL('seq')))") } + } + + @Test + fun createSequenceMinvalueExceedingMaxvalueFails() { + val db = newDatabase() + assertFailsWith { db.run("CREATE SEQUENCE seq MINVALUE 10 MAXVALUE 5") } + } + + @Test + fun createSequenceIncrementByZeroFails() { + val db = newDatabase() + assertFailsWith { db.run("CREATE SEQUENCE seq INCREMENT BY 0") } + } + + @Test + fun createSequenceStartOutsideBoundsFails() { + val db = newDatabase() + assertFailsWith { db.run("CREATE SEQUENCE seq MINVALUE 10 MAXVALUE 20 START WITH 5") } + } + + @Test + fun noMinvalueAndNoMaxvalueOnAlterRecomputeTheDefaultBound() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq MINVALUE 1 MAXVALUE 3") + db.run("ALTER SEQUENCE seq NO MAXVALUE") + db.run("CREATE TABLE t (id INTEGER)") + // Was capped at 3 before the ALTER; NO MAXVALUE lifts it back to the ascending default. + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq')), (NEXTVAL('seq'))") + assertEquals(4, db.select("SELECT id FROM t").rows.size) + } + + /** + * The core new guarantee this session adds over the in-memory engine's own sequence tests: + * [DiskCatalog.nextValue] writes the advanced counter straight through [DiskManager][kestrel.store.page.DiskManager], + * bypassing the WAL entirely - a reopen must still see the advance, not just a live process. + */ + @Test + fun sequencePositionSurvivesAReopen() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq')), (NEXTVAL('seq'))") + db.reopen() + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(2)), listOf(int(3))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + /** Same durability guarantee as [sequencePositionSurvivesAReopen], but for a RESTART WITH position. */ + @Test + fun alterSequenceRestartPositionSurvivesAReopen() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER)") + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + db.run("ALTER SEQUENCE seq RESTART WITH 1000") + db.reopen() + db.run("INSERT INTO t VALUES (NEXTVAL('seq'))") + assertEquals(listOf(listOf(int(1)), listOf(int(1000))), db.select("SELECT id FROM t ORDER BY id").rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorTemporaryTableTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorTemporaryTableTest.kt new file mode 100644 index 0000000..3caa64b --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorTemporaryTableTest.kt @@ -0,0 +1,287 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.CommitResult +import kestrel.engine.CreateTableResult +import kestrel.engine.DeleteResult +import kestrel.engine.DropTableResult +import kestrel.engine.InsertResult +import kestrel.engine.RollbackResult +import kestrel.engine.RollbackToSavepointResult +import kestrel.engine.SavepointResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` on [DiskExecutor] - the disk-backed counterpart of + * the in-memory engine's own feature (see [kestrel.ExecutorTest]'s temp-table cases, added by + * session #63), closing the scope-cut that session left named: "TEMP TABLE na silniku dyskowym + * pozostaje swiadomym scope-cut... tabela tymczasowa na silniku dyskowym zylaby WYLACZNIE w + * pamieci, nigdy dotykajac pager/WAL wcale". A temp table here is a plain [kestrel.storage.Table] + * living only in [DiskCatalog] (see its own KDoc), so it participates in WHERE/JOIN/subqueries/ + * ORDER BY exactly like a real table via [DiskExecutor.resolveFromSource]'s `CteRowSource` reuse, + * but FOREIGN KEY, `CREATE INDEX`, `CREATE TRIGGER`, and `INSERT ... ON CONFLICT` on one are all + * documented, explicit scope-cuts (see the rejection tests below) rather than a silent gap. + */ +class DiskExecutorTemporaryTableTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + fun rowCount(sql: String): Int = select(sql).rows.size + + /** Closes and reopens the catalog against the same files, simulating a fresh process. */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-temp-data"), tempFile("kestrel-disk-temp-wal")) + + @Test + fun createTempTableStartsEmptyAndSupportsFullCrud() { + val db = newDatabase() + assertEquals( + CreateTableResult("scratch"), + db.run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, note TEXT)").single() + ) + assertEquals(SelectResult(listOf("id", "note"), emptyList()), db.select("SELECT * FROM scratch")) + + assertEquals(InsertResult(2, null), db.run("INSERT INTO scratch VALUES (1, 'a'), (2, 'b')").single()) + assertEquals(2, db.rowCount("SELECT * FROM scratch")) + + assertEquals(UpdateResult(1, null), db.run("UPDATE scratch SET note = 'z' WHERE id = 2").single()) + assertEquals(listOf(int(2), text("z")), db.select("SELECT * FROM scratch WHERE id = 2").rows.single()) + + assertEquals(DeleteResult(1, null), db.run("DELETE FROM scratch WHERE id = 1").single()) + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + + assertEquals(DropTableResult("scratch"), db.run("DROP TABLE scratch").single()) + assertFailsWith { db.select("SELECT * FROM scratch") } + } + + @Test + fun createTemporaryTableKeywordIsAnAliasForTemp() { + val db = newDatabase() + db.run("CREATE TEMPORARY TABLE scratch (id INTEGER)") + db.run("INSERT INTO scratch VALUES (1)") + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + } + + @Test + fun aTempTableNeverGetsADirectoryEntrySoItDoesNotSurviveAReopen() { + val db = newDatabase() + db.run("CREATE TABLE regular (id INTEGER PRIMARY KEY)") + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("INSERT INTO regular VALUES (1)") + db.run("INSERT INTO scratch VALUES (99)") + assertEquals(setOf("regular", "scratch"), db.catalog.tableNames.toSet()) + + db.reopen() + + assertEquals(listOf("regular"), db.catalog.tableNames) + assertEquals(1, db.rowCount("SELECT * FROM regular")) + assertFailsWith { db.select("SELECT * FROM scratch") } + } + + @Test + fun tableNamesListsBothRegularAndTemporaryTables() { + val db = newDatabase() + db.run("CREATE TABLE regular (id INTEGER)") + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + assertEquals(setOf("regular", "scratch"), db.catalog.tableNames.toSet()) + } + + @Test + fun aTemporaryTableCannotShadowOrCollideWithAnExistingTableOrView() { + val db = newDatabase() + db.run("CREATE TABLE regular (id INTEGER)") + assertFailsWith { db.run("CREATE TEMP TABLE regular (id INTEGER)") } + + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + assertFailsWith { db.run("CREATE TABLE scratch (id INTEGER)") } + assertFailsWith { db.run("CREATE TEMP TABLE scratch (id INTEGER)") } + + db.run("CREATE VIEW a_view AS SELECT * FROM regular") + assertFailsWith { db.run("CREATE TEMP TABLE a_view (id INTEGER)") } + } + + @Test + fun notNullPrimaryKeyUniqueAndCheckAreAllEnforcedOnATemporaryTable() { + val db = newDatabase() + db.run( + "CREATE TEMP TABLE scratch (" + + "id INTEGER PRIMARY KEY, sku TEXT UNIQUE, qty INTEGER NOT NULL, CHECK (qty >= 0))" + ) + db.run("INSERT INTO scratch VALUES (1, 'a', 5)") + + assertFailsWith { db.run("INSERT INTO scratch VALUES (1, 'b', 1)") } + assertFailsWith { db.run("INSERT INTO scratch VALUES (2, 'a', 1)") } + assertFailsWith { db.run("INSERT INTO scratch (id, sku) VALUES (3, 'c')") } + assertFailsWith { db.run("INSERT INTO scratch VALUES (4, 'd', -1)") } + + // Nothing from any failed multi-row batch partially landed. + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + } + + @Test + fun defaultAndGeneratedColumnsAreEvaluatedOnATemporaryTable() { + val db = newDatabase() + db.run( + "CREATE TEMP TABLE scratch (" + + "id INTEGER, price INTEGER DEFAULT 10, doubled INTEGER GENERATED ALWAYS AS (price * 2) STORED)" + ) + db.run("INSERT INTO scratch (id) VALUES (1)") + assertEquals(listOf(int(1), int(10), int(20)), db.select("SELECT * FROM scratch").rows.single()) + + assertFailsWith { db.run("INSERT INTO scratch (id, doubled) VALUES (2, 99)") } + } + + @Test + fun onConflictOnATemporaryTableIsRejected() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO scratch VALUES (1)") + assertFailsWith { db.run("INSERT INTO scratch VALUES (1) ON CONFLICT (id) DO NOTHING") } + } + + @Test + fun foreignKeyOnATemporaryTableIsRejectedAtCreateTime() { + val db = newDatabase() + db.run("CREATE TABLE parent (id INTEGER PRIMARY KEY)") + assertFailsWith { + db.run("CREATE TEMP TABLE child (id INTEGER, parent_id INTEGER REFERENCES parent(id))") + } + } + + @Test + fun createIndexAndCreateTriggerOnATemporaryTableAreRejected() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + assertFailsWith { db.run("CREATE INDEX scratch_idx ON scratch(id)") } + assertFailsWith { + db.run("CREATE TRIGGER scratch_trg AFTER INSERT ON scratch FOR EACH ROW INSERT INTO scratch VALUES (0)") + } + } + + @Test + fun aTemporaryTableJoinsAndCorrelatesWithARealDiskTable() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, total INTEGER)") + db.run("INSERT INTO orders VALUES (1, 100), (2, 200)") + db.run("CREATE TEMP TABLE flagged (order_id INTEGER)") + db.run("INSERT INTO flagged VALUES (2)") + + val joined = db.select( + "SELECT orders.id, orders.total FROM orders JOIN flagged ON orders.id = flagged.order_id" + ) + assertEquals(listOf(listOf(int(2), int(200))), joined.rows) + + val correlated = db.select( + "SELECT id FROM orders WHERE EXISTS (SELECT 1 FROM flagged WHERE flagged.order_id = orders.id)" + ) + assertEquals(listOf(listOf(int(2))), correlated.rows) + } + + @Test + fun rollbackUndoesWritesToATemporaryTableEvenThoughItNeverTouchesTheWal() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("INSERT INTO scratch VALUES (1)") + + db.run("BEGIN") + db.run("INSERT INTO scratch VALUES (2), (3)") + db.run("DELETE FROM scratch WHERE id = 1") + assertEquals(2, db.rowCount("SELECT * FROM scratch")) + assertEquals(RollbackResult, db.run("ROLLBACK").single()) + + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + assertEquals(listOf(int(1)), db.select("SELECT * FROM scratch").rows.single()) + } + + @Test + fun commitKeepsWritesToATemporaryTable() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("BEGIN") + db.run("INSERT INTO scratch VALUES (1)") + assertEquals(CommitResult, db.run("COMMIT").single()) + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + } + + @Test + fun rollbackToSavepointUndoesOnlyTemporaryTableWritesAfterIt() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("BEGIN") + db.run("INSERT INTO scratch VALUES (1)") + assertEquals(SavepointResult("mark"), db.run("SAVEPOINT mark").single()) + db.run("INSERT INTO scratch VALUES (2)") + assertEquals(2, db.rowCount("SELECT * FROM scratch")) + + assertEquals(RollbackToSavepointResult("mark"), db.run("ROLLBACK TO SAVEPOINT mark").single()) + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + assertEquals(CommitResult, db.run("COMMIT").single()) + assertEquals(1, db.rowCount("SELECT * FROM scratch")) + } + + @Test + fun rollbackAlsoUndoesARealDiskTableWriteInTheSameTransactionAsATemporaryTableWrite() { + val db = newDatabase() + db.run("CREATE TABLE regular (id INTEGER PRIMARY KEY)") + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("BEGIN") + db.run("INSERT INTO regular VALUES (1)") + db.run("INSERT INTO scratch VALUES (1)") + db.run("ROLLBACK") + assertEquals(0, db.rowCount("SELECT * FROM regular")) + assertEquals(0, db.rowCount("SELECT * FROM scratch")) + } + + @Test + fun returningWorksOnATemporaryTable() { + val db = newDatabase() + db.run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY, note TEXT)") + val inserted = db.run("INSERT INTO scratch VALUES (1, 'a') RETURNING id, note").single() as InsertResult + assertEquals(SelectResult(listOf("id", "note"), listOf(listOf(int(1), text("a")))), inserted.returning) + + val updated = db.run("UPDATE scratch SET note = 'b' WHERE id = 1 RETURNING note").single() as UpdateResult + assertEquals(SelectResult(listOf("note"), listOf(listOf(text("b")))), updated.returning) + + val deleted = db.run("DELETE FROM scratch WHERE id = 1 RETURNING id").single() as DeleteResult + assertEquals(SelectResult(listOf("id"), listOf(listOf(int(1)))), deleted.returning) + } + + @Test + fun dropTableRemovesATemporaryTableWithoutAffectingCatalogTables() { + val db = newDatabase() + db.run("CREATE TABLE regular (id INTEGER)") + db.run("CREATE TEMP TABLE scratch (id INTEGER)") + db.run("DROP TABLE scratch") + assertEquals(listOf("regular"), db.catalog.tableNames) + assertFalse(db.catalog.tableNames.contains("scratch")) + assertTrue(db.catalog.tableOrNull("regular") != null) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorTest.kt new file mode 100644 index 0000000..e7e53c4 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorTest.kt @@ -0,0 +1,4039 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.AlterTableResult +import kestrel.engine.CreateIndexResult +import kestrel.engine.CreateSequenceResult +import kestrel.engine.CreateTableResult +import kestrel.engine.CreateViewResult +import kestrel.engine.DeallocateResult +import kestrel.engine.DeleteResult +import kestrel.engine.DropIndexResult +import kestrel.engine.DropTableResult +import kestrel.engine.DropViewResult +import kestrel.engine.InsertResult +import kestrel.engine.PrepareResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.engine.UpdateResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * End-to-end tests that parse real SQL text through the same [Lexer]/[Parser] + * the in-memory engine uses, then run it through [DiskExecutor] against a + * real [DiskCatalog] backed by temp files - proving the disk storage engine + * (page/heap/btree/wal/txn) can actually serve SQL, not just its own + * unit tests. + */ +class DiskExecutorTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private inner class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = parse(sql).map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + /** Closes and reopens the catalog against the same files, simulating a fresh process. */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-exec-data"), tempFile("kestrel-disk-exec-wal")) + + private fun Database.seedUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER DEFAULT 0)") + run("INSERT INTO users VALUES (1, 'ada', 36), (2, 'linus', 54), (3, 'grace', 85)") + } + + @Test + fun createTableReturnsItsNameAndTheTableStartsEmpty() { + val db = newDatabase() + val created = db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)").single() + assertEquals(CreateTableResult("users"), created) + assertEquals(SelectResult(listOf("id", "name"), emptyList()), db.select("SELECT * FROM users")) + } + + @Test + fun creatingTheSameTableTwiceFails() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + assertFailsWith { db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") } + } + + @Test + fun insertFillsOmittedColumnsFromDeclaredDefaultAndReportsRowCount() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER DEFAULT 0)") + + val inserted = db.run("INSERT INTO users (id, name) VALUES (1, 'ada')").single() + assertEquals(InsertResult(1), inserted) + + val result = db.select("SELECT * FROM users") + assertEquals(listOf(listOf(int(1), text("ada"), int(0))), result.rows) + } + + @Test + fun insertValuesDefaultKeywordUsesTheDeclaredDefault() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER DEFAULT 0)") + db.run("INSERT INTO users VALUES (1, 'ada', DEFAULT)") + assertEquals(listOf(listOf(int(1), text("ada"), int(0))), db.select("SELECT * FROM users").rows) + } + + /** Positive coverage lives in [DiskExecutorOnConflictTest] - this table only proves it is no longer a scope-cut. */ + @Test + fun insertOnConflictDoNothingSkipsAConflictingRowInsteadOfThrowing() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("INSERT INTO users VALUES (1, 'ada')") + val inserted = db.run("INSERT INTO users VALUES (1, 'ignored') ON CONFLICT (id) DO NOTHING").single() + assertEquals(InsertResult(0), inserted) + assertEquals(listOf(listOf(int(1), text("ada"))), db.select("SELECT * FROM users").rows) + } + + /** Positive coverage lives in [DiskExecutorTriggerTest] - this table only proves it is no longer a scope-cut. */ + @Test + fun createTriggerSucceedsAndFiresOnInsert() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("INSERT INTO t VALUES (1)") + assertEquals(listOf(listOf(int(1))), db.select("SELECT * FROM audit").rows) + } + + /** Full SEQUENCE/NEXTVAL/CURRVAL coverage lives in [DiskExecutorSequenceTest]; this just proves the wiring. */ + @Test + fun createSequenceSucceedsAndNextvalCanBackAColumnDefault() { + val db = newDatabase() + assertEquals(CreateSequenceResult("seq"), db.run("CREATE SEQUENCE seq").single()) + db.run("CREATE TABLE t (id INTEGER DEFAULT NEXTVAL('seq'))") + db.run("INSERT INTO t (id) VALUES (DEFAULT)") + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM t").rows) + } + + @Test + fun nextvalInACheckConstraintIsRejected() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + val error = assertFailsWith { + db.run("CREATE TABLE t (id INTEGER CHECK (id = NEXTVAL('seq')))") + } + assertTrue(error.message!!.contains("not allowed"), error.message!!) + } + + @Test + fun selectStarReturnsEveryColumnInSchemaOrder() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT * FROM users") + assertEquals(listOf("id", "name", "age"), result.columns) + assertEquals(3, result.rows.size) + assertTrue(listOf(int(1), text("ada"), int(36)) in result.rows) + } + + @Test + fun whereFiltersRowsUsingTheSharedEvaluator() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name FROM users WHERE age > 40") + assertEquals(setOf(text("linus"), text("grace")), result.rows.map { it.single() }.toSet()) + } + + @Test + fun orderByAndLimitApplyAfterFilteringAndBeforeProjection() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name, age FROM users ORDER BY age DESC LIMIT 2") + assertEquals(listOf(listOf(text("grace"), int(85)), listOf(text("linus"), int(54))), result.rows) + } + + @Test + fun offsetSkipsLeadingRowsWithoutALimit() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name FROM users ORDER BY age ASC OFFSET 1") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + } + + @Test + fun limitAndOffsetTogetherPageThroughTheOrderedResult() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name FROM users ORDER BY age ASC LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun offsetWrittenBeforeLimitStillAppliesLimitAfterTheSkip() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name FROM users ORDER BY age ASC OFFSET 1 LIMIT 1") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun offsetPastTheEndOfTheResultReturnsNoRows() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT name FROM users ORDER BY age ASC OFFSET 100") + assertEquals(emptyList>(), result.rows) + } + + @Test + fun offsetSurvivesCloseAndReopen() { + val db = newDatabase() + db.seedUsers() + db.reopen() + val result = db.select("SELECT name FROM users ORDER BY age ASC LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun expressionProjectionEvaluatesFunctionsAndUsesAliasesOrGeneratedLabels() { + val db = newDatabase() + db.seedUsers() + val result = db.select("SELECT UPPER(name) AS loud, age + 1 FROM users WHERE id = 1") + assertEquals(listOf("loud", "column2"), result.columns) + assertEquals(listOf(listOf(text("ADA"), int(37))), result.rows) + } + + @Test + fun updateAppliesExpressionsAgainstTheOldRowAndReportsMatchedCount() { + val db = newDatabase() + db.seedUsers() + val updated = db.run("UPDATE users SET age = age + 1 WHERE age > 40").single() + assertEquals(UpdateResult(2), updated) + val ages = db.select("SELECT name, age FROM users ORDER BY name").rows.associate { (it[0] as Value.TextValue).value to it[1] } + assertEquals(mapOf("ada" to int(36), "linus" to int(55), "grace" to int(86)), ages) + } + + @Test + fun updateSetToDefaultKeywordResetsTheColumn() { + val db = newDatabase() + db.seedUsers() + db.run("UPDATE users SET age = DEFAULT WHERE id = 2") + assertEquals(int(0), db.select("SELECT age FROM users WHERE id = 2").rows.single().single()) + } + + @Test + fun deleteRemovesMatchingRowsAndReportsCount() { + val db = newDatabase() + db.seedUsers() + val deleted = db.run("DELETE FROM users WHERE age > 40").single() + assertEquals(DeleteResult(2), deleted) + assertEquals(listOf(listOf(int(1), text("ada"), int(36))), db.select("SELECT * FROM users").rows) + } + + @Test + fun dropTableRemovesItFromTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + val dropped = db.run("DROP TABLE users").single() + assertEquals(DropTableResult("users"), dropped) + assertFailsWith { db.select("SELECT * FROM users") } + } + + @Test + fun aFailedStatementInAMultiRowInsertLeavesNoPartialRowsBehind() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + // The second row duplicates the first row's primary key: DiskTable rejects it mid-batch, + // so the whole INSERT statement (including the already-applied first row) must abort. + assertFailsWith { + db.run("INSERT INTO users VALUES (1, 'ada'), (1, 'ada-again')") + } + assertEquals(emptyList>(), db.select("SELECT * FROM users").rows) + } + + @Test + fun notNullViolationOnUpdateAbortsTheWholeStatement() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("INSERT INTO users VALUES (1, 'ada'), (2, 'linus')") + assertFailsWith { db.run("UPDATE users SET name = NULL") } + // Neither row's name was actually cleared, because the failure aborted the transaction. + assertEquals(setOf(text("ada"), text("linus")), db.select("SELECT name FROM users").rows.map { it.single() }.toSet()) + } + + @Test + fun integerLiteralIsCoercedToRealForARealColumn() { + val db = newDatabase() + db.run("CREATE TABLE readings (id INTEGER PRIMARY KEY, value REAL NOT NULL)") + db.run("INSERT INTO readings VALUES (1, 5)") + assertEquals(Value.RealValue(5.0), db.select("SELECT value FROM readings").rows.single().single()) + } + + @Test + fun textLiteralIntoAnIntegerColumnIsRejectedWithATypeMismatch() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + assertFailsWith { db.run("INSERT INTO users VALUES ('x', 'ada')") } + } + + @Test + fun dataSurvivesClosingAndReopeningTheCatalogInWhatActsAsAFreshProcess() { + val db = newDatabase() + db.seedUsers() + db.run("UPDATE users SET age = 100 WHERE id = 1") + db.reopen() + val ages = db.select("SELECT name, age FROM users ORDER BY name").rows + assertEquals(listOf(listOf(text("ada"), int(100)), listOf(text("grace"), int(85)), listOf(text("linus"), int(54))), ages) + } + + private fun Database.seedOrders() { + run("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("INSERT INTO customers VALUES (1, 'ada'), (2, 'linus'), (3, 'grace')") + run("CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER NOT NULL, amount INTEGER NOT NULL)") + // grace (id 3) has no orders, so LEFT JOIN has something to pad and INNER JOIN has something to drop. + run("INSERT INTO orders VALUES (1, 1, 100), (2, 1, 50), (3, 2, 200)") + } + + // Adds an order referencing a customer id that does not exist, so RIGHT/FULL JOIN has an + // unmatched *right* row to pad with NULLs on top of grace's unmatched left row from seedOrders. + private fun Database.seedOrdersWithOrphan() { + seedOrders() + run("INSERT INTO orders VALUES (4, 99, 999)") + } + + @Test + fun rightJoinPadsUnmatchedRightRowsWithNulls() { + val db = newDatabase() + db.seedOrdersWithOrphan() + val result = db.select( + "SELECT customers.name, orders.amount FROM customers RIGHT JOIN orders ON customers.id = orders.customer_id ORDER BY orders.id" + ) + assertEquals( + listOf( + listOf(text("ada"), int(100)), + listOf(text("ada"), int(50)), + listOf(text("linus"), int(200)), + listOf(Value.NullValue, int(999)) + ), + result.rows + ) + } + + @Test + fun fullJoinKeepsBothUnmatchedLeftAndRightRows() { + val db = newDatabase() + db.seedOrdersWithOrphan() + val result = db.select( + "SELECT customers.name, orders.amount FROM customers FULL JOIN orders ON customers.id = orders.customer_id" + ) + assertEquals( + setOf( + listOf(text("ada"), int(100)), + listOf(text("ada"), int(50)), + listOf(text("linus"), int(200)), + listOf(text("grace"), Value.NullValue), + listOf(Value.NullValue, int(999)) + ), + result.rows.toSet() + ) + assertEquals(5, result.rows.size) + } + + @Test + fun crossJoinProducesTheFullCartesianProductWithNoOnClause() { + val db = newDatabase() + db.run("CREATE TABLE colors (name TEXT)") + db.run("CREATE TABLE sizes (label TEXT)") + db.run("INSERT INTO colors VALUES ('red'), ('blue')") + db.run("INSERT INTO sizes VALUES ('s'), ('m')") + val result = db.select("SELECT colors.name, sizes.label FROM colors CROSS JOIN sizes ORDER BY colors.name, sizes.label") + assertEquals( + listOf( + listOf(text("blue"), text("m")), listOf(text("blue"), text("s")), + listOf(text("red"), text("m")), listOf(text("red"), text("s")) + ), + result.rows + ) + } + + @Test + fun rightOuterAndFullOuterJoinSpellingsAreAccepted() { + val db = newDatabase() + db.seedOrders() + val right = db.select( + "SELECT customers.name FROM customers RIGHT OUTER JOIN orders ON customers.id = orders.customer_id ORDER BY orders.id" + ) + assertEquals(listOf(listOf(text("ada")), listOf(text("ada")), listOf(text("linus"))), right.rows) + val full = db.select( + "SELECT customers.name FROM customers FULL OUTER JOIN orders ON customers.id = orders.customer_id ORDER BY customers.id" + ) + assertEquals(4, full.rows.size) + } + + @Test + fun innerJoinCombinesMatchingRowsAndDropsUnmatchedLeftRows() { + val db = newDatabase() + db.seedOrders() + val result = db.select( + "SELECT customers.name, orders.amount FROM customers JOIN orders ON customers.id = orders.customer_id ORDER BY orders.id" + ) + assertEquals(listOf("name", "amount"), result.columns) + assertEquals( + listOf(listOf(text("ada"), int(100)), listOf(text("ada"), int(50)), listOf(text("linus"), int(200))), + result.rows + ) + } + + @Test + fun leftJoinPadsUnmatchedLeftRowsWithNulls() { + val db = newDatabase() + db.seedOrders() + val result = db.select( + "SELECT customers.name, orders.amount FROM customers LEFT JOIN orders ON customers.id = orders.customer_id ORDER BY customers.id, orders.id" + ) + assertEquals( + listOf( + listOf(text("ada"), int(100)), + listOf(text("ada"), int(50)), + listOf(text("linus"), int(200)), + listOf(text("grace"), Value.NullValue) + ), + result.rows + ) + } + + @Test + fun joinUsesTableAliasesToQualifyColumnsWithTheSameName() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER PRIMARY KEY, tag TEXT NOT NULL)") + db.run("CREATE TABLE b (id INTEGER PRIMARY KEY, tag TEXT NOT NULL)") + db.run("INSERT INTO a VALUES (1, 'left')") + db.run("INSERT INTO b VALUES (1, 'right')") + val result = db.select("SELECT x.tag, y.tag FROM a x JOIN b y ON x.id = y.id") + assertEquals(listOf(listOf(text("left"), text("right"))), result.rows) + } + + @Test + fun anUnqualifiedColumnPresentInBothJoinedTablesIsRejectedAsAmbiguous() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE b (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO a VALUES (1)") + db.run("INSERT INTO b VALUES (1)") + val failure = assertFailsWith { db.select("SELECT id FROM a JOIN b ON a.id = b.id") } + assertTrue(failure.message!!.contains("ambiguous")) + } + + @Test + fun aDuplicateTableAliasAcrossJoinedSourcesIsRejected() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER PRIMARY KEY)") + val failure = assertFailsWith { db.select("SELECT * FROM a x JOIN a x ON x.id = x.id") } + assertTrue(failure.message!!.contains("duplicate table alias")) + } + + @Test + fun joiningThreeTablesChainsLeftToRight() { + val db = newDatabase() + db.seedOrders() + db.run("CREATE TABLE shipments (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL, carrier TEXT NOT NULL)") + db.run("INSERT INTO shipments VALUES (1, 1, 'ups')") + val result = db.select( + "SELECT customers.name, orders.amount, shipments.carrier " + + "FROM customers JOIN orders ON customers.id = orders.customer_id " + + "JOIN shipments ON orders.id = shipments.order_id" + ) + assertEquals(listOf(listOf(text("ada"), int(100), text("ups"))), result.rows) + } + + @Test + fun joinAgainstADerivedTableWorks() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("CREATE TABLE b (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO a VALUES (1, 'ada'), (2, 'linus')") + db.run("INSERT INTO b VALUES (1)") + val result = db.select("SELECT a.name FROM a JOIN (SELECT id FROM b) sub ON a.id = sub.id") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + private fun Database.seedDerivedTableUsers() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER, dept TEXT)") + run( + "INSERT INTO users VALUES " + + "(1, 'ada', 36, 'eng'), (2, 'linus', 54, 'eng'), (3, 'grace', 85, 'navy'), (4, 'alan', 41, NULL)" + ) + } + + @Test + fun selectsFromASimpleDerivedTable() { + val db = newDatabase() + db.seedDerivedTableUsers() + val result = db.select("SELECT s.name FROM (SELECT id, name FROM users WHERE age >= 40) AS s ORDER BY s.id ASC") + assertEquals(listOf(text("linus"), text("grace"), text("alan")), result.rows.map { it.single() }) + } + + @Test + fun derivedTableOverAnAggregateQueryWorks() { + val db = newDatabase() + db.seedDerivedTableUsers() + val result = db.select( + "SELECT d.dept, d.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d " + + "WHERE d.n > 1 ORDER BY d.dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } + + @Test + fun twoDerivedTablesCanBeJoinedTogether() { + val db = newDatabase() + db.seedDerivedTableUsers() + val result = db.select( + "SELECT a.dept, a.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS a " + + "JOIN (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS b ON a.dept = b.dept " + + "WHERE a.n = b.n ORDER BY a.dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(2)), listOf(text("navy"), int(1))), result.rows) + } + + @Test + fun derivedTableColumnsAreScopedToItsAliasNotTheUnderlyingTableName() { + val db = newDatabase() + db.seedDerivedTableUsers() + assertFailsWith { db.run("SELECT users.name FROM (SELECT id, name FROM users) AS s") } + } + + @Test + fun derivedTableIsReEvaluatedEachQueryReflectingLiveData() { + val db = newDatabase() + db.seedDerivedTableUsers() + val before = db.select("SELECT COUNT(*) AS n FROM (SELECT id FROM users WHERE dept = 'eng') AS e") + assertEquals(listOf(listOf(int(2))), before.rows) + db.run("INSERT INTO users VALUES (5, 'margaret', 61, 'eng')") + val after = db.select("SELECT COUNT(*) AS n FROM (SELECT id FROM users WHERE dept = 'eng') AS e") + assertEquals(listOf(listOf(int(3))), after.rows) + } + + @Test + fun derivedTableCanContainACorrelatedSubqueryOfItsOwn() { + val db = newDatabase() + db.seedDerivedTableUsers() + val result = db.select( + "SELECT s.name FROM (SELECT id, name, dept FROM users u WHERE age > " + + "(SELECT AVG(age) FROM users u2 WHERE u2.dept = u.dept)) AS s ORDER BY s.id ASC" + ) + assertEquals(listOf(text("linus")), result.rows.map { it.single() }) + } + + @Test + fun creatingAViewFromADerivedTableWorks() { + val db = newDatabase() + db.seedDerivedTableUsers() + db.run("CREATE VIEW dept_counts AS SELECT d.dept, d.n FROM (SELECT dept, COUNT(*) AS n FROM users GROUP BY dept) AS d") + val result = db.select("SELECT dept, n FROM dept_counts WHERE n > 1") + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } + + private fun Database.seedDepartmentsAndSalaries() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'sales')") + run( + "INSERT INTO employees VALUES " + + "(1, 1, 'ada', 300), (2, 1, 'linus', 250), (3, 1, 'grace', 400), " + + "(4, 2, 'sam', 150), (5, 2, 'kay', 200)" + ) + } + + @Test + fun lateralJoinComputesTopOnePerGroupWhichAPlainJoinCannotExpress() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + val result = db.select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "JOIN LATERAL (SELECT e.name, e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf( + listOf(text("eng"), text("grace"), int(400)), + listOf(text("sales"), text("kay"), int(200)) + ), + result.rows + ) + } + + @Test + fun lateralJoinWithLimitGreaterThanOneReturnsMultipleRowsPerLeftRow() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + val result = db.select( + "SELECT d.name, top.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 2) AS top ON TRUE " + + "ORDER BY d.name ASC, top.name ASC" + ) + assertEquals( + listOf( + listOf(text("eng"), text("ada")), + listOf(text("eng"), text("grace")), + listOf(text("sales"), text("kay")), + listOf(text("sales"), text("sam")) + ), + result.rows + ) + } + + @Test + fun innerJoinLateralDropsGroupsWithNoMatchingRows() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + db.run("INSERT INTO departments VALUES (3, 'empty')") + val result = db.select( + "SELECT d.name FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id LIMIT 1) AS top ON TRUE" + ) + assertTrue(result.rows.none { it == listOf(text("empty")) }) + assertEquals(2, result.rows.size) + } + + @Test + fun leftJoinLateralKeepsGroupsWithNoMatchingRowsAsNull() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + db.run("INSERT INTO departments VALUES (3, 'empty')") + val result = db.select( + "SELECT d.name, top.name FROM departments d " + + "LEFT JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf( + listOf(text("empty"), Value.NullValue), + listOf(text("eng"), text("grace")), + listOf(text("sales"), text("kay")) + ), + result.rows + ) + } + + @Test + fun leftJoinLateralOverAnEmptyBaseTableProducesNoRowsWithoutRunningTheSubquery() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT)") + val result = db.select( + "SELECT d.name, top.name FROM departments d " + + "LEFT JOIN LATERAL (SELECT e.name FROM employees e WHERE e.deptId = d.id) AS top ON TRUE" + ) + assertEquals(emptyList(), result.rows) + } + + @Test + fun chainedLateralJoinsSeeEarlierLateralAliases() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + db.run("CREATE TABLE dual (x INTEGER)") + db.run("INSERT INTO dual VALUES (1)") + val result = db.select( + "SELECT d.name, bonus.amount FROM departments d " + + "JOIN LATERAL (SELECT e.salary FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "JOIN LATERAL (SELECT top.salary / 10 AS amount FROM dual) AS bonus ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(40)), listOf(text("sales"), int(20))), result.rows) + } + + @Test + fun lateralSubqueryCannotReferenceATableJoinedLater() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + val failure = assertFailsWith { + db.select( + "SELECT * FROM departments d " + + "JOIN LATERAL (SELECT e.name FROM employees e WHERE e.salary = later.salary) AS top ON TRUE " + + "JOIN employees later ON later.deptId = d.id" + ) + } + assertTrue(failure.message!!.contains("not joined yet")) + } + + @Test + fun lateralSubqueryWithStarProjectionResolvesColumnsStructurallyFromItsBaseTable() { + val db = newDatabase() + db.seedDepartmentsAndSalaries() + val result = db.select( + "SELECT d.name, top.name, top.salary FROM departments d " + + "JOIN LATERAL (SELECT * FROM employees e WHERE e.deptId = d.id " + + "ORDER BY e.salary DESC LIMIT 1) AS top ON TRUE " + + "ORDER BY d.name ASC" + ) + assertEquals( + listOf(listOf(text("eng"), text("grace"), int(400)), listOf(text("sales"), text("kay"), int(200))), + result.rows + ) + } + + @Test + fun groupByComputesCountSumAvgMinMaxPerGroup() { + val db = newDatabase() + db.seedOrders() + val result = db.select( + "SELECT customer_id, COUNT(*), SUM(amount), AVG(amount), MIN(amount), MAX(amount) " + + "FROM orders GROUP BY customer_id ORDER BY customer_id" + ) + assertEquals( + listOf( + listOf(int(1), int(2), int(150), Value.RealValue(75.0), int(50), int(100)), + listOf(int(2), int(1), int(200), Value.RealValue(200.0), int(200), int(200)) + ), + result.rows + ) + } + + /** + * STRING_AGG's implementation lives entirely in the shared [kestrel.engine.GroupAggregation]/ + * [kestrel.engine.AggregateFunctions] classes that this disk-backed executor already reuses for + * every other aggregate - this proves that reuse actually extends to STRING_AGG too, not just that + * the in-memory engine (AggregateTest) got it right. + */ + @Test + fun stringAggGroupsAndOrdersPerGroupOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE tags (item TEXT, label TEXT)") + db.run("INSERT INTO tags VALUES ('widget', 'blue'), ('widget', 'small'), ('gadget', 'red')") + val result = db.select( + "SELECT item, STRING_AGG(label, ', ' ORDER BY label) FROM tags GROUP BY item ORDER BY item" + ) + assertEquals( + listOf(listOf(text("gadget"), text("red")), listOf(text("widget"), text("blue, small"))), + result.rows + ) + } + + /** + * `COUNT/SUM/AVG/MIN/MAX(DISTINCT ...)` dedup lives in the same shared [kestrel.engine. + * AggregateFunctions.compute] this executor already reuses for every other aggregate - this + * proves the DISTINCT dedup path itself, not just plain aggregation, actually runs on the + * disk-backed engine too. + */ + @Test + fun countAndSumDistinctDedupOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE t (dept TEXT, amount INTEGER)") + db.run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + assertEquals(listOf(listOf(int(4))), db.select("SELECT COUNT(amount) FROM t").rows) + assertEquals(listOf(listOf(int(2))), db.select("SELECT COUNT(DISTINCT amount) FROM t").rows) + assertEquals(listOf(listOf(int(500))), db.select("SELECT SUM(amount) FROM t").rows) + assertEquals(listOf(listOf(int(300))), db.select("SELECT SUM(DISTINCT amount) FROM t").rows) + } + + @Test + fun countDistinctPerGroupOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE t (dept TEXT, amount INTEGER)") + db.run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + val result = db.select("SELECT dept, COUNT(DISTINCT amount) FROM t GROUP BY dept ORDER BY dept ASC") + assertEquals(listOf(listOf(text("eng"), int(2)), listOf(text("ops"), int(1))), result.rows) + } + + /** + * `FILTER (WHERE ...)` narrows the rows an aggregate call itself sees before + * [kestrel.engine.AggregateFunctions.compute] runs - lives in the same shared + * [kestrel.engine.GroupAggregation.computeAggregate] this executor already reuses for every other + * aggregate, so this proves the filtering itself, not just plain/DISTINCT aggregation, runs here too. + */ + @Test + fun filterRestrictsWhichRowsEachAggregateSeesOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE t (dept TEXT, amount INTEGER)") + db.run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + assertEquals(listOf(listOf(int(1))), db.select("SELECT COUNT(*) FILTER (WHERE amount > 100) FROM t").rows) + assertEquals(listOf(listOf(int(400))), db.select("SELECT SUM(amount) FILTER (WHERE dept = 'eng') FROM t").rows) + assertEquals( + listOf(listOf(int(0), Value.NullValue)), + db.select("SELECT COUNT(*) FILTER (WHERE dept = 'nope') AS c, SUM(amount) FILTER (WHERE dept = 'nope') AS s FROM t").rows + ) + } + + @Test + fun filterCombinesWithDistinctAndGroupByOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE t (dept TEXT, amount INTEGER)") + db.run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + val result = db.select( + "SELECT dept, COUNT(DISTINCT amount) FILTER (WHERE amount > 50) AS matching FROM t GROUP BY dept ORDER BY dept ASC" + ) + assertEquals(listOf(listOf(text("eng"), int(2)), listOf(text("ops"), int(1))), result.rows) + } + + @Test + fun offsetSkipsGroupsAfterOrderingAggregatedResultsOnTheDiskBackedEngine() { + val db = newDatabase() + db.run("CREATE TABLE t (dept TEXT, amount INTEGER)") + db.run("INSERT INTO t VALUES ('eng', 100), ('eng', 100), ('eng', 200), ('ops', 100), ('ops', NULL)") + val result = db.select("SELECT dept, COUNT(DISTINCT amount) FROM t GROUP BY dept ORDER BY dept ASC LIMIT 1 OFFSET 1") + assertEquals(listOf(listOf(text("ops"), int(1))), result.rows) + } + + @Test + fun countStarWithNoGroupByAggregatesTheWholeTable() { + val db = newDatabase() + db.seedOrders() + assertEquals(listOf(listOf(int(3))), db.select("SELECT COUNT(*) FROM orders").rows) + } + + @Test + fun countStarOverAnEmptyTableStillProducesOneRow() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER NOT NULL)") + assertEquals(listOf(listOf(int(0))), db.select("SELECT COUNT(*) FROM orders").rows) + } + + @Test + fun groupByRollupAddsPerCustomerSubtotalsAndAGrandTotal() { + val db = newDatabase() + db.seedOrders() + val result = db.select( + "SELECT customer_id, SUM(amount) AS total FROM orders GROUP BY ROLLUP (customer_id) " + + "ORDER BY GROUPING(customer_id), customer_id" + ) + assertEquals( + listOf( + listOf(int(1), int(150)), + listOf(int(2), int(200)), + listOf(Value.NullValue, int(350)) + ), + result.rows + ) + } + + @Test + fun havingFiltersOutGroupsThatDoNotMatch() { + val db = newDatabase() + db.seedOrders() + val result = db.select("SELECT customer_id, SUM(amount) FROM orders GROUP BY customer_id HAVING SUM(amount) > 150") + assertEquals(listOf(listOf(int(2), int(200))), result.rows) + } + + @Test + fun orderByOnAnAggregateSortsTheGroupedResult() { + val db = newDatabase() + db.seedOrders() + val result = db.select("SELECT customer_id, SUM(amount) AS total FROM orders GROUP BY customer_id ORDER BY total DESC") + assertEquals(listOf(listOf(int(2), int(200)), listOf(int(1), int(150))), result.rows) + } + + @Test + fun aProjectedColumnNotInGroupByOrAnAggregateIsRejected() { + val db = newDatabase() + db.seedOrders() + val failure = assertFailsWith { + db.select("SELECT customer_id, amount FROM orders GROUP BY customer_id") + } + assertTrue(failure.message!!.contains("must appear in GROUP BY")) + } + + @Test + fun starProjectionCombinedWithGroupByIsRejected() { + val db = newDatabase() + db.seedOrders() + val failure = assertFailsWith { db.select("SELECT * FROM orders GROUP BY customer_id") } + assertTrue(failure.message!!.contains("cannot use *")) + } + + @Test + fun groupByAggregatesOverAJoinedResult() { + val db = newDatabase() + db.seedOrders() + val result = db.select( + "SELECT customers.name, SUM(orders.amount) FROM customers JOIN orders ON customers.id = orders.customer_id " + + "GROUP BY customers.name ORDER BY customers.name" + ) + assertEquals(listOf(listOf(text("ada"), int(150)), listOf(text("linus"), int(200))), result.rows) + } + + @Test + fun dropTableCascadeWithNoDependentsBehavesLikeAPlainDrop() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY)") + db.run("DROP TABLE users CASCADE") + assertTrue(db.catalog.tableNames.isEmpty()) + } + + @Test + fun createIndexBuildsANonUniqueSingleColumnSecondaryIndex() { + val db = newDatabase() + db.seedUsers() + val created = db.run("CREATE INDEX idx_age ON users(age)").single() + assertEquals(CreateIndexResult("idx_age"), created) + assertEquals(1, db.catalog.table("users").secondaryIndexes.size) + } + + @Test + fun createUniqueIndexBuildsAUniqueSingleColumnSecondaryIndex() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER)") + val created = db.run("CREATE UNIQUE INDEX idx_age ON users(age)").single() + assertEquals(CreateIndexResult("idx_age"), created) + assertTrue(db.catalog.table("users").secondaryIndexes.single().unique) + } + + @Test + fun createUniqueIndexRejectsAnInsertThatWouldDuplicateAnExistingValue() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + assertFailsWith { db.run("INSERT INTO users VALUES (2, 'ada@example.com')") } + assertEquals(1, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun createUniqueIndexAllowsMultipleNullsSinceNullsAreNeverDuplicates() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, NULL), (2, NULL)") + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun createUniqueIndexOverExistingDuplicateValuesFailsAndLeavesNoTraceOfTheIndex() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("INSERT INTO users VALUES (1, 'dup@example.com'), (2, 'dup@example.com')") + assertFailsWith { db.run("CREATE UNIQUE INDEX idx_email ON users(email)") } + assertTrue(db.catalog.table("users").secondaryIndexes.isEmpty()) + } + + @Test + fun createUniqueIndexRejectsAnUpdateThatWouldDuplicateAnotherRowsValue() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com'), (2, 'grace@example.com')") + assertFailsWith { db.run("UPDATE users SET email = 'ada@example.com' WHERE id = 2") } + assertEquals(text("grace@example.com"), db.select("SELECT email FROM users WHERE id = 2").rows.single().single()) + } + + @Test + fun createUniqueIndexDataAndConstraintSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + db.reopen() + assertFailsWith { db.run("INSERT INTO users VALUES (2, 'ada@example.com')") } + assertTrue(db.catalog.table("users").secondaryIndexes.single().unique) + } + + @Test + fun dropIndexRemovesItFromTheCatalog() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val dropped = db.run("DROP INDEX idx_age").single() + assertEquals(DropIndexResult("idx_age"), dropped) + assertTrue(db.catalog.table("users").secondaryIndexes.isEmpty()) + } + + @Test + fun dropIndexStopsThePlannerFromPushingDownAFormerlyIndexedEquality() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + db.run("DROP INDEX idx_age") + val table = db.catalog.table("users") + val scansBefore = table.scanCount + db.select("SELECT * FROM users WHERE age = 54") + assertEquals(0, table.secondaryLookupCount) + assertEquals(scansBefore + 1, table.scanCount) + } + + @Test + fun droppingAUniqueIndexStopsItFromRejectingFurtherDuplicates() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + db.run("DROP INDEX idx_email") + db.run("INSERT INTO users VALUES (2, 'ada@example.com')") + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun dropIndexOnACompositeIndexRemovesTheWholeIndexNotJustOneColumn() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_both ON users(age, name)") + db.run("DROP INDEX idx_both") + assertTrue(db.catalog.table("users").secondaryIndexes.isEmpty()) + } + + @Test + fun droppingAnUnknownIndexFails() { + val db = newDatabase() + db.seedUsers() + assertFailsWith { db.run("DROP INDEX ghost") } + } + + @Test + fun droppingAnAlreadyDroppedIndexFails() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + db.run("DROP INDEX idx_age") + assertFailsWith { db.run("DROP INDEX idx_age") } + } + + @Test + fun dropIndexIsPersistedAcrossClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + db.run("CREATE UNIQUE INDEX idx_email ON users(email)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + db.run("DROP INDEX idx_email") + db.reopen() + assertTrue(db.catalog.table("users").secondaryIndexes.isEmpty()) + db.run("INSERT INTO users VALUES (2, 'ada@example.com')") + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun dropIndexCannotRunInsideAnExplicitTransaction() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + db.run("BEGIN") + assertFailsWith { db.run("DROP INDEX idx_age") } + db.run("ROLLBACK") + } + + // --- declarative UNIQUE constraints (inline `col TYPE UNIQUE` / table-level `UNIQUE (a, b, ...)`), + // enforced by a full heap scan per write rather than a backing index, mirroring + // kestrel.engine.Executor's own uniqueConstraints handling. + + @Test + fun inlineUniqueColumnRejectsADuplicateValueOnInsert() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + assertFailsWith { db.run("INSERT INTO users VALUES (2, 'ada@example.com')") } + assertEquals(1, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun inlineUniqueColumnAllowsMultipleNulls() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO users VALUES (1, NULL), (2, NULL)") + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun tableLevelCompositeUniqueConstraintRejectsADuplicateColumnCombination() { + val db = newDatabase() + db.run("CREATE TABLE registrations (event_id INTEGER, user_id INTEGER, UNIQUE (event_id, user_id))") + db.run("INSERT INTO registrations VALUES (1, 100)") + assertFailsWith { db.run("INSERT INTO registrations VALUES (1, 100)") } + // Sharing only one of the two columns is fine: the constraint is on the pair, not either column alone. + db.run("INSERT INTO registrations VALUES (1, 200)") + assertEquals(2, db.select("SELECT * FROM registrations").rows.size) + } + + @Test + fun uniqueConstraintRejectsAnUpdateThatWouldDuplicateAnotherRowsValue() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com'), (2, 'grace@example.com')") + assertFailsWith { db.run("UPDATE users SET email = 'ada@example.com' WHERE id = 2") } + } + + @Test + fun uniqueConstraintReferencingAnUnknownColumnIsRejectedWithAClearError() { + val db = newDatabase() + val failure = assertFailsWith { + db.run("CREATE TABLE bad (id INTEGER PRIMARY KEY, UNIQUE (missing_column))") + } + assertTrue(failure.message!!.contains("missing_column")) + } + + @Test + fun uniqueConstraintDataAndEnforcementSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO users VALUES (1, 'ada@example.com')") + db.reopen() + assertFailsWith { db.run("INSERT INTO users VALUES (2, 'ada@example.com')") } + assertEquals(1, db.select("SELECT * FROM users").rows.size) + } + + // --- composite (multi-column) CREATE [UNIQUE] INDEX + + @Test + fun createIndexBuildsANonUniqueCompositeSecondaryIndex() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + val created = db.run("CREATE INDEX idx_both ON users(age, name)").single() + assertEquals(CreateIndexResult("idx_both"), created) + val handle = db.catalog.table("users").secondaryIndexes.single() + assertEquals(listOf("age", "name"), handle.columnNames) + assertEquals(false, handle.unique) + } + + @Test + fun createUniqueCompositeIndexRejectsADuplicateColumnCombinationOnInsert() { + val db = newDatabase() + db.run("CREATE TABLE registrations (event_id INTEGER, user_id INTEGER)") + db.run("CREATE UNIQUE INDEX idx_reg ON registrations(event_id, user_id)") + db.run("INSERT INTO registrations VALUES (1, 100)") + assertFailsWith { db.run("INSERT INTO registrations VALUES (1, 100)") } + // Sharing only one of the two columns is fine: the index is on the pair, not either column alone. + db.run("INSERT INTO registrations VALUES (1, 200)") + assertEquals(2, db.select("SELECT * FROM registrations").rows.size) + } + + @Test + fun createUniqueCompositeIndexAllowsMultipleRowsWithANullInEitherColumn() { + val db = newDatabase() + db.run("CREATE TABLE registrations (event_id INTEGER, user_id INTEGER)") + db.run("CREATE UNIQUE INDEX idx_reg ON registrations(event_id, user_id)") + db.run("INSERT INTO registrations VALUES (1, NULL), (1, NULL), (NULL, 100), (NULL, 100)") + assertEquals(4, db.select("SELECT * FROM registrations").rows.size) + } + + @Test + fun createUniqueCompositeIndexOverExistingDuplicateCombinationsFailsAndLeavesNoTraceOfTheIndex() { + val db = newDatabase() + db.run("CREATE TABLE registrations (event_id INTEGER, user_id INTEGER)") + db.run("INSERT INTO registrations VALUES (1, 100), (1, 100)") + assertFailsWith { db.run("CREATE UNIQUE INDEX idx_reg ON registrations(event_id, user_id)") } + assertTrue(db.catalog.table("registrations").secondaryIndexes.isEmpty()) + } + + @Test + fun createUniqueCompositeIndexRejectsAnUpdateThatWouldDuplicateAnotherRowsCombination() { + val db = newDatabase() + db.run("CREATE TABLE registrations (id INTEGER PRIMARY KEY, event_id INTEGER, user_id INTEGER)") + db.run("CREATE UNIQUE INDEX idx_reg ON registrations(event_id, user_id)") + db.run("INSERT INTO registrations VALUES (1, 1, 100), (2, 1, 200)") + assertFailsWith { db.run("UPDATE registrations SET user_id = 100 WHERE id = 2") } + assertEquals(int(200), db.select("SELECT user_id FROM registrations WHERE id = 2").rows.single().single()) + } + + @Test + fun createUniqueCompositeIndexDataAndConstraintSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE registrations (event_id INTEGER, user_id INTEGER)") + db.run("CREATE UNIQUE INDEX idx_reg ON registrations(event_id, user_id)") + db.run("INSERT INTO registrations VALUES (1, 100)") + db.reopen() + assertFailsWith { db.run("INSERT INTO registrations VALUES (1, 100)") } + val handle = db.catalog.table("registrations").secondaryIndexes.single() + assertEquals(listOf("event_id", "user_id"), handle.columnNames) + assertTrue(handle.unique) + } + + // --- index pushdown: proven by DiskTable's access counters, not just by correct results, + // since a full scan and an index lookup return identical rows for these queries. + + @Test + fun whereEqualityOnThePrimaryKeyUsesAPointLookupNotAFullScan() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val result = db.select("SELECT name FROM users WHERE id = 2") + assertEquals(listOf(listOf(text("linus"))), result.rows) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.scanCount) + } + + @Test + fun aMissingPrimaryKeyPointLookupReturnsNoRowsWithoutScanning() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val result = db.select("SELECT name FROM users WHERE id = 999") + assertEquals(emptyList>(), result.rows) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.scanCount) + } + + @Test + fun aRangeOnThePrimaryKeyUsesARangeScanNotAFullScan() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val result = db.select("SELECT name FROM users WHERE id > 1 AND id <= 3 ORDER BY id") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + assertEquals(1, table.rangeScanCount) + assertEquals(0, table.scanCount) + } + + @Test + fun equalityOnAnUnindexedColumnStillFallsBackToAFullScan() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val result = db.select("SELECT id FROM users WHERE name = 'linus'") + assertEquals(listOf(listOf(int(2))), result.rows) + assertEquals(1, table.scanCount) + assertEquals(0, table.pointLookupCount) + } + + @Test + fun equalityOnASecondaryIndexedColumnUsesTheSecondaryIndexNotAFullScan() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + val scansBeforeSelect = table.scanCount // backfilling the new index already scanned once + val result = db.select("SELECT name FROM users WHERE age = 54") + assertEquals(listOf(listOf(text("linus"))), result.rows) + assertEquals(1, table.secondaryLookupCount) + assertEquals(scansBeforeSelect, table.scanCount) + } + + @Test + fun equalityOnAllColumnsOfACompositeSecondaryIndexUsesTheSecondaryIndexNotAFullScan() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + db.run("INSERT INTO users VALUES (1, 36, 'ada'), (2, 36, 'grace')") + db.run("CREATE INDEX idx_both ON users(age, name)") + val table = db.catalog.table("users") + val scansBeforeSelect = table.scanCount // backfilling the new index already scanned once + val result = db.select("SELECT id FROM users WHERE age = 36 AND name = 'grace'") + assertEquals(listOf(listOf(int(2))), result.rows) + assertEquals(1, table.secondaryLookupCount) + assertEquals(scansBeforeSelect, table.scanCount) + } + + @Test + fun equalityOnOnlyOneColumnOfACompositeSecondaryIndexFallsBackToAFullScan() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, age INTEGER, name TEXT)") + db.run("INSERT INTO users VALUES (1, 36, 'ada'), (2, 36, 'grace')") + db.run("CREATE INDEX idx_both ON users(age, name)") + val table = db.catalog.table("users") + val scansBeforeSelect = table.scanCount + val result = db.select("SELECT id FROM users WHERE age = 36") + assertEquals(2, result.rows.size) + assertEquals(0, table.secondaryLookupCount) + assertEquals(scansBeforeSelect + 1, table.scanCount) + } + + @Test + fun aRangeOnASingleColumnSecondaryIndexUsesARangeScanNotAFullScan() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + val scansBeforeSelect = table.scanCount // backfilling the new index already scanned once + val result = db.select("SELECT name FROM users WHERE age > 40 AND age <= 85 ORDER BY age") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + assertEquals(1, table.rangeScanCount) + assertEquals(0, table.secondaryLookupCount) + assertEquals(scansBeforeSelect, table.scanCount) + } + + @Test + fun aRangeOnTheFirstColumnOfACompositeSecondaryIndexUsesARangeScanNotAFullScan() { + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT)") + db.run("INSERT INTO order_lines VALUES (1, 1, 'apple'), (2, 1, 'kiwi'), (3, 1, 'mango')") + db.run("CREATE INDEX idx_order_sku ON order_lines(order_id, sku)") + val table = db.catalog.table("order_lines") + val scansBeforeSelect = table.scanCount // backfilling the new index already scanned once + val result = db.select("SELECT sku FROM order_lines WHERE order_id > 1 ORDER BY order_id") + assertEquals(listOf(listOf(text("kiwi")), listOf(text("mango"))), result.rows) + assertEquals(1, table.rangeScanCount) + assertEquals(scansBeforeSelect, table.scanCount) + } + + @Test + fun aRangeOnTheSecondColumnOfACompositeSecondaryIndexAfterAnEqualityPrefixUsesARangeScanNotAFullScan() { + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT)") + db.run("INSERT INTO order_lines VALUES (1, 1, 'apple'), (1, 2, 'mango'), (2, 1, 'kiwi')") + db.run("CREATE INDEX idx_order_sku ON order_lines(order_id, sku)") + val table = db.catalog.table("order_lines") + val scansBeforeSelect = table.scanCount // backfilling the new index already scanned once + val result = db.select("SELECT line_no FROM order_lines WHERE order_id = 1 AND sku > 'apple'") + assertEquals(listOf(listOf(int(2))), result.rows) + assertEquals(1, table.rangeScanCount) + assertEquals(0, table.secondaryLookupCount) + assertEquals(scansBeforeSelect, table.scanCount) + } + + @Test + fun aRangeOnTheSecondColumnOfACompositeSecondaryIndexWithoutAnEqualityPrefixFallsBackToAFullScan() { + // Mirrors equalityOnOnlyOneColumnOfACompositeSecondaryIndexFallsBackToAFullScan - the index + // is sorted by order_id first, so a bare range on sku cannot narrow the scan without an + // equality prefix pinning order_id down first. + val db = newDatabase() + db.run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT)") + db.run("INSERT INTO order_lines VALUES (1, 1, 'apple'), (1, 2, 'mango'), (2, 1, 'kiwi')") + db.run("CREATE INDEX idx_order_sku ON order_lines(order_id, sku)") + val table = db.catalog.table("order_lines") + val scansBeforeSelect = table.scanCount + val result = db.select("SELECT line_no FROM order_lines WHERE sku > 'apple'") + assertEquals(2, result.rows.size) + assertEquals(0, table.rangeScanCount) + assertEquals(scansBeforeSelect + 1, table.scanCount) + } + + @Test + fun equalityOnASecondaryIndexIsPreferredOverARangeOnTheSameColumnAtRuntime() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + val result = db.select("SELECT name FROM users WHERE age = 54 AND age > 0") + assertEquals(listOf(listOf(text("linus"))), result.rows) + assertEquals(1, table.secondaryLookupCount) + assertEquals(0, table.rangeScanCount) + } + + @Test + fun aPrimaryKeyEqualityIsPreferredOverASecondaryIndexInTheSamePredicate() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + val table = db.catalog.table("users") + // age = 999 matches nobody, but a point lookup on id must still run (and the extra + // conjunct is applied afterwards by the ordinary full-predicate filter), proving the + // planner picked the primary key rather than the secondary index for this predicate. + val result = db.select("SELECT name FROM users WHERE id = 2 AND age = 999") + assertEquals(emptyList>(), result.rows) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.secondaryLookupCount) + } + + @Test + fun updateWithAPrimaryKeyEqualityWhereUsesAPointLookupNotAFullScan() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val updated = db.run("UPDATE users SET age = 99 WHERE id = 1").single() + assertEquals(UpdateResult(1), updated) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.scanCount) + assertEquals(int(99), db.select("SELECT age FROM users WHERE id = 1").rows.single().single()) + } + + @Test + fun deleteWithAPrimaryKeyEqualityWhereUsesAPointLookupNotAFullScan() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val deleted = db.run("DELETE FROM users WHERE id = 3").single() + assertEquals(DeleteResult(1), deleted) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.scanCount) + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + // --- table-level composite PRIMARY KEY (a, b): PRIMARY KEY (a, b) instead of an inline + // `col TYPE PRIMARY KEY`, mirroring kestrel.engine.Executor's own handling of the same clause. + + private fun Database.seedOrderLines() { + run("CREATE TABLE order_lines (order_id INTEGER, line_no INTEGER, sku TEXT, PRIMARY KEY (order_id, line_no))") + run("INSERT INTO order_lines VALUES (1, 1, 'widget'), (1, 2, 'gadget'), (2, 1, 'widget')") + } + + @Test + fun compositePrimaryKeyColumnsAreImplicitlyNotNullLikeAnInlinePrimaryKey() { + val db = newDatabase() + db.seedOrderLines() + assertFailsWith { db.run("INSERT INTO order_lines (line_no, sku) VALUES (1, 'x')") } + } + + @Test + fun compositePrimaryKeyRejectsADuplicateColumnCombinationButAllowsSharedPartialValues() { + val db = newDatabase() + db.seedOrderLines() + assertFailsWith { db.run("INSERT INTO order_lines VALUES (1, 1, 'duplicate')") } + assertEquals(3, db.select("SELECT * FROM order_lines").rows.size) + } + + @Test + fun compositePrimaryKeyEqualityOnAllColumnsUsesAPointLookupNotAFullScan() { + val db = newDatabase() + db.seedOrderLines() + val table = db.catalog.table("order_lines") + val result = db.select("SELECT sku FROM order_lines WHERE order_id = 1 AND line_no = 2") + assertEquals(listOf(listOf(text("gadget"))), result.rows) + assertEquals(1, table.pointLookupCount) + assertEquals(0, table.scanCount) + } + + @Test + fun compositePrimaryKeyEqualityOnOnlyOneColumnFallsBackToAFullScan() { + val db = newDatabase() + db.seedOrderLines() + val table = db.catalog.table("order_lines") + val result = db.select("SELECT sku FROM order_lines WHERE order_id = 1") + assertEquals(setOf("widget", "gadget"), result.rows.map { (it.single() as Value.TextValue).value }.toSet()) + assertEquals(1, table.scanCount) + assertEquals(0, table.pointLookupCount) + } + + @Test + fun aRangeOnTheLastCompositePrimaryKeyColumnAfterAnEqualityPrefixUsesARangeScanNotAFullScan() { + val db = newDatabase() + db.seedOrderLines() + val table = db.catalog.table("order_lines") + val result = db.select("SELECT sku FROM order_lines WHERE order_id = 1 AND line_no > 1") + assertEquals(listOf(listOf(text("gadget"))), result.rows) + assertEquals(1, table.rangeScanCount) + assertEquals(0, table.scanCount) + } + + @Test + fun aTwoSidedRangeOnTheLastCompositePrimaryKeyColumnAfterAnEqualityPrefixUsesARangeScan() { + val db = newDatabase() + db.seedOrderLines() + db.run("INSERT INTO order_lines VALUES (1, 3, 'sprocket')") + val table = db.catalog.table("order_lines") + val result = db.select("SELECT sku FROM order_lines WHERE order_id = 1 AND line_no >= 1 AND line_no <= 2") + assertEquals(setOf("widget", "gadget"), result.rows.map { (it.single() as Value.TextValue).value }.toSet()) + assertEquals(1, table.rangeScanCount) + assertEquals(0, table.scanCount) + } + + @Test + fun aRangeOnAnyOtherCompositePrimaryKeyColumnWithoutTheFullEqualityPrefixFallsBackToAFullScan() { + val db = newDatabase() + db.seedOrderLines() + val table = db.catalog.table("order_lines") + val result = db.select("SELECT sku FROM order_lines WHERE order_id > 1") + assertEquals(listOf(listOf(text("widget"))), result.rows) + assertEquals(1, table.scanCount) + assertEquals(0, table.rangeScanCount) + } + + @Test + fun updateAndDeleteByFullCompositePrimaryKeyUsePointLookups() { + val db = newDatabase() + db.seedOrderLines() + val table = db.catalog.table("order_lines") + + val updated = db.run("UPDATE order_lines SET sku = 'sprocket' WHERE order_id = 1 AND line_no = 1").single() + assertEquals(UpdateResult(1), updated) + assertEquals(1, table.pointLookupCount) + + val deleted = db.run("DELETE FROM order_lines WHERE order_id = 2 AND line_no = 1").single() + assertEquals(DeleteResult(1), deleted) + assertEquals(2, table.pointLookupCount) + assertEquals(0, table.scanCount) + assertEquals(2, db.select("SELECT * FROM order_lines").rows.size) + } + + @Test + fun aTableCannotDeclareBothAnInlinePrimaryKeyColumnAndATableLevelCompositePrimaryKey() { + val db = newDatabase() + assertFailsWith { + db.run("CREATE TABLE bad (id INTEGER PRIMARY KEY, other INTEGER, PRIMARY KEY (id, other))") + } + } + + @Test + fun aCompositePrimaryKeyReferencingAnUnknownColumnIsRejectedWithAClearError() { + val db = newDatabase() + val failure = assertFailsWith { + db.run("CREATE TABLE bad (order_id INTEGER, line_no INTEGER, PRIMARY KEY (order_id, missing_column))") + } + assertTrue(failure.message!!.contains("missing_column")) + } + + @Test + fun compositePrimaryKeyDataAndUniquenessSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedOrderLines() + db.reopen() + + assertFailsWith { db.run("INSERT INTO order_lines VALUES (1, 1, 'duplicate')") } + val result = db.select("SELECT sku FROM order_lines WHERE order_id = 1 AND line_no = 2") + assertEquals(listOf(listOf(text("gadget"))), result.rows) + val table = db.catalog.table("order_lines") + assertEquals(0, table.scanCount, "the point lookup above must not have fallen back to a full scan after reopening") + } + + private fun Database.seedDepartments() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL, budget INTEGER)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, department_id INTEGER, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng', 1000), (2, 'sales', 500), (3, 'ops', 0)") + run( + "INSERT INTO employees VALUES " + + "(1, 'ada', 1, 120), (2, 'linus', 1, 90), (3, 'grace', 2, 80), (4, 'guido', 2, 70)" + ) + } + + @Test + fun scalarSubqueryComparesAgainstAnUncorrelatedAggregate() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT name FROM employees WHERE salary > (SELECT AVG(salary) FROM employees) ORDER BY name") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun scalarSubqueryReturningNoRowsEvaluatesToNull() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT (SELECT id FROM departments WHERE name = 'nope') FROM departments WHERE id = 1") + assertEquals(listOf(listOf(Value.NullValue)), result.rows) + } + + @Test + fun scalarSubqueryReturningMoreThanOneRowFails() { + val db = newDatabase() + db.seedDepartments() + val failure = assertFailsWith { + db.select("SELECT (SELECT salary FROM employees) FROM departments WHERE id = 1") + } + assertTrue(failure.message!!.contains("more than one row")) + } + + @Test + fun scalarSubqueryReturningMoreThanOneColumnFails() { + val db = newDatabase() + db.seedDepartments() + val failure = assertFailsWith { + db.select("SELECT (SELECT id, name FROM departments) FROM departments WHERE id = 1") + } + assertTrue(failure.message!!.contains("exactly one column")) + } + + @Test + fun correlatedScalarSubqueryInProjectionCountsPerOuterRow() { + val db = newDatabase() + db.seedDepartments() + val result = db.select( + "SELECT d.name, (SELECT COUNT(*) FROM employees e WHERE e.department_id = d.id) " + + "FROM departments d ORDER BY d.id" + ) + assertEquals( + listOf(listOf(text("eng"), int(2)), listOf(text("sales"), int(2)), listOf(text("ops"), int(0))), + result.rows + ) + } + + @Test + fun correlatedScalarSubqueryInWhereComparesAgainstTheEmployeesOwnDepartmentAverage() { + val db = newDatabase() + db.seedDepartments() + val result = db.select( + "SELECT name FROM employees e WHERE e.salary > " + + "(SELECT AVG(salary) FROM employees e2 WHERE e2.department_id = e.department_id) ORDER BY name" + ) + assertEquals(listOf(listOf(text("ada")), listOf(text("grace"))), result.rows) + } + + @Test + fun inSubqueryMatchesAnUncorrelatedResultSet() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT name FROM departments WHERE id IN (SELECT department_id FROM employees) ORDER BY name") + assertEquals(listOf(listOf(text("eng")), listOf(text("sales"))), result.rows) + } + + @Test + fun notInSubqueryExcludesMatchingRows() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT name FROM departments WHERE id NOT IN (SELECT department_id FROM employees)") + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun notInSubqueryWithANullInTheResultSetExcludesEverythingByThreeValuedLogic() { + val db = newDatabase() + db.run("CREATE TABLE d (id INTEGER)") + db.run("CREATE TABLE e (dept INTEGER)") + db.run("INSERT INTO d VALUES (1), (2)") + db.run("INSERT INTO e VALUES (1), (NULL)") + val result = db.select("SELECT id FROM d WHERE id NOT IN (SELECT dept FROM e)") + assertEquals(emptyList(), result.rows) + } + + @Test + fun inSubqueryRequiresExactlyOneColumn() { + val db = newDatabase() + db.seedDepartments() + val failure = assertFailsWith { + db.select("SELECT name FROM departments WHERE id IN (SELECT id, name FROM departments)") + } + assertTrue(failure.message!!.contains("exactly one column")) + } + + @Test + fun existsWithoutCorrelationChecksOnlyWhetherTheSubqueryHasAnyRows() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT name FROM departments WHERE EXISTS (SELECT 1 FROM employees) ORDER BY name") + assertEquals(listOf(listOf(text("eng")), listOf(text("ops")), listOf(text("sales"))), result.rows) + } + + @Test + fun correlatedExistsFindsDepartmentsWithEmployees() { + val db = newDatabase() + db.seedDepartments() + val result = db.select( + "SELECT name FROM departments d WHERE EXISTS " + + "(SELECT 1 FROM employees e WHERE e.department_id = d.id) ORDER BY name" + ) + assertEquals(listOf(listOf(text("eng")), listOf(text("sales"))), result.rows) + } + + @Test + fun correlatedNotExistsFindsDepartmentsWithoutEmployees() { + val db = newDatabase() + db.seedDepartments() + val result = db.select( + "SELECT name FROM departments d WHERE NOT EXISTS (SELECT 1 FROM employees e WHERE e.department_id = d.id)" + ) + assertEquals(listOf(listOf(text("ops"))), result.rows) + } + + @Test + fun nestedSubqueriesCorrelateAcrossTwoLevelsOfOuterRows() { + val db = newDatabase() + db.seedDepartments() + val result = db.select( + "SELECT d.name FROM departments d WHERE EXISTS (" + + "SELECT 1 FROM employees e WHERE e.department_id = d.id AND e.salary > (" + + "SELECT AVG(salary) FROM employees e2 WHERE e2.department_id = d.id" + + ")) ORDER BY d.name" + ) + assertEquals(listOf(listOf(text("eng")), listOf(text("sales"))), result.rows) + } + + @Test + fun correlatedScalarSubqueryIsUsableInUpdateSet() { + val db = newDatabase() + db.seedDepartments() + db.run("CREATE TABLE dept_stats (id INTEGER PRIMARY KEY, headcount INTEGER)") + db.run("INSERT INTO dept_stats VALUES (1, 0), (2, 0), (3, 0)") + db.run( + "UPDATE dept_stats SET headcount = " + + "(SELECT COUNT(*) FROM employees WHERE employees.department_id = dept_stats.id)" + ) + val result = db.select("SELECT headcount FROM dept_stats ORDER BY id") + assertEquals(listOf(listOf(int(2)), listOf(int(2)), listOf(int(0))), result.rows) + } + + @Test + fun inSubqueryIsUsableInDeleteWhereClause() { + val db = newDatabase() + db.seedDepartments() + db.run("INSERT INTO employees VALUES (5, 'temp', 3, 10)") + db.run("DELETE FROM employees WHERE department_id IN (SELECT id FROM departments WHERE budget = 0)") + val result = db.select("SELECT name FROM employees ORDER BY name") + assertEquals( + listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("guido")), listOf(text("linus"))), + result.rows + ) + } + + @Test + fun scalarSubqueryIsUsableAsAnInsertValue() { + val db = newDatabase() + db.seedDepartments() + db.run("CREATE TABLE summary (id INTEGER PRIMARY KEY, employee_count INTEGER)") + db.run("INSERT INTO summary VALUES (1, (SELECT COUNT(*) FROM employees))") + val result = db.select("SELECT employee_count FROM summary") + assertEquals(listOf(listOf(int(4))), result.rows) + } + + @Test + fun aggregateOverAnUncorrelatedScalarSubqueryEvaluatesItPerRow() { + val db = newDatabase() + db.seedDepartments() + val result = db.select("SELECT SUM((SELECT budget FROM departments WHERE id = 1)) FROM employees") + assertEquals(listOf(listOf(int(1000 * 4))), result.rows) + } + + @Test + fun existsResultCannotBeUsedAsAnArithmeticOperand() { + val db = newDatabase() + db.seedDepartments() + assertFailsWith { + db.select("SELECT 1 + EXISTS (SELECT 1 FROM employees) FROM departments WHERE id = 1") + } + } + + @Test + fun aSubqueryReferencingAnUnknownColumnFailsWithAClearError() { + val db = newDatabase() + db.seedDepartments() + val failure = assertFailsWith { + db.select("SELECT (SELECT missing_column FROM employees) FROM departments") + } + assertTrue(failure.message!!.contains("missing_column")) + } + + @Test + fun subqueryAgainstADerivedTableInsideItselfWorks() { + val db = newDatabase() + db.seedDepartments() + val matches = db.select( + "SELECT name FROM departments WHERE EXISTS " + + "(SELECT 1 FROM (SELECT id FROM employees WHERE salary > 100) sub) ORDER BY name" + ) + assertEquals(listOf(text("eng"), text("ops"), text("sales")), matches.rows.map { it.single() }) + + val none = db.select( + "SELECT name FROM departments WHERE EXISTS (SELECT 1 FROM (SELECT id FROM employees WHERE salary > 1000) sub)" + ) + assertEquals(emptyList(), none.rows) + } + + private fun Database.seedCompound() { + run("CREATE TABLE dual (x INTEGER)") + run("INSERT INTO dual VALUES (1)") + run("CREATE TABLE t1 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE t2 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("INSERT INTO t1 VALUES (1, 'ada'), (2, 'linus')") + run("INSERT INTO t2 VALUES (2, 'linus'), (3, 'grace')") + } + + @Test + fun unionRemovesDuplicateRows() { + val db = newDatabase() + db.seedCompound() + val result = db.select("SELECT name FROM t1 UNION SELECT name FROM t2 ORDER BY name") + assertEquals(listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus"))), result.rows) + } + + @Test + fun unionAllKeepsDuplicateRows() { + val db = newDatabase() + db.seedCompound() + val result = db.select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name") + assertEquals( + listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus")), listOf(text("linus"))), + result.rows + ) + } + + @Test + fun intersectReturnsRowsPresentInBothQueries() { + val db = newDatabase() + db.seedCompound() + assertEquals(listOf(listOf(text("linus"))), db.select("SELECT name FROM t1 INTERSECT SELECT name FROM t2").rows) + } + + @Test + fun exceptReturnsRowsOnlyInTheLeftQuery() { + val db = newDatabase() + db.seedCompound() + assertEquals(listOf(listOf(text("ada"))), db.select("SELECT name FROM t1 EXCEPT SELECT name FROM t2").rows) + } + + @Test + fun intersectAllAndExceptAllPreserveMultisetCounts() { + val db = newDatabase() + db.run("CREATE TABLE bag_a (v INTEGER)") + db.run("CREATE TABLE bag_b (v INTEGER)") + db.run("INSERT INTO bag_a VALUES (1), (1), (1), (2)") + db.run("INSERT INTO bag_b VALUES (1), (1), (2), (2), (3)") + assertEquals( + listOf(listOf(int(1)), listOf(int(1)), listOf(int(2))), + db.select("SELECT v FROM bag_a INTERSECT ALL SELECT v FROM bag_b ORDER BY v").rows + ) + assertEquals(listOf(listOf(int(1))), db.select("SELECT v FROM bag_a EXCEPT ALL SELECT v FROM bag_b ORDER BY v").rows) + } + + @Test + fun compoundQueryWithMismatchedColumnCountsFails() { + val db = newDatabase() + db.seedCompound() + assertFailsWith { db.select("SELECT id, name FROM t1 UNION SELECT id FROM t2") } + } + + @Test + fun compoundOperatorsChainLeftAssociatively() { + val db = newDatabase() + db.seedCompound() + db.run("CREATE TABLE t3 (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.run("INSERT INTO t3 VALUES (2, 'linus')") + val result = db.select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 INTERSECT SELECT name FROM t3 ORDER BY name") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun orderByAndLimitApplyToTheWholeCompoundResultNotEachSide() { + val db = newDatabase() + db.seedCompound() + val result = db.select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name DESC LIMIT 2") + assertEquals(listOf(listOf(text("linus")), listOf(text("linus"))), result.rows) + } + + @Test + fun offsetSkipsRowsOfTheWholeCompoundResultBeforeLimitApplies() { + val db = newDatabase() + db.seedCompound() + val result = db.select("SELECT name FROM t1 UNION ALL SELECT name FROM t2 ORDER BY name DESC LIMIT 2 OFFSET 1") + assertEquals(listOf(listOf(text("linus")), listOf(text("grace"))), result.rows) + } + + @Test + fun withClauseMakesACteAvailableInFrom() { + val db = newDatabase() + db.seedCompound() + val result = db.select("WITH recent AS (SELECT id, name FROM t1) SELECT name FROM recent WHERE id = 1") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun aCteShadowsARealTableOfTheSameName() { + val db = newDatabase() + db.seedCompound() + val result = db.select("WITH t1 AS (SELECT id FROM t1 WHERE id = 1) SELECT * FROM t1") + assertEquals(listOf("id"), result.columns) + assertEquals(listOf(listOf(int(1))), result.rows) + } + + @Test + fun laterCteCanReferenceAnEarlierCteInTheSameWithClause() { + val db = newDatabase() + db.seedCompound() + val result = db.select( + "WITH first_cte AS (SELECT id, name FROM t1), second_cte AS (SELECT name FROM first_cte WHERE id = 2) " + + "SELECT name FROM second_cte" + ) + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun cteCanBeJoinedWithARealDiskTable() { + val db = newDatabase() + db.seedCompound() + val result = db.select("WITH t1_names AS (SELECT id, name FROM t1) SELECT t2.name FROM t1_names JOIN t2 ON t1_names.id = t2.id") + assertEquals(listOf(listOf(text("linus"))), result.rows) + } + + @Test + fun cteWithExplicitColumnListRenamesOutputColumns() { + val db = newDatabase() + db.seedCompound() + val result = db.select("WITH renamed(label) AS (SELECT name FROM t1) SELECT label FROM renamed ORDER BY label") + assertEquals(listOf(listOf(text("ada")), listOf(text("linus"))), result.rows) + } + + @Test + fun cteColumnListSizeMustMatchQueryColumnCount() { + val db = newDatabase() + db.seedCompound() + val failure = assertFailsWith { db.select("WITH bad(a, b, c) AS (SELECT id FROM t1) SELECT * FROM bad") } + assertTrue(failure.message!!.contains("bad")) + } + + @Test + fun recursiveCteGeneratesABoundedSequence() { + val db = newDatabase() + db.seedCompound() + val result = db.select( + "WITH RECURSIVE counter(n) AS (" + + "SELECT 1 FROM dual " + + "UNION ALL " + + "SELECT n + 1 FROM counter WHERE n < 5" + + ") SELECT n FROM counter ORDER BY n" + ) + assertEquals((1..5).map { listOf(int(it.toLong())) }, result.rows) + } + + @Test + fun recursiveCteWithUnionDeduplicatesAndTerminatesOnACycleThroughARealDiskTable() { + val db = newDatabase() + db.seedCompound() + db.run("CREATE TABLE edges (from_node INTEGER, to_node INTEGER)") + db.run("INSERT INTO edges VALUES (1, 2), (2, 3), (3, 1), (3, 4)") + val result = db.select( + "WITH RECURSIVE reachable(node) AS (" + + "SELECT 1 FROM dual " + + "UNION " + + "SELECT edges.to_node FROM edges JOIN reachable ON edges.from_node = reachable.node" + + ") SELECT node FROM reachable ORDER BY node" + ) + assertEquals(listOf(1L, 2L, 3L, 4L).map { listOf(int(it)) }, result.rows) + } + + @Test + fun recursiveCteWithoutSelfReferenceBehavesAsAPlainCompoundQuery() { + val db = newDatabase() + db.seedCompound() + val result = db.select( + "WITH RECURSIVE notActuallyRecursive AS (SELECT 1 AS n FROM dual UNION ALL SELECT 2 FROM dual) " + + "SELECT n FROM notActuallyRecursive ORDER BY n" + ) + assertEquals(listOf(listOf(int(1)), listOf(int(2))), result.rows) + } + + @Test + fun recursiveCteReferencingItselfOnBothSidesFails() { + val db = newDatabase() + db.seedCompound() + val failure = assertFailsWith { + db.select("WITH RECURSIVE bad(n) AS (SELECT n FROM bad UNION ALL SELECT n FROM bad) SELECT n FROM bad") + } + assertTrue(failure.message!!.contains("bad")) + } + + @Test + fun nonTerminatingRecursiveCteFailsInsteadOfLoopingForever() { + val db = newDatabase() + db.seedCompound() + val failure = assertFailsWith { + db.select( + "WITH RECURSIVE counter(n) AS (SELECT 1 FROM dual UNION ALL SELECT n + 1 FROM counter) SELECT n FROM counter" + ) + } + assertTrue(failure.message!!.contains("did not terminate")) + } + + @Test + fun aWithClauseThatNeverReferencesItsCteLeavesOrdinaryPrimaryKeyPushdownUntouched() { + val db = newDatabase() + db.seedUsers() + val table = db.catalog.table("users") + val pointLookupsBefore = table.pointLookupCount + val scansBefore = table.scanCount + val result = db.select("WITH unused AS (SELECT id FROM users) SELECT name FROM users WHERE id = 1") + assertEquals(listOf(listOf(text("ada"))), result.rows) + // Every CTE is materialized up front regardless of whether the body ends up using it (mirroring + // kestrel.engine.SelectRunner), so "unused" itself contributes exactly one full scan - the point is + // that the OUTER query's own WHERE id = 1 still gets its point lookup rather than also falling + // back to a scan just because a WITH clause is present. + assertEquals(scansBefore + 1, table.scanCount) + assertEquals(pointLookupsBefore + 1, table.pointLookupCount) + } + + /** + * eng (deptId 1): ada/linus tie at 400 (ada inserted first), grace at 300. sales (deptId 2): + * sam/kay tie at 200 (sam inserted first), meg at 100. Mirrors kestrel.WindowFunctionTest's seed() + * exactly so results can be cross-checked against the in-memory engine's own known-good values. + */ + private fun Database.seedWindowEmployees() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run("CREATE TABLE employees (id INTEGER PRIMARY KEY, deptId INTEGER, name TEXT, salary INTEGER)") + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'sales')") + run( + "INSERT INTO employees VALUES " + + "(1, 1, 'ada', 400), (2, 1, 'linus', 400), (3, 1, 'grace', 300), " + + "(4, 2, 'sam', 200), (5, 2, 'kay', 200), (6, 2, 'meg', 100)" + ) + } + + @Test + fun rowNumberAssignsSequentialNumbersWithinEachPartition() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, ROW_NUMBER() OVER (PARTITION BY deptId ORDER BY salary DESC) AS rn " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(2)), listOf(int(3), int(3)), + listOf(int(4), int(1)), listOf(int(5), int(2)), listOf(int(6), int(3)) + ), + result.rows + ) + } + + @Test + fun rowNumberWithoutOverOrderByFallsBackToPartitionArrivalOrder() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, ROW_NUMBER() OVER (PARTITION BY deptId) AS rn FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(2)), listOf(int(3), int(3)), + listOf(int(4), int(1)), listOf(int(5), int(2)), listOf(int(6), int(3)) + ), + result.rows + ) + } + + @Test + fun rankLeavesGapsAfterTiesButDenseRankDoesNot() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, RANK() OVER (PARTITION BY deptId ORDER BY salary DESC) AS r, " + + "DENSE_RANK() OVER (PARTITION BY deptId ORDER BY salary DESC) AS dr " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(1), int(1)), + listOf(int(2), int(1), int(1)), + listOf(int(3), int(3), int(2)), + listOf(int(4), int(1), int(1)), + listOf(int(5), int(1), int(1)), + listOf(int(6), int(3), int(2)) + ), + result.rows + ) + } + + @Test + fun rankWithoutPartitionOrdersTheWholeResultSet() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, RANK() OVER (ORDER BY salary DESC) AS r FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(1)), listOf(int(2), int(1)), listOf(int(3), int(3)), + listOf(int(4), int(4)), listOf(int(5), int(4)), listOf(int(6), int(6)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithoutOrderByRepeatsTheWholePartitionTotalOnEveryRow() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, SUM(salary) OVER (PARTITION BY deptId) AS deptTotal FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(1100)), listOf(int(2), int(1100)), listOf(int(3), int(1100)), + listOf(int(4), int(500)), listOf(int(5), int(500)), listOf(int(6), int(500)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithOrderByRunsCumulativelyAndTiesSharePeerGroupTotal() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS running FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(800)), listOf(int(2), int(800)), listOf(int(3), int(1100)), + listOf(int(4), int(400)), listOf(int(5), int(400)), listOf(int(6), int(500)) + ), + result.rows + ) + } + + @Test + fun aggregateWindowWithUniqueOrderKeyProducesARunningTotal() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, SUM(salary) OVER (ORDER BY id) AS running FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), int(400)), listOf(int(2), int(800)), listOf(int(3), int(1100)), + listOf(int(4), int(1300)), listOf(int(5), int(1500)), listOf(int(6), int(1600)) + ), + result.rows + ) + } + + @Test + fun countStarOverPartitionCountsRowsRegardlessOfNulls() { + val db = newDatabase() + db.seedWindowEmployees() + db.run("INSERT INTO employees VALUES (7, 3, 'unpaid', NULL)") + val result = db.select( + "SELECT COUNT(*) OVER (PARTITION BY deptId) AS n, COUNT(salary) OVER (PARTITION BY deptId) AS paid " + + "FROM employees WHERE deptId = 3" + ) + assertEquals(listOf(listOf(int(1), int(0))), result.rows) + } + + @Test + fun windowFunctionOverAJoinPartitionsByTheJoinedColumn() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT e.name, d.name AS dept, " + + "ROW_NUMBER() OVER (PARTITION BY d.name ORDER BY e.salary DESC) AS rn " + + "FROM employees e JOIN departments d ON e.deptId = d.id " + + "WHERE d.name = 'eng' ORDER BY rn ASC" + ) + assertEquals( + listOf( + listOf(text("ada"), text("eng"), int(1)), + listOf(text("linus"), text("eng"), int(2)), + listOf(text("grace"), text("eng"), int(3)) + ), + result.rows + ) + } + + @Test + fun orderByCanReferenceAWindowFunctionByItsProjectionAlias() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, RANK() OVER (ORDER BY salary DESC) AS r FROM employees ORDER BY r ASC, id ASC") + assertEquals((1..6).map { it.toLong() }, result.rows.map { (it[0] as Value.IntegerValue).value }) + } + + @Test + fun selectStarWithAWindowFunctionOnlyInOrderByDoesNotLeakTheExtraColumn() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT * FROM departments ORDER BY ROW_NUMBER() OVER (ORDER BY id) DESC") + assertEquals(listOf("id", "name"), result.columns) + assertEquals(listOf(listOf(int(2), text("sales")), listOf(int(1), text("eng"))), result.rows) + } + + @Test + fun windowFunctionIsRejectedInWhere() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT id FROM employees WHERE ROW_NUMBER() OVER (ORDER BY id) = 1") + } + assertTrue(error.message!!.contains("WHERE"), error.message!!) + } + + @Test + fun windowFunctionIsRejectedInGroupBy() { + val db = newDatabase() + db.seedWindowEmployees() + assertFailsWith { + db.run("SELECT COUNT(*) FROM employees GROUP BY ROW_NUMBER() OVER (ORDER BY id)") + } + } + + @Test + fun windowFunctionIsRejectedInHaving() { + val db = newDatabase() + db.seedWindowEmployees() + assertFailsWith { + db.run("SELECT deptId, COUNT(*) FROM employees GROUP BY deptId HAVING ROW_NUMBER() OVER (ORDER BY deptId) > 1") + } + } + + @Test + fun windowFunctionIsRejectedInAnOnCondition() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT e.id FROM employees e JOIN departments d ON ROW_NUMBER() OVER (ORDER BY e.id) = d.id") + } + assertTrue(error.message!!.contains("ON condition"), error.message!!) + } + + @Test + fun windowFunctionCannotBeCombinedWithGroupByInTheSameQuery() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT deptId, COUNT(*), ROW_NUMBER() OVER (ORDER BY deptId) FROM employees GROUP BY deptId") + } + assertTrue(error.message!!.contains("combined"), error.message!!) + } + + @Test + fun rankRequiresAnOrderByInsideOver() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT RANK() OVER (PARTITION BY deptId) FROM employees") + } + assertTrue(error.message!!.contains("ORDER BY"), error.message!!) + } + + @Test + fun denseRankRequiresAnOrderByInsideOver() { + val db = newDatabase() + db.seedWindowEmployees() + assertFailsWith { + db.run("SELECT DENSE_RANK() OVER (PARTITION BY deptId) FROM employees") + } + } + + @Test + fun nestedWindowFunctionsAreRejected() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT SUM(salary) OVER (ORDER BY ROW_NUMBER() OVER (ORDER BY id)) FROM employees") + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun lagReturnsThePreviousRowsValueWithinEachPartitionAndNullAtTheEdge() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, LAG(salary) OVER (PARTITION BY deptId ORDER BY id) AS prev FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), Value.NullValue), listOf(int(2), int(400)), listOf(int(3), int(400)), + listOf(int(4), Value.NullValue), listOf(int(5), int(200)), listOf(int(6), int(200)) + ), + result.rows + ) + } + + @Test + fun leadReturnsTheNextRowsValueWithinEachPartitionAndNullAtTheEdge() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, LEAD(salary) OVER (PARTITION BY deptId ORDER BY id) AS next FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(400)), listOf(int(2), int(300)), listOf(int(3), Value.NullValue), + listOf(int(4), int(200)), listOf(int(5), int(100)), listOf(int(6), Value.NullValue) + ), + result.rows + ) + } + + @Test + fun lagWithAnExplicitOffsetSkipsMultipleRows() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select("SELECT id, LAG(salary, 2) OVER (ORDER BY id) AS prev2 FROM employees ORDER BY id ASC") + assertEquals( + listOf( + listOf(int(1), Value.NullValue), listOf(int(2), Value.NullValue), listOf(int(3), int(400)), + listOf(int(4), int(400)), listOf(int(5), int(300)), listOf(int(6), int(200)) + ), + result.rows + ) + } + + @Test + fun lagPastThePartitionEdgeFallsBackToTheDefaultExpressionInsteadOfNull() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, LAG(salary, 1, 0) OVER (PARTITION BY deptId ORDER BY id) AS prev FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(int(0), int(400), int(400), int(0), int(200), int(200)), result.rows.map { it[1] }) + } + + @Test + fun lagOffsetMustNotBeNegative() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT LAG(salary, -1) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun lagOffsetMustBeAnInteger() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT LAG(salary, 1.5) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun firstValueReturnsTheFirstRowsValueForEveryRowInThePartition() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, FIRST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY id) AS f FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(400L, 400L, 400L, 200L, 200L, 200L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun lastValueSharesOneValuePerPeerGroupRatherThanTheWholePartition() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, LAST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY salary DESC) AS l FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(400L, 400L, 300L, 200L, 200L, 100L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun lastValueWithoutOverOrderByUsesTheWholePartitionsLastRow() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, LAST_VALUE(salary) OVER (PARTITION BY deptId) AS l FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(300L, 300L, 300L, 100L, 100L, 100L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileSplitsEachPartitionIntoAsEvenBucketsAsPossible() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, NTILE(2) OVER (PARTITION BY deptId ORDER BY salary DESC) AS bucket FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(1L, 1L, 2L, 1L, 1L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileWithMoreBucketsThanRowsLeavesTheHighestBucketsUnused() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, NTILE(5) OVER (PARTITION BY deptId ORDER BY salary DESC) AS bucket FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(1L, 2L, 3L, 1L, 2L, 3L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun ntileBucketCountMustBePositive() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT NTILE(0) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("positive"), error.message!!) + } + + @Test + fun ntileBucketCountMustBeAnInteger() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT NTILE(1.5) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun windowFunctionNestedInsideLagsOffsetIsRejected() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT LAG(salary, ROW_NUMBER() OVER (ORDER BY id)) OVER (ORDER BY id) FROM employees") + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + @Test + fun explicitRowsFrameProducesAMovingAverageInsteadOfARunningTotal() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(400L, 800L, 700L, 200L, 400L, 300L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rowsBetweenCurrentRowAndFollowingLooksForward() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(listOf(800L, 700L, 300L, 400L, 300L, 100L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rowsBetweenUnboundedPrecedingAndCurrentRowMatchesTheImplicitDefaultOnAUniqueOrderKey() { + val db = newDatabase() + db.seedWindowEmployees() + val explicit = db.select( + "SELECT id, SUM(salary) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + val implicit = db.select("SELECT id, SUM(salary) OVER (ORDER BY id) AS s FROM employees ORDER BY id ASC") + assertEquals(implicit.rows, explicit.rows) + } + + @Test + fun frameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 5 FOLLOWING AND 9 FOLLOWING) AS s, " + + "COUNT(*) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 5 FOLLOWING AND 9 FOLLOWING) AS c " + + "FROM employees WHERE deptId = 1 ORDER BY id ASC" + ) + assertEquals( + listOf(listOf(int(1), Value.NullValue, int(0)), listOf(int(2), Value.NullValue, int(0)), listOf(int(3), Value.NullValue, int(0))), + result.rows + ) + } + + @Test + fun firstValueAndLastValueRespectAnExplicitFrameInsteadOfThePeerGroupDefault() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, " + + "FIRST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS f, " + + "LAST_VALUE(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS l " + + "FROM employees ORDER BY id ASC" + ) + assertEquals( + listOf( + listOf(int(1), int(400), int(400)), + listOf(int(2), int(400), int(300)), + listOf(int(3), int(400), int(300)), + listOf(int(4), int(200), int(200)), + listOf(int(5), int(200), int(100)), + listOf(int(6), int(200), int(100)) + ), + result.rows + ) + } + + @Test + fun rowsFrameShorthandWithoutBetweenEndsAtCurrentRow() { + val db = newDatabase() + db.seedWindowEmployees() + val explicit = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS 1 PRECEDING) AS s FROM employees ORDER BY id ASC" + ) + val spelledOut = db.select( + "SELECT id, SUM(salary) OVER (PARTITION BY deptId ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS s " + + "FROM employees ORDER BY id ASC" + ) + assertEquals(spelledOut.rows, explicit.rows) + } + + @Test + fun rowsFrameOffsetMustNotBeNegative() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN -1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun rowsFrameOffsetMustBeAnInteger() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN 1.5 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("integer"), error.message!!) + } + + @Test + fun rowsFrameIsRejectedOnRankingAndOffsetWindowFunctions() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT ROW_NUMBER() OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("frame"), error.message!!) + } + + @Test + fun windowFunctionNestedInsideAFrameOffsetIsRejected() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run( + "SELECT SUM(salary) OVER (ORDER BY id ROWS BETWEEN ROW_NUMBER() OVER (ORDER BY id) PRECEDING AND CURRENT ROW) " + + "FROM employees" + ) + } + assertTrue(error.message!!.contains("nested"), error.message!!) + } + + /** amount is monotonic in id (10, 20, 20, 30, 50) so `ORDER BY id` and `ORDER BY amount` agree, with a tie at 20. */ + private fun Database.seedWindowSales() { + run("CREATE TABLE sales (id INTEGER PRIMARY KEY, amount INTEGER)") + run("INSERT INTO sales VALUES (1, 10), (2, 20), (3, 20), (4, 30), (5, 50)") + } + + @Test + fun rangeBetweenPrecedingAndFollowingIncludesRowsWithinValueDistanceRegardlessOfPhysicalRowCount() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(50L, 80L, 80L, 70L, 50L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeBetweenPrecedingAndFollowingOnADescendingOrderByFlipsWhichDirectionIsPreceding() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount DESC RANGE BETWEEN 15 PRECEDING AND 5 FOLLOWING) AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(50L, 70L, 70L, 30L, 50L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeCurrentRowExpandsToTheWholePeerGroupUnlikeRowsCurrentRow() { + val db = newDatabase() + db.seedWindowSales() + val range = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM sales ORDER BY id ASC" + ) + val rows = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(10L, 50L, 50L, 80L, 130L), range.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(listOf(10L, 30L, 50L, 80L, 130L), rows.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameMeasuresDistanceOnADecimalColumn() { + val db = newDatabase() + db.run("CREATE TABLE prices (id INTEGER PRIMARY KEY, price DECIMAL(10, 2))") + db.run("INSERT INTO prices VALUES (1, 1.00), (2, 1.40), (3, 1.50), (4, 2.00)") + val result = db.select( + "SELECT id, COUNT(*) OVER (ORDER BY price RANGE BETWEEN 0.5 PRECEDING AND 0.5 FOLLOWING) AS c FROM prices ORDER BY id ASC" + ) + assertEquals(listOf(3L, 3L, 4L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameMeasuresDistanceInDaysOnADateColumn() { + val db = newDatabase() + db.run("CREATE TABLE events (id INTEGER PRIMARY KEY, day DATE)") + db.run( + "INSERT INTO events VALUES " + + "(1, CAST('2024-01-01' AS DATE)), (2, CAST('2024-01-02' AS DATE)), " + + "(3, CAST('2024-01-05' AS DATE)), (4, CAST('2024-01-06' AS DATE)), (5, CAST('2024-01-10' AS DATE))" + ) + val result = db.select( + "SELECT id, COUNT(*) OVER (ORDER BY day RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS c FROM events ORDER BY id ASC" + ) + assertEquals(listOf(2L, 2L, 2L, 2L, 1L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun rangeFrameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS s, " + + "COUNT(*) OVER (ORDER BY amount RANGE BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS c " + + "FROM sales ORDER BY id ASC" + ) + assertEquals((1L..5L).map { listOf(int(it), Value.NullValue, int(0)) }, result.rows) + } + + @Test + fun rangeFrameWithAPrecedingBoundRequiresExactlyOneOrderByColumn() { + val db = newDatabase() + db.seedWindowSales() + val error = assertFailsWith { + db.run("SELECT SUM(amount) OVER (ORDER BY amount, id RANGE BETWEEN 10 PRECEDING AND 10 FOLLOWING) FROM sales") + } + assertTrue(error.message!!.contains("exactly one ORDER BY column"), error.message!!) + } + + @Test + fun rangeFrameWithAPrecedingBoundRequiresANumericOrDateOrderColumn() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT SUM(salary) OVER (ORDER BY name RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("numeric or DATE"), error.message!!) + } + + @Test + fun rangeFrameOffsetMustBeNumeric() { + val db = newDatabase() + db.seedWindowSales() + val error = assertFailsWith { + db.run("SELECT SUM(amount) OVER (ORDER BY amount RANGE BETWEEN 'x' PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("numeric"), error.message!!) + } + + @Test + fun rangeFrameOffsetMustNotBeNegative() { + val db = newDatabase() + db.seedWindowSales() + val error = assertFailsWith { + db.run("SELECT SUM(amount) OVER (ORDER BY amount RANGE BETWEEN -1 PRECEDING AND CURRENT ROW) FROM sales") + } + assertTrue(error.message!!.contains("negative"), error.message!!) + } + + @Test + fun groupsBetweenPrecedingAndFollowingCountsPeerGroupsNotPhysicalRows() { + val db = newDatabase() + db.seedWindowSales() + val groups = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s FROM sales ORDER BY id ASC" + ) + val rows = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(50L, 80L, 80L, 120L, 80L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(listOf(30L, 50L, 70L, 100L, 80L), rows.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsCurrentRowExpandsToTheWholePeerGroupJustLikeRangeCurrentRow() { + val db = newDatabase() + db.seedWindowSales() + val groups = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM sales ORDER BY id ASC" + ) + val range = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(10L, 50L, 50L, 80L, 130L), groups.rows.map { (it[1] as Value.IntegerValue).value }) + assertEquals(groups.rows, range.rows) + } + + @Test + fun groupsFrameWorksWithAMultiColumnNonNumericOrderByUnlikeRange() { + val db = newDatabase() + db.seedWindowEmployees() + val result = db.select( + "SELECT id, COUNT(*) OVER (ORDER BY deptId, name GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS c " + + "FROM employees ORDER BY deptId ASC, name ASC" + ) + assertEquals(listOf(int(1), int(3), int(2), int(5), int(6), int(4)), result.rows.map { it[0] }) + assertEquals(listOf(2L, 3L, 3L, 3L, 3L, 2L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun groupsFrameEntirelyPastThePartitionEdgeAggregatesToNullAndCountsZero() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (ORDER BY amount GROUPS BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS s, " + + "COUNT(*) OVER (ORDER BY amount GROUPS BETWEEN 100 FOLLOWING AND 200 FOLLOWING) AS c " + + "FROM sales ORDER BY id ASC" + ) + assertEquals((1L..5L).map { listOf(int(it), Value.NullValue, int(0)) }, result.rows) + } + + @Test + fun groupsFrameIsRejectedOnRankingAndOffsetWindowFunctions() { + val db = newDatabase() + db.seedWindowEmployees() + val error = assertFailsWith { + db.run("SELECT ROW_NUMBER() OVER (ORDER BY id GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM employees") + } + assertTrue(error.message!!.contains("GROUPS"), error.message!!) + } + + @Test + fun excludeCurrentRowRemovesOnlyTheCurrentRowFromTheWholePartitionFrame() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(120L, 110L, 110L, 100L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeGroupRemovesTheWholePeerGroupIncludingTheCurrentRow() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(120L, 90L, 90L, 100L, 80L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeTiesRemovesOnlyTheOtherTiedRowsAndKeepsTheCurrentRow() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE TIES" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(130L, 110L, 110L, 130L, 130L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + @Test + fun excludeCurrentRowInteractsWithABoundedFrameInsteadOfOnlyTheWholePartition() { + val db = newDatabase() + db.seedWindowSales() + val result = db.select( + "SELECT id, SUM(amount) OVER (" + + "ORDER BY amount ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW" + + ") AS s FROM sales ORDER BY id ASC" + ) + assertEquals(listOf(20L, 30L, 50L, 70L, 30L), result.rows.map { (it[1] as Value.IntegerValue).value }) + } + + private fun Database.seedUsersForViews() { + run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER, dept TEXT)") + run( + "INSERT INTO users VALUES " + + "(1, 'ada', 36, 'eng'), (2, 'linus', 54, 'eng'), (3, 'grace', 85, 'navy'), (4, 'alan', 41, NULL)" + ) + } + + @Test + fun createViewReturnsResultAndRegistersName() { + val db = newDatabase() + db.seedUsersForViews() + val result = db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40").single() + assertEquals(CreateViewResult("adults"), result) + assertEquals(listOf("adults"), db.catalog.viewNames) + } + + @Test + fun queryingAViewRunsItsDefiningQueryFresh() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40") + val first = db.select("SELECT name FROM adults ORDER BY id ASC") + assertEquals(listOf(text("linus"), text("grace"), text("alan")), first.rows.map { it.single() }) + + db.run("INSERT INTO users VALUES (5, 'margaret', 61, 'navy')") + val second = db.select("SELECT name FROM adults ORDER BY id ASC") + assertEquals( + listOf(text("linus"), text("grace"), text("alan"), text("margaret")), + second.rows.map { it.single() } + ) + } + + @Test + fun viewCanBeUsedAsAJoinTarget() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE TABLE bonuses (dept TEXT, amount INTEGER)") + db.run("INSERT INTO bonuses VALUES ('eng', 1000), ('navy', 500)") + db.run("CREATE VIEW eng_staff AS SELECT id, name, dept FROM users WHERE dept = 'eng'") + val result = db.select( + "SELECT eng_staff.name, bonuses.amount FROM eng_staff " + + "JOIN bonuses ON eng_staff.dept = bonuses.dept ORDER BY eng_staff.id ASC" + ) + assertEquals( + listOf(listOf(text("ada"), int(1000)), listOf(text("linus"), int(1000))), + result.rows + ) + } + + @Test + fun viewOverAggregateQueryWorks() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW dept_counts AS SELECT dept, COUNT(*) AS n FROM users GROUP BY dept") + val result = db.select("SELECT dept, n FROM dept_counts WHERE n > 1 ORDER BY dept ASC") + assertEquals(listOf(listOf(text("eng"), int(2))), result.rows) + } + + @Test + fun viewDefinitionCanReferenceAnotherView() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name, age FROM users WHERE age >= 40") + db.run("CREATE VIEW seniors AS SELECT id, name FROM adults WHERE age >= 60") + val result = db.select("SELECT name FROM seniors") + assertEquals(listOf(listOf(text("grace"))), result.rows) + } + + @Test + fun viewCanBeReferencedInsideAWithCteBody() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name, age FROM users WHERE age >= 40") + val result = db.select("WITH a AS (SELECT name FROM adults WHERE age >= 60) SELECT name FROM a") + assertEquals(listOf(listOf(text("grace"))), result.rows) + } + + @Test + fun viewCanAppearOnEitherSideOfASetOperation() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW eng AS SELECT name FROM users WHERE dept = 'eng'") + db.run("CREATE VIEW navy AS SELECT name FROM users WHERE dept = 'navy'") + val result = db.select("SELECT name FROM eng UNION SELECT name FROM navy ORDER BY name") + assertEquals(listOf(listOf(text("ada")), listOf(text("grace")), listOf(text("linus"))), result.rows) + } + + @Test + fun creatingAViewValidatesColumnsEagerly() { + val db = newDatabase() + db.seedUsersForViews() + assertFailsWith { db.run("CREATE VIEW broken AS SELECT nope FROM users") } + assertTrue(db.catalog.viewNames.isEmpty()) + } + + @Test + fun creatingAViewOverAnUnknownTableFails() { + val db = newDatabase() + assertFailsWith { db.run("CREATE VIEW v AS SELECT * FROM missing") } + } + + @Test + fun duplicateViewNameFailsWithoutOrReplace() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 21") } + } + + @Test + fun createOrReplaceViewSwapsTheDefinition() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + db.run("CREATE OR REPLACE VIEW adults AS SELECT id FROM users WHERE age >= 50") + val result = db.select("SELECT id FROM adults ORDER BY id ASC") + assertEquals(listOf(listOf(int(2)), listOf(int(3))), result.rows) + } + + @Test + fun dropViewRemovesItAndItStopsBeingQueryable() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + val result = db.run("DROP VIEW adults").single() + assertEquals(DropViewResult("adults"), result) + assertFailsWith { db.run("SELECT * FROM adults") } + } + + @Test + fun droppingAnUnknownViewFails() { + val db = newDatabase() + assertFailsWith { db.run("DROP VIEW nope") } + } + + @Test + fun cannotCreateTableWithSameNameAsAnExistingView() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { db.run("CREATE TABLE adults (id INTEGER)") } + } + + @Test + fun cannotCreateViewWithSameNameAsAnExistingTable() { + val db = newDatabase() + db.seedUsersForViews() + assertFailsWith { db.run("CREATE VIEW users AS SELECT id FROM users") } + } + + @Test + fun dropTableRefusesAViewNameAndDropViewRefusesATableName() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { db.run("DROP TABLE adults") } + assertFailsWith { db.run("DROP VIEW users") } + } + + @Test + fun insertUpdateDeleteAgainstAViewAreRejected() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") + assertFailsWith { db.run("INSERT INTO adults VALUES (9, 'x')") } + assertFailsWith { db.run("UPDATE adults SET name = 'x'") } + assertFailsWith { db.run("DELETE FROM adults") } + } + + @Test + fun creatingAnIndexOnAViewIsRejected() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 18") + assertFailsWith { db.run("CREATE INDEX idx ON adults (id)") } + } + + @Test + fun mutuallyRecursiveViewsFailInsteadOfOverflowingTheStack() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW a AS SELECT id FROM users") + db.run("CREATE VIEW b AS SELECT id FROM a") + db.run("CREATE OR REPLACE VIEW a AS SELECT id FROM b") + assertFailsWith { db.run("SELECT * FROM a") } + } + + @Test + fun selfReferencingViewFailsCleanly() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW a AS SELECT id FROM users") + db.run("CREATE OR REPLACE VIEW a AS SELECT id FROM a") + assertFailsWith { db.run("SELECT * FROM a") } + } + + @Test + fun dropTableWithADependentViewFailsByDefaultAndAlsoWithExplicitRestrict() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + assertFailsWith { db.run("DROP TABLE users") } + assertFailsWith { db.run("DROP TABLE users RESTRICT") } + assertEquals(listOf("users"), db.catalog.tableNames) + assertEquals(listOf("adults"), db.catalog.viewNames) + } + + @Test + fun dropViewWithADependentViewFailsByDefault() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + db.run("CREATE VIEW seniorAdults AS SELECT id FROM adults WHERE id > 0") + assertFailsWith { db.run("DROP VIEW adults") } + assertEquals(listOf("adults", "seniorAdults"), db.catalog.viewNames) + } + + @Test + fun dropViewCascadeDropsATransitiveChainOfDependentViews() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW a AS SELECT id FROM users WHERE age >= 18") + db.run("CREATE VIEW b AS SELECT id FROM a WHERE id > 0") + db.run("CREATE VIEW c AS SELECT id FROM b WHERE id > 0") + db.run("DROP VIEW a CASCADE") + assertTrue(db.catalog.viewNames.isEmpty()) + } + + @Test + fun dropTableWithADiamondOfDependentViewsIsBlockedByEveryDirectDependent() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW b AS SELECT id FROM users WHERE age >= 18") + db.run("CREATE VIEW c AS SELECT id FROM users WHERE age >= 21") + assertFailsWith { db.run("DROP TABLE users") } + assertEquals(listOf("users"), db.catalog.tableNames) + assertEquals(listOf("b", "c"), db.catalog.viewNames) + } + + @Test + fun dropTableCascadeDropsTheTableAndItsDependentView() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + db.run("DROP TABLE users CASCADE") + assertTrue(db.catalog.tableNames.isEmpty()) + assertTrue(db.catalog.viewNames.isEmpty()) + } + + @Test + fun dropTableCascadeHandlesADiamondOfDependentViewsWithoutFailing() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW b AS SELECT id FROM users WHERE age >= 18") + db.run("CREATE VIEW c AS SELECT id FROM users WHERE age >= 21") + db.run("CREATE VIEW d AS SELECT b.id FROM b JOIN c ON b.id = c.id") + db.run("DROP TABLE users CASCADE") + assertTrue(db.catalog.tableNames.isEmpty()) + assertTrue(db.catalog.viewNames.isEmpty()) + } + + @Test + fun dropTableCascadeDroppedViewsSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id FROM users WHERE age >= 18") + db.run("DROP TABLE users CASCADE") + db.reopen() + assertTrue(db.catalog.tableNames.isEmpty()) + assertTrue(db.catalog.viewNames.isEmpty()) + } + + @Test + fun columnTypeOfAViewIsInferredFromActualData() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW names AS SELECT name FROM users") + val result = db.select("SELECT name FROM names WHERE name = 'ada'") + assertEquals(listOf(listOf(text("ada"))), result.rows) + } + + @Test + fun viewDefinitionSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name FROM users WHERE age >= 40") + db.reopen() + assertEquals(listOf("adults"), db.catalog.viewNames) + val result = db.select("SELECT name FROM adults ORDER BY id ASC") + assertEquals(listOf(text("linus"), text("grace"), text("alan")), result.rows.map { it.single() }) + + db.run("INSERT INTO users VALUES (5, 'margaret', 61, 'navy')") + db.reopen() + val afterInsertAndReopen = db.select("SELECT name FROM adults ORDER BY id ASC") + assertEquals( + listOf(text("linus"), text("grace"), text("alan"), text("margaret")), + afterInsertAndReopen.rows.map { it.single() } + ) + } + + @Test + fun viewOverViewAndDropViewSurviveReopen() { + val db = newDatabase() + db.seedUsersForViews() + db.run("CREATE VIEW adults AS SELECT id, name, age FROM users WHERE age >= 40") + db.run("CREATE VIEW seniors AS SELECT id, name FROM adults WHERE age >= 60") + db.reopen() + assertEquals(listOf(listOf(text("grace"))), db.select("SELECT name FROM seniors").rows) + + db.run("DROP VIEW seniors") + db.reopen() + assertEquals(listOf("adults"), db.catalog.viewNames) + assertFailsWith { db.run("SELECT * FROM seniors") } + } + + @Test + fun columnLevelCheckAcceptsSatisfyingRowsAndRejectsViolatingOnes() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + assertEquals(InsertResult(1), db.run("INSERT INTO t VALUES (1, 30)").single()) + val error = assertFailsWith { db.run("INSERT INTO t VALUES (2, -5)") } + assertTrue(error.message!!.contains("CHECK")) + } + + @Test + fun checkPassesWhenExpressionIsUnknownBecauseOfNull() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + // three-valued logic: NULL > 0 is UNKNOWN, not FALSE, so the row is accepted + assertEquals(InsertResult(1), db.run("INSERT INTO t (id) VALUES (1)").single()) + } + + @Test + fun tableLevelCheckCanSpanMultipleColumns() { + val db = newDatabase() + db.run("CREATE TABLE ranges (id INTEGER PRIMARY KEY, low INTEGER, high INTEGER, CHECK (low <= high))") + assertEquals(InsertResult(1), db.run("INSERT INTO ranges VALUES (1, 1, 10)").single()) + assertFailsWith { db.run("INSERT INTO ranges VALUES (2, 10, 1)") } + } + + @Test + fun multipleCheckConstraintsAllMustBeSatisfied() { + val db = newDatabase() + db.run( + "CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age >= 0), " + + "score INTEGER CHECK (score BETWEEN 0 AND 100))" + ) + assertEquals(InsertResult(1), db.run("INSERT INTO t VALUES (1, 30, 90)").single()) + assertFailsWith { db.run("INSERT INTO t VALUES (2, 30, 150)") } + assertFailsWith { db.run("INSERT INTO t VALUES (3, -1, 90)") } + } + + @Test + fun checkIsEnforcedOnUpdateToo() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + db.run("INSERT INTO t VALUES (1, 30)") + assertFailsWith { db.run("UPDATE t SET age = -1 WHERE id = 1") } + assertEquals(UpdateResult(1), db.run("UPDATE t SET age = 40 WHERE id = 1").single()) + } + + @Test + fun checkRejectsAggregateFunctions() { + val db = newDatabase() + assertFailsWith { + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, amount INTEGER CHECK (SUM(amount) > 0))") + } + } + + @Test + fun checkRejectsSubqueries() { + val db = newDatabase() + db.run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + assertFailsWith { + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY CHECK (id IN (SELECT id FROM other)))") + } + } + + @Test + fun checkRejectsUnknownColumn() { + val db = newDatabase() + assertFailsWith { + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, CHECK (nope > 0))") + } + } + + @Test + fun checkCanReferenceStringAndCallExpressions() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT CHECK (LENGTH(name) > 0))") + assertEquals(InsertResult(1), db.run("INSERT INTO t VALUES (1, 'ada')").single()) + assertFailsWith { db.run("INSERT INTO t VALUES (2, '')") } + } + + @Test + fun checkDefinitionSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0), CHECK (id > 0))") + db.run("INSERT INTO t VALUES (1, 30)") + db.reopen() + + assertEquals(InsertResult(1), db.run("INSERT INTO t VALUES (2, 40)").single()) + assertFailsWith { db.run("INSERT INTO t VALUES (3, -1)") } + assertFailsWith { db.run("UPDATE t SET age = -5 WHERE id = 1") } + + db.reopen() + assertFailsWith { db.run("INSERT INTO t VALUES (4, -1)") } + assertEquals(2, db.select("SELECT * FROM t").rows.size) + } + + private fun Database.seedDepartmentsAndEmployees() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run( + "CREATE TABLE employees (" + + "id INTEGER PRIMARY KEY, name TEXT NOT NULL, dept_id INTEGER REFERENCES departments(id))" + ) + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'ops')") + } + + private fun Database.seedCascadeDepartments() { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE ON UPDATE CASCADE)" + ) + run("INSERT INTO departments VALUES (1, 'eng'), (2, 'ops')") + run("INSERT INTO employees VALUES (1, 'ada', 1), (2, 'linus', 1), (3, 'grace', 2)") + } + + private fun Database.seedSetNullDepartments(childNotNull: Boolean) { + run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + val childColumn = if (childNotNull) "dept_id INTEGER NOT NULL" else "dept_id INTEGER" + run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "$childColumn REFERENCES departments(id) ON DELETE SET NULL)" + ) + run("INSERT INTO departments VALUES (1, 'eng')") + run("INSERT INTO employees VALUES (1, 'ada', 1)") + } + + @Test + fun insertRejectsForeignKeyViolation() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + val error = assertFailsWith { db.run("INSERT INTO employees VALUES (1, 'ada', 99)") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun insertAllowsNullForeignKeyValue() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (1, 'ada', NULL)").single()) + } + + @Test + fun insertSucceedsWhenReferencedRowExists() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (1, 'ada', 1)").single()) + } + + @Test + fun updateRejectsForeignKeyViolationOnChildColumn() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + assertFailsWith { db.run("UPDATE employees SET dept_id = 99 WHERE id = 1") } + } + + @Test + fun foreignKeyRequiresReferencedTableToExist() { + val db = newDatabase() + assertFailsWith { + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER REFERENCES missing(id))") + } + } + + @Test + fun foreignKeyRequiresReferencedColumnToBeAPrimaryKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + assertFailsWith { + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, x TEXT REFERENCES departments(name))") + } + } + + @Test + fun tableLevelForeignKeySyntaxIsEquivalentToColumnLevel() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, name TEXT)") + db.run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER, " + + "FOREIGN KEY (dept_id) REFERENCES departments(id))" + ) + db.run("INSERT INTO departments VALUES (1, 'eng')") + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (1, 1)").single()) + assertFailsWith { db.run("INSERT INTO employees VALUES (2, 99)") } + } + + @Test + fun deleteIsRestrictedByDefaultWhenChildrenReferenceTheRow() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + val error = assertFailsWith { db.run("DELETE FROM departments WHERE id = 1") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + assertEquals(1, db.select("SELECT * FROM departments WHERE id = 1").rows.size) + } + + @Test + fun deleteWithoutReferencingChildrenSucceeds() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + assertEquals(DeleteResult(1), db.run("DELETE FROM departments WHERE id = 2").single()) + } + + @Test + fun onDeleteCascadeRemovesReferencingChildren() { + val db = newDatabase() + db.seedCascadeDepartments() + assertEquals(DeleteResult(1), db.run("DELETE FROM departments WHERE id = 1").single()) + assertEquals(listOf(listOf(text("grace"))), db.select("SELECT name FROM employees ORDER BY id").rows) + } + + @Test + fun onUpdateCascadeChangesChildForeignKeyWhenParentPrimaryKeyChanges() { + val db = newDatabase() + db.seedCascadeDepartments() + db.run("UPDATE departments SET id = 10 WHERE id = 1") + val depts = db.select("SELECT dept_id FROM employees ORDER BY id") + assertEquals(listOf(listOf(int(10)), listOf(int(10)), listOf(int(2))), depts.rows) + } + + @Test + fun onUpdateRestrictByDefaultBlocksPrimaryKeyChangeWhenChildrenReference() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + assertFailsWith { db.run("UPDATE departments SET id = 99 WHERE id = 1") } + } + + @Test + fun onDeleteSetNullNullsOutReferencingChildren() { + val db = newDatabase() + db.seedSetNullDepartments(childNotNull = false) + db.run("DELETE FROM departments WHERE id = 1") + assertEquals(listOf(listOf(Value.NullValue)), db.select("SELECT dept_id FROM employees WHERE id = 1").rows) + } + + @Test + fun onDeleteSetNullFailsWhenChildColumnIsNotNull() { + val db = newDatabase() + db.seedSetNullDepartments(childNotNull = true) + val error = assertFailsWith { db.run("DELETE FROM departments WHERE id = 1") } + assertTrue(error.message!!.contains("NOT NULL")) + } + + @Test + fun selfReferencingForeignKeyIsAllowedAndCascadesThroughMultipleLevels() { + val db = newDatabase() + db.run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "manager_id INTEGER REFERENCES employees(id) ON DELETE CASCADE)" + ) + db.run( + "INSERT INTO employees VALUES " + + "(1, 'ceo', NULL), (2, 'vp', 1), (3, 'manager', 2), (4, 'ic', 3)" + ) + assertEquals(DeleteResult(1), db.run("DELETE FROM employees WHERE id = 1").single()) + assertEquals(emptyList(), db.select("SELECT id FROM employees").rows) + } + + @Test + fun selfReferencingForeignKeyAllowsForwardAndBackwardSiblingReferencesInTheSameInsertBatch() { + val db = newDatabase() + db.run( + "CREATE TABLE employees (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + + "manager_id INTEGER REFERENCES employees(id))" + ) + // row 1 references row 2 (forward, not yet on disk) and row 3 references row 1 (backward) - + // order-independence within a single multi-row INSERT statement. + assertEquals( + InsertResult(3), + db.run("INSERT INTO employees VALUES (1, 'a', 2), (2, 'b', NULL), (3, 'c', 1)").single() + ) + } + + @Test + fun dropTableIsRestrictedByDefaultWhenAForeignKeyReferencesIt() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + val error = assertFailsWith { db.run("DROP TABLE departments") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropTableCascadeStripsTheForeignKeyButKeepsTheReferencingTableAndItsRows() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + db.run("DROP TABLE departments CASCADE") + assertEquals(listOf("employees"), db.catalog.tableNames) + assertEquals(1, db.select("SELECT * FROM employees").rows.size) + // the constraint is gone, so a now-dangling value is accepted + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (2, 'linus', 99)").single()) + } + + @Test + fun dropTableCascadeStrippedForeignKeySurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + db.run("DROP TABLE departments CASCADE") + db.reopen() + assertEquals(listOf("employees"), db.catalog.tableNames) + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (2, 'linus', 99)").single()) + } + + @Test + fun dropTableCascadeDropsDependentViewsAndStripsForeignKeysInTheSameStatement() { + val db = newDatabase() + db.seedDepartmentsAndEmployees() + db.run("CREATE VIEW deptNames AS SELECT name FROM departments") + db.run("INSERT INTO employees VALUES (1, 'ada', 1)") + db.run("DROP TABLE departments CASCADE") + assertEquals(listOf("employees"), db.catalog.tableNames) + assertTrue(db.catalog.viewNames.isEmpty()) + assertEquals(InsertResult(1), db.run("INSERT INTO employees VALUES (2, 'linus', 99)").single()) + } + + @Test + fun foreignKeyConstraintSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedCascadeDepartments() + db.reopen() + + assertFailsWith { db.run("INSERT INTO employees VALUES (4, 'ed', 99)") } + db.run("DELETE FROM departments WHERE id = 1") + assertEquals(listOf(listOf(text("grace"))), db.select("SELECT name FROM employees ORDER BY id").rows) + + db.reopen() + assertFailsWith { db.run("DROP TABLE departments") } + assertEquals(1, db.select("SELECT * FROM employees").rows.size) + } + + // ---- ALTER TABLE ADD COLUMN ---- + + @Test + fun addColumnAppendsNullToExistingRowsWhenNoDefaultIsGiven() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + val result = db.run("ALTER TABLE t ADD COLUMN age INTEGER").single() + assertEquals(AlterTableResult("t"), result) + assertEquals(listOf(listOf(int(1), text("ada"), Value.NullValue)), db.select("SELECT * FROM t").rows) + } + + @Test + fun addColumnBackfillsExistingRowsWithDefaultValue() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada'), (2, 'linus')") + db.run("ALTER TABLE t ADD COLUMN status TEXT DEFAULT 'active'") + assertEquals( + listOf(listOf(text("active")), listOf(text("active"))), + db.select("SELECT status FROM t ORDER BY id ASC").rows + ) + } + + @Test + fun addColumnRejectsNotNullWithoutDefaultWhenTableHasRows() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1)") + val error = assertFailsWith { db.run("ALTER TABLE t ADD COLUMN name TEXT NOT NULL") } + assertTrue(error.message!!.contains("NOT NULL")) + assertEquals(listOf("id"), db.select("SELECT * FROM t").columns) + } + + @Test + fun addColumnAllowsNotNullWithoutDefaultWhenTableIsEmpty() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("ALTER TABLE t ADD COLUMN name TEXT NOT NULL") + db.run("INSERT INTO t VALUES (1, 'ada')") + assertEquals(listOf(listOf(text("ada"))), db.select("SELECT name FROM t").rows) + assertFailsWith { db.run("INSERT INTO t (id) VALUES (2)") } + } + + @Test + fun addColumnRejectsDuplicateColumnName() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { db.run("ALTER TABLE t ADD COLUMN name TEXT") } + assertTrue(error.message!!.contains("duplicate column")) + } + + @Test + fun addColumnRequiresTableToExist() { + val db = newDatabase() + assertFailsWith { db.run("ALTER TABLE missing ADD COLUMN x INTEGER") } + } + + @Test + fun newColumnParticipatesInInsertUpdateWhereAndSelect() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + db.run("ALTER TABLE t ADD COLUMN age INTEGER DEFAULT 0") + db.run("INSERT INTO t VALUES (2, 'linus', 54)") + db.run("UPDATE t SET age = 36 WHERE id = 1") + assertEquals(listOf(listOf(text("linus"))), db.select("SELECT name FROM t WHERE age > 40").rows) + } + + @Test + fun addColumnOnATableWithASecondaryIndexKeepsThePushdownWorking() { + val db = newDatabase() + db.seedUsers() + db.run("CREATE INDEX idx_age ON users(age)") + db.run("ALTER TABLE users ADD COLUMN dept TEXT DEFAULT 'eng'") + val table = db.catalog.table("users") + val scansBefore = table.scanCount + val result = db.select("SELECT name FROM users WHERE age = 54") + assertEquals(listOf(listOf(text("linus"))), result.rows) + assertEquals(1, table.secondaryLookupCount) + assertEquals(scansBefore, table.scanCount) + assertEquals(listOf(listOf(text("eng"))), db.select("SELECT dept FROM users WHERE id = 1").rows) + } + + @Test + fun addColumnDefinitionAndBackfilledDataSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + db.run("ALTER TABLE t ADD COLUMN status TEXT DEFAULT 'active'") + db.reopen() + assertEquals(listOf(listOf(int(1), text("ada"), text("active"))), db.select("SELECT * FROM t").rows) + db.run("INSERT INTO t VALUES (2, 'linus', DEFAULT)") + assertEquals(listOf(listOf(text("active"))), db.select("SELECT status FROM t WHERE id = 2").rows) + } + + // ---- ALTER TABLE DROP COLUMN ---- + + @Test + fun dropColumnRemovesColumnFromSchemaAndData() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, bio TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada', 'mathematician')") + db.run("ALTER TABLE t DROP COLUMN bio") + val result = db.select("SELECT * FROM t") + assertEquals(listOf("id", "name"), result.columns) + assertEquals(listOf(listOf(int(1), text("ada"))), result.rows) + } + + @Test + fun dropColumnRejectsWhenItIsTheOnlyColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN id") } + assertTrue(error.message!!.contains("no columns")) + } + + @Test + fun dropColumnRejectsPrimaryKeyColumnEvenWithCascade() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN id CASCADE") } + assertTrue(error.message!!.contains("PRIMARY KEY")) + } + + @Test + fun dropColumnRestrictsWhenReferencedByAnotherTablesForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT UNIQUE)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT REFERENCES departments(code))") + val error = assertFailsWith { db.run("ALTER TABLE departments DROP COLUMN code") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropColumnCascadeRemovesTheReferencingForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY, code TEXT UNIQUE)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_code TEXT REFERENCES departments(code))") + db.run("INSERT INTO departments VALUES (1, 'eng')") + db.run("ALTER TABLE departments DROP COLUMN code CASCADE") + db.run("INSERT INTO employees VALUES (1, 'ghost')") + assertEquals(listOf(listOf(text("ghost"))), db.select("SELECT dept_code FROM employees").rows) + } + + @Test + fun dropColumnRestrictsWhenUsedByItsOwnForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + val error = assertFailsWith { db.run("ALTER TABLE employees DROP COLUMN dept_id") } + assertTrue(error.message!!.contains("FOREIGN KEY")) + } + + @Test + fun dropColumnCascadeRemovesItsOwnForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + db.run("ALTER TABLE employees DROP COLUMN dept_id CASCADE") + assertEquals(listOf("id"), db.select("SELECT * FROM employees").columns) + } + + @Test + fun dropColumnRestrictsWhenUsedByAUniqueConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN email") } + assertTrue(error.message!!.contains("UNIQUE")) + } + + @Test + fun dropColumnCascadeRemovesTheUniqueConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO t VALUES (1, 'a@x.com')") + db.run("ALTER TABLE t DROP COLUMN email CASCADE") + db.run("INSERT INTO t VALUES (2)") + assertEquals(2, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun dropColumnRestrictsWhenUsedByACheckConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN age") } + assertTrue(error.message!!.contains("CHECK")) + } + + @Test + fun dropColumnCascadeRemovesTheCheckConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + db.run("ALTER TABLE t DROP COLUMN age CASCADE") + db.run("INSERT INTO t VALUES (1)") + assertEquals(1, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun dropColumnRestrictsWhenUsedByAnIndex() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + db.run("CREATE INDEX idx_age ON t (age)") + val error = assertFailsWith { db.run("ALTER TABLE t DROP COLUMN age") } + assertTrue(error.message!!.contains("idx_age")) + } + + /** + * Beyond mirroring the in-memory engine's own coverage, this also proves the position + * realignment [DiskTable.rewriteRows] performs for every surviving secondary index: `dept` + * shifts from column position 2 to 1 once `age` (position 1) is dropped, and `idx_dept` must + * keep resolving against the new position rather than the stale one it was built with. + */ + @Test + fun dropColumnCascadeRemovesTheIndexAndASurvivingLaterIndexStillResolvesAtItsShiftedPosition() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER, dept TEXT)") + db.run("INSERT INTO t VALUES (1, 30, 'eng'), (2, 40, 'ops')") + db.run("CREATE INDEX idx_age ON t (age)") + db.run("CREATE INDEX idx_dept ON t (dept)") + db.run("ALTER TABLE t DROP COLUMN age CASCADE") + + val table = db.catalog.table("t") + assertEquals(listOf("idx_dept"), table.secondaryIndexes.map { it.name }) + val result = db.select("SELECT id FROM t WHERE dept = 'ops'") + assertEquals(listOf(listOf(int(2))), result.rows) + assertEquals(1, table.secondaryLookupCount) + } + + @Test + fun dropColumnDefinitionAndRemainingDataSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, bio TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada', 'mathematician')") + db.run("ALTER TABLE t DROP COLUMN bio") + db.reopen() + assertEquals(listOf("id", "name"), db.select("SELECT * FROM t").columns) + assertEquals(listOf(listOf(int(1), text("ada"))), db.select("SELECT * FROM t").rows) + } + + // ---- ALTER TABLE RENAME COLUMN ---- + + @Test + fun renameColumnRenamesSchemaAndPreservesData() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, bio TEXT)") + db.run("INSERT INTO t VALUES (1, 'mathematician')") + db.run("ALTER TABLE t RENAME COLUMN bio TO biography") + val result = db.select("SELECT * FROM t") + assertEquals(listOf("id", "biography"), result.columns) + assertEquals(listOf(listOf(int(1), text("mathematician"))), result.rows) + } + + @Test + fun renameColumnRejectsUnknownColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + assertFailsWith { db.run("ALTER TABLE t RENAME COLUMN missing TO whatever") } + } + + @Test + fun renameColumnRejectsWhenNewNameAlreadyExists() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + val error = assertFailsWith { db.run("ALTER TABLE t RENAME COLUMN name TO id") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameColumnUpdatesCompositePrimaryKeyListing() { + val db = newDatabase() + db.run("CREATE TABLE t (region TEXT, seq INTEGER, PRIMARY KEY (region, seq))") + db.run("INSERT INTO t VALUES ('eu', 1)") + db.run("ALTER TABLE t RENAME COLUMN region TO area") + assertFailsWith { db.run("INSERT INTO t VALUES ('eu', 1)") } + db.run("INSERT INTO t VALUES ('eu', 2)") + assertEquals(2, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun renameColumnUpdatesItsOwnForeignKeyColumns() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + db.run("INSERT INTO departments VALUES (1)") + db.run("ALTER TABLE employees RENAME COLUMN dept_id TO department_id") + assertFailsWith { db.run("INSERT INTO employees VALUES (1, 99)") } + db.run("INSERT INTO employees VALUES (1, 1)") + assertEquals(1, db.select("SELECT * FROM employees").rows.size) + } + + @Test + fun renameColumnPropagatesToOtherTablesForeignKeyReferencedColumns() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE)") + db.run("INSERT INTO departments VALUES (1)") + db.run("INSERT INTO employees VALUES (1, 1)") + db.run("ALTER TABLE departments RENAME COLUMN id TO dept_pk") + db.run("DELETE FROM departments WHERE dept_pk = 1") + assertTrue(db.select("SELECT * FROM employees").rows.isEmpty()) + } + + @Test + fun renameColumnUpdatesAUniqueConstraint() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, email TEXT UNIQUE)") + db.run("INSERT INTO t VALUES (1, 'a@x.com')") + db.run("ALTER TABLE t RENAME COLUMN email TO contact_email") + assertFailsWith { db.run("INSERT INTO t VALUES (2, 'a@x.com')") } + } + + @Test + fun renameColumnUpdatesACheckConstraintAndStillEnforcesIt() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER CHECK (age > 0))") + db.run("ALTER TABLE t RENAME COLUMN age TO years") + assertFailsWith { db.run("INSERT INTO t VALUES (1, -5)") } + db.run("INSERT INTO t VALUES (1, 30)") + assertEquals(listOf(listOf(int(30))), db.select("SELECT years FROM t").rows) + } + + @Test + fun renameColumnUpdatesASecondaryIndexAndPushdownStillWorks() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + db.run("INSERT INTO t VALUES (1, 30), (2, 40)") + db.run("CREATE INDEX idx_age ON t (age)") + db.run("ALTER TABLE t RENAME COLUMN age TO years") + + val table = db.catalog.table("t") + val result = db.select("SELECT id FROM t WHERE years = 30") + assertEquals(listOf(listOf(int(1))), result.rows) + assertEquals(1, table.secondaryLookupCount) + assertEquals(listOf("years"), table.secondaryIndexes.single().columnNames) + } + + @Test + fun renameColumnDefinitionSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, bio TEXT)") + db.run("INSERT INTO t VALUES (1, 'mathematician')") + db.run("ALTER TABLE t RENAME COLUMN bio TO biography") + db.reopen() + assertEquals(listOf("id", "biography"), db.select("SELECT * FROM t").columns) + assertEquals(listOf(listOf(int(1), text("mathematician"))), db.select("SELECT * FROM t").rows) + } + + // ---- ALTER TABLE RENAME TO ---- + + @Test + fun renameTableRenamesAndPreservesData() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + val result = db.run("ALTER TABLE t RENAME TO people").single() + assertEquals(AlterTableResult("people"), result) + assertEquals(listOf(listOf(int(1), text("ada"))), db.select("SELECT * FROM people").rows) + assertFailsWith { db.run("SELECT * FROM t") } + assertEquals(listOf("people"), db.catalog.tableNames) + } + + @Test + fun renameTableRejectsWhenNewNameAlreadyExistsAsTable() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + val error = assertFailsWith { db.run("ALTER TABLE t RENAME TO other") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameTableRejectsWhenNewNameAlreadyExistsAsView() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE other (id INTEGER PRIMARY KEY)") + db.run("CREATE VIEW v AS SELECT * FROM other") + val error = assertFailsWith { db.run("ALTER TABLE t RENAME TO v") } + assertTrue(error.message!!.contains("already exists")) + } + + @Test + fun renameTableRejectsRenamingAView() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE VIEW v AS SELECT * FROM t") + val error = assertFailsWith { db.run("ALTER TABLE v RENAME TO w") } + assertTrue(error.message!!.contains("cannot modify view")) + } + + @Test + fun renameTablePropagatesToOtherTablesForeignKeyReference() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id) ON DELETE CASCADE)") + db.run("INSERT INTO departments VALUES (1)") + db.run("INSERT INTO employees VALUES (1, 1)") + db.run("ALTER TABLE departments RENAME TO divisions") + db.run("INSERT INTO divisions VALUES (2)") + db.run("INSERT INTO employees VALUES (2, 2)") + assertFailsWith { db.run("INSERT INTO employees VALUES (3, 99)") } + db.run("DELETE FROM divisions WHERE id = 1") + assertEquals(listOf(listOf(int(2))), db.select("SELECT id FROM employees").rows) + } + + @Test + fun renameTableUpdatesItsOwnSelfReferencingForeignKey() { + val db = newDatabase() + db.run("CREATE TABLE staff (id INTEGER PRIMARY KEY, manager_id INTEGER REFERENCES staff(id))") + db.run("INSERT INTO staff VALUES (1, NULL)") + db.run("ALTER TABLE staff RENAME TO personnel") + db.run("INSERT INTO personnel VALUES (2, 1)") + assertFailsWith { db.run("INSERT INTO personnel VALUES (3, 99)") } + assertEquals(2, db.select("SELECT * FROM personnel").rows.size) + } + + @Test + fun renameTableUpdatesASecondaryIndexAndPushdownStillWorks() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, age INTEGER)") + db.run("INSERT INTO t VALUES (1, 30), (2, 40)") + db.run("CREATE INDEX idx_age ON t (age)") + db.run("ALTER TABLE t RENAME TO people") + + val table = db.catalog.table("people") + val result = db.select("SELECT id FROM people WHERE age = 30") + assertEquals(listOf(listOf(int(1))), result.rows) + assertEquals(1, table.secondaryLookupCount) + } + + @Test + fun renameTablePropagatesIntoASimpleViewDefinition() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + db.run("CREATE VIEW v AS SELECT * FROM t") + db.run("ALTER TABLE t RENAME TO people") + assertEquals(listOf(listOf(int(1), text("ada"))), db.select("SELECT * FROM v").rows) + } + + @Test + fun renameTableRenamingToItsOwnNameIsANoOp() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1)") + db.run("ALTER TABLE t RENAME TO t") + assertEquals(listOf(listOf(int(1))), db.select("SELECT * FROM t").rows) + } + + @Test + fun renameTableDefinitionAndViewSurviveClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") + db.run("INSERT INTO t VALUES (1, 'ada')") + db.run("CREATE VIEW v AS SELECT * FROM t") + db.run("ALTER TABLE t RENAME TO people") + db.reopen() + assertEquals(listOf("people"), db.catalog.tableNames) + assertEquals(listOf(listOf(int(1), text("ada"))), db.select("SELECT * FROM v").rows) + assertFailsWith { db.run("SELECT * FROM t") } + } + + /** + * The regression this guards against: [DiskCatalog.renameTable] used to capture the directory + * key a [DiskTable]'s `onRootsChanged` callback persists under *by value* at table-creation + * time. After a rename moved the row to a new map key, any *later* write that actually split a + * B+Tree (not the rename itself, which persists synchronously) would still try to persist under + * the stale, now-removed key and blow up with `NoSuchElementException`. 300 sequential inserts + * reliably force at least one primary-key index split (same row count [DiskCatalogTest] already + * relies on for the same reason). + */ + @Test + fun renameTableFollowedByEnoughInsertsToSplitTheIndexPersistsCorrectlyAfterReopen() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1)") + db.run("ALTER TABLE t RENAME TO renamed") + for (id in 2..300) db.run("INSERT INTO renamed VALUES ($id)") + + db.reopen() + assertEquals(300, db.select("SELECT * FROM renamed").rows.size) + for (id in listOf(1, 150, 300)) { + assertEquals(listOf(listOf(int(id.toLong()))), db.select("SELECT id FROM renamed WHERE id = $id").rows) + } + } + + /** + * `PREPARE`/`EXECUTE`/`DEALLOCATE` ported to the disk-backed executor - see [DiskExecutor]'s own + * class doc. These mirror [kestrel.PreparedStatementTest] (the in-memory engine's own suite for + * the same feature) case by case, plus two disk-specific ones proving via a real close/reopen of + * the backing files that the registry is genuinely session-scoped rather than durable - the same + * distinction [kestrel.PersistenceTest] draws for the in-memory engine's own text WAL. + */ + @Test + fun prepareReturnsItsNameAndExecuteRunsAParameterizedSelect() { + val db = newDatabase() + db.seedUsers() + assertEquals( + PrepareResult("byId"), + db.run("PREPARE byId AS SELECT name, age FROM users WHERE id = \$1").single() + ) + val result = db.select("EXECUTE byId(2)") + assertEquals(listOf("name", "age"), result.columns) + assertEquals(listOf(listOf(text("linus"), int(54))), result.rows) + } + + @Test + fun theSamePreparedStatementCanBeExecutedRepeatedlyWithDifferentArguments() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + assertEquals(listOf(listOf(text("ada"))), db.select("EXECUTE byId(1)").rows) + assertEquals(listOf(listOf(text("grace"))), db.select("EXECUTE byId(3)").rows) + } + + @Test + fun executeSubstitutesMultipleParametersPositionally() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE inRange AS SELECT name FROM users WHERE age >= \$1 AND age <= \$2 ORDER BY age") + assertEquals( + listOf(listOf(text("ada")), listOf(text("linus"))), + db.select("EXECUTE inRange(30, 60)").rows + ) + } + + @Test + fun prepareCanWrapInsertAndExecuteInsertsARealRow() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER DEFAULT 0)") + db.run("PREPARE addUser AS INSERT INTO users VALUES (\$1, \$2, \$3)") + assertEquals(InsertResult(1), db.run("EXECUTE addUser(1, 'ada', 36)").single()) + assertEquals(listOf(listOf(int(1), text("ada"), int(36))), db.select("SELECT * FROM users").rows) + assertEquals(InsertResult(1), db.run("EXECUTE addUser(2, 'linus', 54)").single()) + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun prepareCanWrapUpdate() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE birthday AS UPDATE users SET age = age + \$1 WHERE id = \$2") + assertEquals(UpdateResult(1), db.run("EXECUTE birthday(1, 1)").single()) + assertEquals(listOf(listOf(int(37))), db.select("SELECT age FROM users WHERE id = 1").rows) + } + + @Test + fun prepareCanWrapDelete() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE removeUser AS DELETE FROM users WHERE id = \$1") + assertEquals(DeleteResult(1), db.run("EXECUTE removeUser(2)").single()) + assertEquals(2, db.select("SELECT * FROM users").rows.size) + } + + @Test + fun prepareCanWrapMergeAndAParameterReachesTheMatchedUpdateAssignment() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + db.run("CREATE TABLE staging (id INTEGER PRIMARY KEY, name TEXT NOT NULL, balance INTEGER)") + db.run("INSERT INTO accounts VALUES (1, 'ada', 100)") + db.run("INSERT INTO staging VALUES (1, 'ada', 500)") + db.run( + "PREPARE topUp AS MERGE INTO accounts USING staging ON accounts.id = staging.id " + + "WHEN MATCHED THEN UPDATE SET balance = staging.balance + \$1" + ) + db.run("EXECUTE topUp(10)") + assertEquals(listOf(listOf(int(510))), db.select("SELECT balance FROM accounts WHERE id = 1").rows) + // A second EXECUTE with a DIFFERENT argument proves substitution recomputes fresh every call + // rather than baking in whatever value the first call used. + db.run("EXECUTE topUp(20)") + assertEquals(listOf(listOf(int(520))), db.select("SELECT balance FROM accounts WHERE id = 1").rows) + } + + @Test + fun executeWithTheWrongNumberOfArgumentsFailsWithAClearMessage() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + val tooMany = assertFailsWith { db.run("EXECUTE byId(1, 2)") } + assertTrue(tooMany.message!!.contains("expects 1 parameter"), tooMany.message) + assertFailsWith { db.run("EXECUTE byId()") } + } + + @Test + fun executeOfAnUnknownNameFails() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + assertFailsWith { db.run("EXECUTE nope(1)") } + } + + @Test + fun preparingTheSameNameTwiceFails() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + assertFailsWith { db.run("PREPARE byId AS SELECT age FROM users WHERE id = \$1") } + } + + @Test + fun deallocateRemovesAPreparedStatementSoExecuteNoLongerFindsIt() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + assertEquals(DeallocateResult("byId", all = false), db.run("DEALLOCATE byId").single()) + assertFailsWith { db.run("EXECUTE byId(1)") } + } + + @Test + fun deallocateAllRemovesEveryPreparedStatement() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE p1 AS SELECT name FROM users WHERE id = \$1") + db.run("PREPARE p2 AS SELECT age FROM users WHERE id = \$1") + assertEquals(DeallocateResult(null, all = true), db.run("DEALLOCATE ALL").single()) + assertFailsWith { db.run("EXECUTE p1(1)") } + assertFailsWith { db.run("EXECUTE p2(1)") } + } + + @Test + fun aNameCanBeReparedAfterBeingDeallocated() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + db.run("DEALLOCATE byId") + db.run("PREPARE byId AS SELECT age FROM users WHERE id = \$1") + assertEquals(listOf(listOf(int(36))), db.select("EXECUTE byId(1)").rows) + } + + @Test + fun preparedStatementsAreNotUndoneByRollback() { + val db = newDatabase() + db.seedUsers() + db.run("BEGIN") + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + db.run("ROLLBACK") + // Session-scoped, deliberately outside the storage-engine transaction that ROLLBACK undoes - + // see DiskExecutor's own doc on preparedStatements for why. + assertEquals(listOf(listOf(text("ada"))), db.select("EXECUTE byId(1)").rows) + } + + @Test + fun executeArgumentsMustBeConstantNotColumnReferences() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + // The bare Evaluator() resolveExecute uses to evaluate EXECUTE arguments as constants is + // shared with the in-memory engine and throws the shared KestrelException directly, not a + // DiskExecutorException - unlike disk-executor-specific validation elsewhere in this class. + assertFailsWith { db.run("EXECUTE byId(id)") } + } + + @Test + fun preparedStatementDoesNotSurviveReopen() { + val db = newDatabase() + db.seedUsers() + db.run("PREPARE byId AS SELECT name FROM users WHERE id = \$1") + assertEquals(listOf(listOf(text("ada"))), db.select("EXECUTE byId(1)").rows) + + db.reopen() + assertFailsWith { db.run("EXECUTE byId(1)") } + } + + @Test + fun aPreparedInsertsRowSurvivesReopenEvenThoughThePrepareItselfDoesNot() { + val db = newDatabase() + db.run("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, age INTEGER DEFAULT 0)") + db.run("PREPARE addUser AS INSERT INTO users VALUES (\$1, \$2, \$3)") + db.run("EXECUTE addUser(1, 'ada', 36)") + + db.reopen() + assertEquals(listOf(listOf(int(1), text("ada"), int(36))), db.select("SELECT * FROM users").rows) + assertFailsWith { db.run("EXECUTE addUser(2, 'linus', 54)") } + } + + /** + * `CREATE TEMP TABLE`/`CREATE TEMPORARY TABLE` used to be rejected outright here - see + * [DiskExecutorTemporaryTableTest] for the full feature (now a real, live in-memory-only table + * on this executor) this pair of tests only confirms is reachable through both spellings. + */ + @Test + fun createTempTableIsAcceptedByTheDiskBackedExecutor() { + val db = newDatabase() + assertEquals(CreateTableResult("scratch"), db.run("CREATE TEMP TABLE scratch (id INTEGER PRIMARY KEY)").single()) + db.run("INSERT INTO scratch VALUES (1)") + assertEquals(listOf(listOf(int(1))), db.select("SELECT * FROM scratch").rows) + } + + @Test + fun createTemporaryTableIsAcceptedByTheDiskBackedExecutor() { + val db = newDatabase() + assertEquals(CreateTableResult("scratch"), db.run("CREATE TEMPORARY TABLE scratch (id INTEGER PRIMARY KEY)").single()) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorTriggerTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorTriggerTest.kt new file mode 100644 index 0000000..4cda550 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorTriggerTest.kt @@ -0,0 +1,436 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `CREATE`/`DROP TRIGGER` `BEFORE`/`AFTER INSERT`/`UPDATE`/`DELETE` through [DiskExecutor] against a + * real [DiskCatalog] backed by temp files - see [DiskExecutorTest] for the general end-to-end pattern + * this reuses. Scenarios are ported 1:1 from [kestrel.TriggerTest] (the in-memory engine's equivalent) + * wherever they translate, since firing/WHEN/validation semantics are identical (both engines share + * [kestrel.engine.TriggerRowContext]); [triggerCascadeAtomicitySurvivesAReopen] and + * [triggerFiringInsideAnExplicitTransactionParticipatesInItsCommit] are new to this file, proving what + * is actually different on the disk-backed path: atomicity comes from an internal storage-engine + * savepoint (see [DiskExecutor.withTriggerAtomicity]) rather than a whole-catalog snapshot/restore. + */ +class DiskExecutorTriggerTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + private fun text(value: String) = Value.TextValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-trigger-data"), tempFile("kestrel-disk-trigger-wal")) + + @Test + fun afterInsertTriggerRunsBodyForEachInsertedRow() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("CREATE TABLE audit (order_id INTEGER)") + db.run( + "CREATE TRIGGER log_insert AFTER INSERT ON orders FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + db.run("INSERT INTO orders VALUES (1, 100), (2, 200)") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), db.select("SELECT order_id FROM audit ORDER BY order_id").rows) + } + + @Test + fun beforeInsertTriggerCanAbortTheWholeStatementBeforeThisExecutorsOwnConstraintsRun() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE guard (x INTEGER NOT NULL)") + db.run("CREATE TRIGGER reject_all BEFORE INSERT ON t FOR EACH ROW BEGIN INSERT INTO guard VALUES (NULL); END") + // The row itself is perfectly valid for t's own PRIMARY KEY - only the trigger rejects it. + assertFailsWith { db.run("INSERT INTO t VALUES (1)") } + assertEquals(0, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun beforeTriggerSideEffectsAreRolledBackWhenTheFiringStatementItselfFails() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY CHECK (id > 0))") + db.run("CREATE TABLE counter (n INTEGER)") + db.run("INSERT INTO counter VALUES (0)") + db.run("CREATE TRIGGER bump BEFORE INSERT ON t FOR EACH ROW BEGIN UPDATE counter SET n = n + 1; END") + assertFailsWith { db.run("INSERT INTO t VALUES (-1)") } + assertEquals(listOf(listOf(int(0))), db.select("SELECT n FROM counter").rows) + assertEquals(0, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun afterInsertTriggerFailureRollsBackTheOriginalInsertToo() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE audit (order_id INTEGER NOT NULL)") + db.run("CREATE TRIGGER log_fail AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NULL); END") + assertFailsWith { db.run("INSERT INTO t VALUES (1)") } + assertEquals(0, db.select("SELECT * FROM t").rows.size) + assertEquals(0, db.select("SELECT * FROM audit").rows.size) + } + + @Test + fun whenClauseGatesFiringPerRow() { + val db = newDatabase() + db.run("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount INTEGER)") + db.run("CREATE TABLE audit (order_id INTEGER)") + db.run( + "CREATE TRIGGER log_big AFTER INSERT ON orders FOR EACH ROW WHEN NEW.amount > 100 " + + "BEGIN INSERT INTO audit VALUES (NEW.id); END" + ) + db.run("INSERT INTO orders VALUES (1, 50), (2, 200)") + assertEquals(listOf(listOf(int(2))), db.select("SELECT order_id FROM audit").rows) + } + + @Test + fun updateTriggerSeesBothOldAndNew() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + db.run("CREATE TABLE audit (account_id INTEGER, old_balance INTEGER, new_balance INTEGER)") + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run( + "CREATE TRIGGER log_balance_change AFTER UPDATE ON accounts FOR EACH ROW " + + "BEGIN INSERT INTO audit VALUES (NEW.id, OLD.balance, NEW.balance); END" + ) + db.run("UPDATE accounts SET balance = 150 WHERE id = 1") + assertEquals(listOf(listOf(int(1), int(100), int(150))), db.select("SELECT * FROM audit").rows) + } + + @Test + fun updateTriggerFiresForEveryMatchedRowEvenWhenTheValueIsUnchanged() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + db.run("CREATE TABLE audit (n INTEGER)") + db.run("INSERT INTO t VALUES (1, 5)") + db.run("CREATE TRIGGER log AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (1); END") + db.run("UPDATE t SET x = 5 WHERE id = 1") + assertEquals(1, db.select("SELECT * FROM audit").rows.size) + } + + @Test + fun deleteTriggerSeesOld() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT)") + db.run("CREATE TABLE audit (label TEXT)") + db.run("INSERT INTO t VALUES (1, 'gone')") + db.run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (OLD.label); END") + db.run("DELETE FROM t WHERE id = 1") + assertEquals(listOf(listOf(text("gone"))), db.select("SELECT * FROM audit").rows) + } + + @Test + fun multipleTriggersOnSameEventFireInCreationOrder() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (step TEXT)") + db.run("CREATE TRIGGER first_trigger AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES ('first'); END") + db.run("CREATE TRIGGER second_trigger AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES ('second'); END") + db.run("INSERT INTO t VALUES (1)") + assertEquals(listOf(listOf(text("first")), listOf(text("second"))), db.select("SELECT * FROM audit").rows) + } + + @Test + fun cascadingTriggerFiresATriggerOnADifferentTable() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER)") + db.run("CREATE TABLE b (id INTEGER)") + db.run("CREATE TABLE c (id INTEGER)") + db.run("CREATE TRIGGER a_to_b AFTER INSERT ON a FOR EACH ROW BEGIN INSERT INTO b VALUES (NEW.id); END") + db.run("CREATE TRIGGER b_to_c AFTER INSERT ON b FOR EACH ROW BEGIN INSERT INTO c VALUES (NEW.id); END") + db.run("INSERT INTO a VALUES (7)") + assertEquals(listOf(listOf(int(7))), db.select("SELECT * FROM b").rows) + assertEquals(listOf(listOf(int(7))), db.select("SELECT * FROM c").rows) + } + + @Test + fun recursiveTriggerHitsTheDepthGuardInsteadOfLoopingForeverAndRollsBackCleanly() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TRIGGER loopy AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO t VALUES (NEW.id + 1); END") + val exception = assertFailsWith { db.run("INSERT INTO t VALUES (1)") } + assertTrue(exception.message!!.contains("recursion", ignoreCase = true), exception.message!!) + assertEquals(0, db.select("SELECT * FROM t").rows.size) + } + + @Test + fun dropTriggerStopsItFromFiringAndCannotBeDroppedTwice() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("DROP TRIGGER log") + db.run("INSERT INTO t VALUES (1)") + assertEquals(0, db.select("SELECT * FROM audit").rows.size) + assertFailsWith { db.run("DROP TRIGGER log") } + } + + @Test + fun droppingTheTableDropsItsTriggersToo() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN UPDATE t SET id = id; END") + db.run("DROP TABLE t") + assertFailsWith { db.run("DROP TRIGGER log") } + } + + @Test + fun renamingTheTableKeepsItsTriggersFiring() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("ALTER TABLE t RENAME TO t2") + db.run("INSERT INTO t2 VALUES (42)") + assertEquals(listOf(listOf(int(42))), db.select("SELECT * FROM audit").rows) + } + + @Test + fun createTriggerRejectsUnknownTable() { + val db = newDatabase() + assertFailsWith { + db.run("CREATE TRIGGER log AFTER INSERT ON missing FOR EACH ROW BEGIN DELETE FROM missing WHERE 1 = 0; END") + } + } + + @Test + fun createTriggerRejectsDuplicateName() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + assertFailsWith { + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsAnUnqualifiedColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsNewOnADeleteTrigger() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run("CREATE TRIGGER log BEFORE DELETE ON t FOR EACH ROW WHEN NEW.id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsOldOnAnInsertTrigger() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run("CREATE TRIGGER log BEFORE INSERT ON t FOR EACH ROW WHEN OLD.id > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseAcceptsBothSidesOnAnUpdateTrigger() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TRIGGER log BEFORE UPDATE ON t FOR EACH ROW WHEN NEW.id > OLD.id BEGIN DELETE FROM t WHERE 1 = 0; END") + } + + @Test + fun whenClauseRejectsAnUnknownColumn() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN NEW.nope > 0 BEGIN DELETE FROM t WHERE 1 = 0; END") + } + } + + @Test + fun whenClauseRejectsASubquery() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run( + "CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW WHEN EXISTS (SELECT 1 FROM t) " + + "BEGIN DELETE FROM t WHERE 1 = 0; END" + ) + } + } + + @Test + fun bodyStatementRejectsAnUnknownTargetTable() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + assertFailsWith { + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN DELETE FROM missing WHERE 1 = 0; END") + } + } + + @Test + fun onConflictRowThatLandsWithNoConflictFiresInsertTriggers() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("INSERT INTO t VALUES (1, 1) ON CONFLICT (id) DO NOTHING") + assertEquals(listOf(listOf(int(1))), db.select("SELECT * FROM audit").rows) + } + + @Test + fun onConflictDoNothingFiresNoTriggerAtAllForTheConflictingRow() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, x INTEGER)") + db.run("INSERT INTO t VALUES (1, 1)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("INSERT INTO t VALUES (1, 99) ON CONFLICT (id) DO NOTHING") + assertEquals(0, db.select("SELECT * FROM audit").rows.size) + } + + @Test + fun onConflictDoUpdateFiresUpdateTriggersNotInsertTriggers() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("CREATE TABLE inserts (id INTEGER)") + db.run("CREATE TABLE updates (id INTEGER, old_balance INTEGER, new_balance INTEGER)") + db.run("CREATE TRIGGER log_insert AFTER INSERT ON accounts FOR EACH ROW BEGIN INSERT INTO inserts VALUES (NEW.id); END") + db.run( + "CREATE TRIGGER log_update AFTER UPDATE ON accounts FOR EACH ROW " + + "BEGIN INSERT INTO updates VALUES (NEW.id, OLD.balance, NEW.balance); END" + ) + db.run("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + assertEquals(0, db.select("SELECT * FROM inserts").rows.size) + assertEquals(listOf(listOf(int(1), int(100), int(999))), db.select("SELECT * FROM updates").rows) + } + + @Test + fun onConflictBeforeUpdateTriggerCanRejectTheWholeStatement() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("CREATE TABLE guard (x INTEGER NOT NULL)") + db.run("CREATE TRIGGER reject BEFORE UPDATE ON accounts FOR EACH ROW BEGIN INSERT INTO guard VALUES (NULL); END") + assertFailsWith { + db.run("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + } + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + } + + /** Disk-specific: the trigger cascade and the ON CONFLICT statement itself share one savepoint, so a BEFORE trigger's rejection survives a real close/reopen as if the whole statement never ran. */ + @Test + fun onConflictBeforeUpdateTriggerRejectionSurvivesAReopen() { + val db = newDatabase() + db.run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + db.run("INSERT INTO accounts VALUES (1, 100)") + db.run("CREATE TABLE guard (x INTEGER NOT NULL)") + db.run("CREATE TRIGGER reject BEFORE UPDATE ON accounts FOR EACH ROW BEGIN INSERT INTO guard VALUES (NULL); END") + assertFailsWith { + db.run("INSERT INTO accounts VALUES (1, 999) ON CONFLICT (id) DO UPDATE SET balance = excluded.balance") + } + db.reopen() + assertEquals(int(100), db.select("SELECT balance FROM accounts WHERE id = 1").rows[0][0]) + assertEquals(0, db.select("SELECT * FROM guard").rows.size) + } + + @Test + fun onConflictMultiRowStatementFiresTheRightTriggerPerRow() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, hits INTEGER)") + db.run("INSERT INTO t VALUES (1, 10)") + db.run("CREATE TABLE inserts (id INTEGER)") + db.run("CREATE TABLE updates (id INTEGER)") + db.run("CREATE TRIGGER log_insert AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO inserts VALUES (NEW.id); END") + db.run("CREATE TRIGGER log_update AFTER UPDATE ON t FOR EACH ROW BEGIN INSERT INTO updates VALUES (NEW.id); END") + db.run( + """ + INSERT INTO t VALUES (1, 1), (2, 2) + ON CONFLICT (id) DO UPDATE SET hits = t.hits + excluded.hits + """.trimIndent() + ) + assertEquals(listOf(listOf(int(2))), db.select("SELECT * FROM inserts").rows) + assertEquals(listOf(listOf(int(1))), db.select("SELECT * FROM updates").rows) + } + + @Test + fun bodyStatementInAnInsertTriggerCanOnlySeeNewNotOld() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("INSERT INTO t VALUES (5)") + assertEquals(listOf(listOf(int(5))), db.select("SELECT * FROM audit").rows) + // OLD is simply not bound for an INSERT trigger, so an OLD.id reference in the body falls + // through exactly like any other unresolvable column in an INSERT's VALUES list would. + db.run("CREATE TRIGGER bad AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (OLD.id); END") + assertFailsWith { db.run("INSERT INTO t VALUES (6)") } + } + + /** + * Disk-specific: proves a fired trigger's writes and the firing statement's own writes share the + * user's explicit `BEGIN`, not a savepoint-scoped transaction of their own - a `ROLLBACK` after + * both undoes everything, same as [DiskExecutorOnConflictTest.onConflictInsideAnExplicitTransactionParticipatesInItsCommit]. + */ + @Test + fun triggerFiringInsideAnExplicitTransactionParticipatesInItsCommit() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER)") + db.run("CREATE TABLE audit (id INTEGER)") + db.run("CREATE TRIGGER log AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NEW.id); END") + db.run("BEGIN") + db.run("INSERT INTO t VALUES (1)") + db.run("ROLLBACK") + assertEquals(0, db.select("SELECT * FROM t").rows.size) + assertEquals(0, db.select("SELECT * FROM audit").rows.size) + } + + /** + * Durability proof unique to the disk-backed executor: a trigger cascade's rollback-on-failure + * (via the internal storage-engine savepoint [DiskExecutor.withTriggerAtomicity] takes, not a + * whole-catalog snapshot) must leave nothing behind even after a real close/reopen, and a + * trigger's own definition must itself survive to keep firing after reopen - mirrors the reopen + * tests already established for ON CONFLICT ([DiskExecutorOnConflictTest.conflictResolutionSurvivesAReopen]). + */ + @Test + fun triggerCascadeAtomicitySurvivesAReopen() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE audit (order_id INTEGER NOT NULL)") + db.run("CREATE TRIGGER log_fail AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit VALUES (NULL); END") + assertFailsWith { db.run("INSERT INTO t VALUES (1)") } + db.reopen() + assertEquals(0, db.select("SELECT * FROM t").rows.size) + assertEquals(0, db.select("SELECT * FROM audit").rows.size) + db.run("DROP TRIGGER log_fail") + db.run("CREATE TABLE audit2 (id INTEGER)") + db.run("CREATE TRIGGER log_ok AFTER INSERT ON t FOR EACH ROW BEGIN INSERT INTO audit2 VALUES (NEW.id); END") + db.reopen() + db.run("INSERT INTO t VALUES (9)") + assertEquals(listOf(listOf(int(9))), db.select("SELECT * FROM audit2").rows) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskExecutorTruncateTest.kt b/src/test/kotlin/kestrel/store/sql/DiskExecutorTruncateTest.kt new file mode 100644 index 0000000..3b6a3cc --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskExecutorTruncateTest.kt @@ -0,0 +1,201 @@ +package kestrel.store.sql + +import kestrel.KestrelException +import kestrel.engine.StatementResult +import kestrel.engine.TruncateTableResult +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * `TRUNCATE TABLE` through [DiskExecutor] against a real [DiskCatalog] backed by temp files - see + * [DiskExecutorTest] for the general end-to-end pattern this reuses, and [kestrel.TruncateTest] for + * the in-memory engine's equivalent scenarios, ported 1:1 wherever they translate. + * [restartIdentityIsNotUndoneByARollbackUnlikeTheRowTruncationItself] has no in-memory equivalent - + * see [DiskExecutor.executeTruncate]'s own doc for the deliberate asymmetry it demonstrates. + */ +class DiskExecutorTruncateTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun exec(sql: String): StatementResult = run(sql).single() + fun select(sql: String): kestrel.engine.SelectResult = exec(sql) as kestrel.engine.SelectResult + + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-truncate-data"), tempFile("kestrel-disk-truncate-wal")) + + @Test + fun truncateRemovesEveryRowButKeepsTheTable() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY, label TEXT NOT NULL)") + db.run("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + val result = db.exec("TRUNCATE TABLE t") as TruncateTableResult + assertEquals(listOf("t"), result.tableNames) + assertEquals(emptyList(), db.select("SELECT * FROM t").rows) + db.run("INSERT INTO t VALUES (1, 'fresh')") + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM t").rows) + } + + @Test + fun multipleTablesInOneStatementAreAllTruncated() { + val db = newDatabase() + db.run("CREATE TABLE a (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE b (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO a VALUES (1)") + db.run("INSERT INTO b VALUES (1)") + val result = db.exec("TRUNCATE TABLE a, b") as TruncateTableResult + assertEquals(setOf("a", "b"), result.tableNames.toSet()) + assertEquals(emptyList(), db.select("SELECT * FROM a").rows) + assertEquals(emptyList(), db.select("SELECT * FROM b").rows) + } + + @Test + fun truncatingAnUnknownTableFails() { + val db = newDatabase() + assertFailsWith { db.run("TRUNCATE TABLE nope") } + } + + @Test + fun restartIdentityResetsTheOwningSequenceBackToItsStartValue() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + db.run("INSERT INTO t (label) VALUES ('a'), ('b'), ('c')") + db.exec("TRUNCATE TABLE t RESTART IDENTITY") + db.run("INSERT INTO t (label) VALUES ('fresh')") + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM t").rows) + } + + @Test + fun continueIdentityIsTheDefaultAndLeavesTheSequencePositionAlone() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + db.run("INSERT INTO t (label) VALUES ('a'), ('b')") + db.exec("TRUNCATE TABLE t") + db.run("INSERT INTO t (label) VALUES ('c')") + assertEquals(listOf(listOf(int(3))), db.select("SELECT id FROM t").rows) + } + + @Test + fun truncateIsRestrictedByDefaultWhenAnotherTableHasAForeignKeyIntoIt() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + db.run("INSERT INTO departments VALUES (1)") + db.run("INSERT INTO employees VALUES (1, 1)") + val error = assertFailsWith { db.run("TRUNCATE TABLE departments") } + assertTrue(error.message!!.contains("employees"), error.message) + assertEquals(listOf(listOf(int(1))), db.select("SELECT id FROM departments").rows) + } + + @Test + fun truncateCascadeAlsoEmptiesEveryTransitiveReferencer() { + val db = newDatabase() + db.run("CREATE TABLE departments (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, dept_id INTEGER REFERENCES departments(id))") + db.run("CREATE TABLE timesheets (id INTEGER PRIMARY KEY, employee_id INTEGER REFERENCES employees(id))") + db.run("INSERT INTO departments VALUES (1)") + db.run("INSERT INTO employees VALUES (1, 1)") + db.run("INSERT INTO timesheets VALUES (1, 1)") + val result = db.exec("TRUNCATE TABLE departments CASCADE") as TruncateTableResult + assertEquals(setOf("departments", "employees", "timesheets"), result.tableNames.toSet()) + assertEquals(emptyList(), db.select("SELECT * FROM timesheets").rows) + } + + @Test + fun selfReferencingForeignKeyDoesNotBlockTruncatingItsOwnTable() { + val db = newDatabase() + db.run("CREATE TABLE employees (id INTEGER PRIMARY KEY, manager_id INTEGER REFERENCES employees(id))") + db.run("INSERT INTO employees VALUES (1, NULL), (2, 1)") + db.exec("TRUNCATE TABLE employees") + assertEquals(emptyList(), db.select("SELECT * FROM employees").rows) + } + + @Test + fun truncateNeverFiresRowLevelTriggersUnlikeAWhereLessDelete() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("CREATE TABLE audit (n INTEGER)") + db.run("INSERT INTO audit VALUES (0)") + db.run("CREATE TRIGGER log_delete AFTER DELETE ON t FOR EACH ROW BEGIN UPDATE audit SET n = n + 1; END") + db.run("INSERT INTO t VALUES (1), (2), (3)") + db.exec("TRUNCATE TABLE t") + assertEquals(listOf(listOf(int(0))), db.select("SELECT n FROM audit").rows) + assertEquals(emptyList(), db.select("SELECT * FROM t").rows) + } + + @Test + fun truncateSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1), (2)") + db.exec("TRUNCATE TABLE t") + db.reopen() + assertEquals(emptyList(), db.select("SELECT * FROM t").rows) + } + + @Test + fun truncateCanRunInsideAnExplicitTransactionAndSharesItsCommit() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1)") + db.run("BEGIN") + db.exec("TRUNCATE TABLE t") + db.run("COMMIT") + assertEquals(emptyList(), db.select("SELECT * FROM t").rows) + } + + @Test + fun rollbackAfterTruncateInsideAnExplicitTransactionRestoresTheRows() { + val db = newDatabase() + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY)") + db.run("INSERT INTO t VALUES (1), (2)") + db.run("BEGIN") + db.exec("TRUNCATE TABLE t") + db.run("ROLLBACK") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), db.select("SELECT id FROM t ORDER BY id").rows) + } + + /** + * Unlike the in-memory engine (where ALTER SEQUENCE mutates a catalog map entry covered by the + * whole-catalog snapshot ROLLBACK restores), [DiskCatalog.alterSequence] always opens and commits + * its own storage-engine transaction, independent of the caller's [txn] - the same + * already-established non-transactional behavior [DiskCatalog.nextValue] has. So a ROLLBACK here + * undoes the row truncation but NOT the sequence restart: the very next NEXTVAL keeps counting + * from 1 (post-restart), not from 3 (where it would have been had TRUNCATE never run at all). + */ + @Test + fun restartIdentityIsNotUndoneByARollbackUnlikeTheRowTruncationItself() { + val db = newDatabase() + db.run("CREATE SEQUENCE seq") + db.run("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT NEXTVAL('seq'), label TEXT)") + db.run("INSERT INTO t (label) VALUES ('a'), ('b')") + db.run("BEGIN") + db.exec("TRUNCATE TABLE t RESTART IDENTITY") + db.run("ROLLBACK") + assertEquals(listOf(listOf(int(1)), listOf(int(2))), db.select("SELECT id FROM t ORDER BY id").rows) + val error = assertFailsWith { db.run("INSERT INTO t (label) VALUES ('c')") } + assertTrue(error.message!!.contains("id"), error.message) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskIndexAccessPlannerTest.kt b/src/test/kotlin/kestrel/store/sql/DiskIndexAccessPlannerTest.kt new file mode 100644 index 0000000..857989e --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskIndexAccessPlannerTest.kt @@ -0,0 +1,411 @@ +package kestrel.store.sql + +import kestrel.catalog.Column +import kestrel.catalog.TableSchema +import kestrel.sql.ast.BinaryExpression +import kestrel.sql.ast.BinaryOperator +import kestrel.sql.ast.ColumnExpression +import kestrel.sql.ast.Expression +import kestrel.sql.ast.LiteralExpression +import kestrel.storage.ColumnType +import kestrel.storage.Value +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNull + +/** + * Pure unit tests for [DiskIndexAccessPlanner]'s decision logic, independent + * of any real [DiskTable]/disk I/O - hand-built [TableSchema]/[Expression] + * ASTs (the same shapes [Parser][kestrel.sql.parser.Parser] would produce) + * are enough to exercise every branch. End-to-end proof that the chosen plan + * is actually *used* (not just correctly chosen) lives in [DiskExecutorTest], + * which asserts on [DiskTable]'s access counters. + */ +class DiskIndexAccessPlannerTest { + private val planner = DiskIndexAccessPlanner() + + private val schema = TableSchema( + "users", + listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("age", ColumnType.Integer, notNull = false, primaryKey = false), + Column("name", ColumnType.Text, notNull = false, primaryKey = false) + ) + ) + + private fun col(name: String, qualifier: String? = null) = ColumnExpression(name, qualifier) + private fun int(value: Long) = LiteralExpression(Value.IntegerValue(value)) + private fun text(value: String) = LiteralExpression(Value.TextValue(value)) + private fun cmp(left: Expression, operator: BinaryOperator, right: Expression) = BinaryExpression(operator, left, right) + private fun and(left: Expression, right: Expression) = BinaryExpression(BinaryOperator.And, left, right) + + @Test + fun aNullPredicateIsAFullScan() { + assertEquals(DiskAccessPlan.FullScan, planner.plan(schema, emptyList(), null)) + } + + @Test + fun equalityOnThePrimaryKeyIsAPointLookup() { + val plan = planner.plan(schema, emptyList(), cmp(col("id"), BinaryOperator.Equal, int(5))) + val point = assertIs(plan) + assertEquals(Value.IntegerValue(5), point.key) + } + + @Test + fun equalityWithTheLiteralOnTheLeftIsStillAPointLookup() { + val plan = planner.plan(schema, emptyList(), cmp(int(5), BinaryOperator.Equal, col("id"))) + val point = assertIs(plan) + assertEquals(Value.IntegerValue(5), point.key) + } + + @Test + fun aSingleSidedRangeOnThePrimaryKeyIsAPrimaryKeyRange() { + val plan = planner.plan(schema, emptyList(), cmp(col("id"), BinaryOperator.Greater, int(5))) + val range = assertIs(plan) + assertEquals(Value.IntegerValue(5), range.lower) + assertEquals(false, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun aTwoSidedRangeOnThePrimaryKeyCombinesBothBounds() { + val predicate = and( + cmp(col("id"), BinaryOperator.GreaterEqual, int(2)), + cmp(col("id"), BinaryOperator.LessEqual, int(8)) + ) + val range = assertIs(planner.plan(schema, emptyList(), predicate)) + assertEquals(Value.IntegerValue(2), range.lower) + assertEquals(true, range.lowerInclusive) + assertEquals(Value.IntegerValue(8), range.upper) + assertEquals(true, range.upperInclusive) + } + + @Test + fun aRangeSideWithTheLiteralOnTheLeftIsFlippedCorrectly() { + // "5 < id" means id > 5, not id < 5. + val plan = planner.plan(schema, emptyList(), cmp(int(5), BinaryOperator.Less, col("id"))) + val range = assertIs(plan) + assertEquals(Value.IntegerValue(5), range.lower) + assertEquals(false, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun equalityOnAnUnindexedColumnFallsBackToAFullScan() { + assertEquals(DiskAccessPlan.FullScan, planner.plan(schema, emptyList(), cmp(col("name"), BinaryOperator.Equal, LiteralExpression(Value.TextValue("bob"))))) + } + + @Test + fun equalityOnAnIndexedNonPrimaryColumnUsesTheSecondaryIndex() { + // A live TransactionalIndex is not needed to prove this branch of the + // planner - only the (name, columnName) pairing it dispatches by. + val plan = planner.plan(schema, listOf(secondaryHandleStub("idx_age", "age")), cmp(col("age"), BinaryOperator.Equal, int(30))) + val secondary = assertIs(plan) + assertEquals("idx_age", secondary.indexName) + assertEquals(listOf(Value.IntegerValue(30)), secondary.keys) + } + + @Test + fun aPrimaryKeyEqualityIsPreferredOverASecondaryIndexEqualityInTheSamePredicate() { + val predicate = and(cmp(col("id"), BinaryOperator.Equal, int(2)), cmp(col("age"), BinaryOperator.Equal, int(999))) + val plan = planner.plan(schema, listOf(secondaryHandleStub("idx_age", "age")), predicate) + val point = assertIs(plan) + assertEquals(Value.IntegerValue(2), point.key) + } + + @Test + fun anOrPredicateIsNotPushedDownAndFallsBackToAFullScan() { + // conjuncts() only ever splits on AND; an OR clause is one opaque conjunct that + // asColumnPredicate cannot turn into a single column/operator/literal triple. + val predicate = BinaryExpression(BinaryOperator.Or, cmp(col("id"), BinaryOperator.Equal, int(1)), cmp(col("id"), BinaryOperator.Equal, int(2))) + assertEquals(DiskAccessPlan.FullScan, planner.plan(schema, emptyList(), predicate)) + } + + @Test + fun aQualifierMismatchAgainstTheGivenAliasIsRejected() { + val plan = planner.plan(schema, emptyList(), cmp(col("id", qualifier = "other"), BinaryOperator.Equal, int(5)), alias = "users") + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun aQualifiedColumnMatchesWhenTheAliasAgrees() { + val plan = planner.plan(schema, emptyList(), cmp(col("id", qualifier = "u"), BinaryOperator.Equal, int(5)), alias = "u") + assertIs(plan) + } + + @Test + fun aQualifiedColumnMatchesWhenNoAliasWasGiven() { + // UPDATE/DELETE statements have no FROM alias to compare against, so a caller + // passing alias = null (the default) intentionally ignores qualifiers entirely. + val plan = planner.plan(schema, emptyList(), cmp(col("id", qualifier = "whatever"), BinaryOperator.Equal, int(5))) + assertIs(plan) + } + + private val orderLineSchema = TableSchema( + "order_lines", + listOf( + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("line_no", ColumnType.Integer, notNull = true, primaryKey = false), + Column("sku", ColumnType.Text, notNull = false, primaryKey = false) + ), + compositePrimaryKey = listOf("order_id", "line_no") + ) + + @Test + fun equalityOnEveryCompositePrimaryKeyColumnIsAPointLookupInSchemaOrder() { + val predicate = and( + cmp(col("line_no"), BinaryOperator.Equal, int(2)), + cmp(col("order_id"), BinaryOperator.Equal, int(1)) + ) + val plan = planner.plan(orderLineSchema, emptyList(), predicate) + val point = assertIs(plan) + assertEquals(listOf(Value.IntegerValue(1), Value.IntegerValue(2)), point.keys) + } + + @Test + fun equalityOnOnlySomeCompositePrimaryKeyColumnsFallsBackToAFullScan() { + // A partial equality match with no range on the remaining column either cannot be + // turned into a point lookup (line_no is left entirely unbound) - see + // DiskAccessPlan.CompositePrimaryKeyRange's own documentation for the narrower case where + // a range on the remaining last column IS pushed down instead. + val plan = planner.plan(orderLineSchema, emptyList(), cmp(col("order_id"), BinaryOperator.Equal, int(1))) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun aRangeOnTheLastColumnOfACompositePrimaryKeyAfterAnEqualityPrefixIsPushedDown() { + val predicate = and( + cmp(col("order_id"), BinaryOperator.Equal, int(1)), + cmp(col("line_no"), BinaryOperator.Greater, int(2)) + ) + val plan = planner.plan(orderLineSchema, emptyList(), predicate) + val range = assertIs(plan) + assertEquals(listOf(Value.IntegerValue(1)), range.prefix) + assertEquals(Value.IntegerValue(2), range.lower) + assertEquals(false, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun aTwoSidedRangeOnTheLastColumnOfACompositePrimaryKeyCombinesBothBounds() { + val predicate = and( + cmp(col("order_id"), BinaryOperator.Equal, int(1)), + and( + cmp(col("line_no"), BinaryOperator.GreaterEqual, int(2)), + cmp(col("line_no"), BinaryOperator.LessEqual, int(8)) + ) + ) + val range = assertIs(planner.plan(orderLineSchema, emptyList(), predicate)) + assertEquals(listOf(Value.IntegerValue(1)), range.prefix) + assertEquals(Value.IntegerValue(2), range.lower) + assertEquals(true, range.lowerInclusive) + assertEquals(Value.IntegerValue(8), range.upper) + assertEquals(true, range.upperInclusive) + } + + @Test + fun aRangeOnTheFirstColumnOfACompositePrimaryKeyWithNoEqualityPrefixIsNotPushedDown() { + // order_id is not the LAST composite PRIMARY KEY column - see + // DiskAccessPlan.CompositePrimaryKeyRange's own documentation on why leaving line_no + // entirely unconstrained after a range on order_id is a deliberate scope cut. + val plan = planner.plan(orderLineSchema, emptyList(), cmp(col("order_id"), BinaryOperator.Greater, int(1))) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun anEqualityPrefixOnACompositePrimaryKeyIsPreferredOverARangeOnTheSameLastColumn() { + val predicate = and( + cmp(col("order_id"), BinaryOperator.Equal, int(1)), + and( + cmp(col("line_no"), BinaryOperator.Equal, int(2)), + cmp(col("line_no"), BinaryOperator.Greater, int(0)) + ) + ) + val point = assertIs(planner.plan(orderLineSchema, emptyList(), predicate)) + assertEquals(listOf(Value.IntegerValue(1), Value.IntegerValue(2)), point.keys) + } + + private val orderLineWithWarehouseSchema = TableSchema( + "order_lines3", + listOf( + Column("warehouse", ColumnType.Text, notNull = true, primaryKey = false), + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("line_no", ColumnType.Integer, notNull = true, primaryKey = false) + ), + compositePrimaryKey = listOf("warehouse", "order_id", "line_no") + ) + + @Test + fun aRangeOnTheLastColumnOfAThreeColumnCompositePrimaryKeyAfterATwoColumnEqualityPrefixIsPushedDown() { + val predicate = and( + cmp(col("warehouse"), BinaryOperator.Equal, text("west")), + and( + cmp(col("order_id"), BinaryOperator.Equal, int(1)), + cmp(col("line_no"), BinaryOperator.Less, int(10)) + ) + ) + val range = assertIs(planner.plan(orderLineWithWarehouseSchema, emptyList(), predicate)) + assertEquals(listOf(Value.TextValue("west"), Value.IntegerValue(1)), range.prefix) + assertNull(range.lower) + assertEquals(Value.IntegerValue(10), range.upper) + assertEquals(false, range.upperInclusive) + } + + @Test + fun aRangeOnAMiddleColumnOfAThreeColumnCompositePrimaryKeyLeavingTheLastColumnUnconstrainedIsNotPushedDown() { + // line_no would be left entirely unconstrained after a range on order_id - the same + // "must always fill every PRIMARY KEY column" restriction as the two-column schema's + // aRangeOnTheFirstColumnOfACompositePrimaryKeyWithNoEqualityPrefixIsNotPushedDown, just one + // column deeper. + val predicate = and( + cmp(col("warehouse"), BinaryOperator.Equal, text("west")), + cmp(col("order_id"), BinaryOperator.Greater, int(1)) + ) + val plan = planner.plan(orderLineWithWarehouseSchema, emptyList(), predicate) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun comparingToANullLiteralIsNeverPushedDown() { + // "id = NULL" is never true under three-valued SQL logic and must not be + // treated as an equality lookup for NULL (which the encoding cannot represent). + val plan = planner.plan(schema, emptyList(), cmp(col("id"), BinaryOperator.Equal, LiteralExpression(Value.NullValue))) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + /** + * A real [SecondaryIndexHandle] backed by a genuine, temp-file [DiskCatalog]. + * The planner only ever reads a handle's `name`/`columnName` - never its + * live [kestrel.store.engine.TransactionalIndex] - but that index has no + * lightweight fake (its constructor is private and wired to a real + * [kestrel.store.page.PageStore]/[kestrel.store.txn.TransactionManager]), + * so real disk infrastructure is the simplest way to obtain a valid one. + */ + private fun secondaryHandleStub(name: String, columnName: String): SecondaryIndexHandle { + val dataFile = Files.createTempFile("kestrel-planner-test-data", ".db").toFile().also { it.deleteOnExit() } + val walFile = Files.createTempFile("kestrel-planner-test-wal", ".db").toFile().also { it.deleteOnExit() } + val catalog = DiskCatalog.open(dataFile, walFile) + catalog.createTable("stub", schema.columns) + return catalog.createIndex("stub", name, columnName) + } + + /** Composite-column counterpart of [secondaryHandleStub], over [orderLineSchema]'s columns. */ + private fun compositeSecondaryHandleStub(name: String, columnNames: List): SecondaryIndexHandle { + val dataFile = Files.createTempFile("kestrel-planner-test-data", ".db").toFile().also { it.deleteOnExit() } + val walFile = Files.createTempFile("kestrel-planner-test-wal", ".db").toFile().also { it.deleteOnExit() } + val catalog = DiskCatalog.open(dataFile, walFile) + catalog.createTable("stub", orderLineSchema.columns) + return catalog.createIndex("stub", name, columnNames) + } + + @Test + fun equalityOnEveryColumnOfACompositeSecondaryIndexUsesIt() { + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val predicate = and( + cmp(col("sku"), BinaryOperator.Equal, text("widget")), + cmp(col("order_id"), BinaryOperator.Equal, int(1)) + ) + val plan = planner.plan(orderLineSchema, listOf(handle), predicate) + val secondary = assertIs(plan) + assertEquals("idx_order_sku", secondary.indexName) + assertEquals(listOf(Value.IntegerValue(1), Value.TextValue("widget")), secondary.keys) + } + + @Test + fun equalityOnOnlySomeColumnsOfACompositeSecondaryIndexFallsBackToAFullScan() { + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val plan = planner.plan(orderLineSchema, listOf(handle), cmp(col("order_id"), BinaryOperator.Equal, int(1))) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun aSingleSidedRangeOnASingleColumnSecondaryIndexIsASecondaryRange() { + val handle = secondaryHandleStub("idx_age", "age") + val plan = planner.plan(schema, listOf(handle), cmp(col("age"), BinaryOperator.GreaterEqual, int(30))) + val range = assertIs(plan) + assertEquals("idx_age", range.indexName) + assertEquals(emptyList(), range.prefix) + assertEquals(Value.IntegerValue(30), range.lower) + assertEquals(true, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun aTwoSidedRangeOnASingleColumnSecondaryIndexCombinesBothBounds() { + val handle = secondaryHandleStub("idx_age", "age") + val predicate = and( + cmp(col("age"), BinaryOperator.Greater, int(20)), + cmp(col("age"), BinaryOperator.Less, int(40)) + ) + val plan = planner.plan(schema, listOf(handle), predicate) + val range = assertIs(plan) + assertEquals("idx_age", range.indexName) + assertEquals(emptyList(), range.prefix) + assertEquals(Value.IntegerValue(20), range.lower) + assertEquals(false, range.lowerInclusive) + assertEquals(Value.IntegerValue(40), range.upper) + assertEquals(false, range.upperInclusive) + } + + @Test + fun anEqualityOnASecondaryIndexIsPreferredOverARangeOnTheSameColumn() { + // A range predicate alone would push down to SecondaryRange, but a stricter equality + // conjunct on the very same column is always more selective and must win. + val handle = secondaryHandleStub("idx_age", "age") + val predicate = and(cmp(col("age"), BinaryOperator.Equal, int(30)), cmp(col("age"), BinaryOperator.Greater, int(20))) + val plan = planner.plan(schema, listOf(handle), predicate) + val equality = assertIs(plan) + assertEquals(listOf(Value.IntegerValue(30)), equality.keys) + } + + @Test + fun aRangeOnTheFirstColumnOfACompositeSecondaryIndexWithNoPrefixIsPushedDown() { + // The composite index is sorted by order_id first, so a bare range on it (no equality + // on any column at all) is just as pushable as the single-column case above - an empty + // prefix, mirroring kestrel.engine.IndexAccessPlanner's own "even an empty run" case. + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val plan = planner.plan(orderLineSchema, listOf(handle), cmp(col("order_id"), BinaryOperator.Greater, int(1))) + val range = assertIs(plan) + assertEquals("idx_order_sku", range.indexName) + assertEquals(emptyList(), range.prefix) + assertEquals(Value.IntegerValue(1), range.lower) + assertEquals(false, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun aRangeOnTheSecondColumnOfACompositeSecondaryIndexAfterAnEqualityPrefixIsPushedDown() { + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val predicate = and(cmp(col("order_id"), BinaryOperator.Equal, int(1)), cmp(col("sku"), BinaryOperator.GreaterEqual, text("m"))) + val plan = planner.plan(orderLineSchema, listOf(handle), predicate) + val range = assertIs(plan) + assertEquals("idx_order_sku", range.indexName) + assertEquals(listOf(Value.IntegerValue(1)), range.prefix) + assertEquals(Value.TextValue("m"), range.lower) + assertEquals(true, range.lowerInclusive) + assertNull(range.upper) + } + + @Test + fun aRangeOnTheSecondColumnOfACompositeSecondaryIndexWithNoEqualityPrefixIsNotPushedDown() { + // The index is sorted by order_id first - a range on sku alone cannot narrow the scan + // without first pinning order_id down, the same restriction as an equality-only prefix. + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val plan = planner.plan(orderLineSchema, listOf(handle), cmp(col("sku"), BinaryOperator.Greater, text("m"))) + assertEquals(DiskAccessPlan.FullScan, plan) + } + + @Test + fun anEqualityPrefixOnACompositeSecondaryIndexIsPreferredOverARangeOnTheSameNextColumn() { + val handle = compositeSecondaryHandleStub("idx_order_sku", listOf("order_id", "sku")) + val predicate = and( + cmp(col("order_id"), BinaryOperator.Equal, int(1)), + and(cmp(col("sku"), BinaryOperator.Equal, text("widget")), cmp(col("sku"), BinaryOperator.Greater, text("a"))) + ) + val plan = planner.plan(orderLineSchema, listOf(handle), predicate) + val equality = assertIs(plan) + assertEquals(listOf(Value.IntegerValue(1), Value.TextValue("widget")), equality.keys) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskTableTest.kt b/src/test/kotlin/kestrel/store/sql/DiskTableTest.kt new file mode 100644 index 0000000..0086e0e --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskTableTest.kt @@ -0,0 +1,488 @@ +package kestrel.store.sql + +import kestrel.catalog.Column +import kestrel.catalog.UniqueConstraint +import kestrel.storage.ColumnType +import kestrel.storage.Row +import kestrel.storage.Value +import java.io.File +import java.math.BigDecimal +import java.nio.file.Files +import java.time.LocalDate +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class DiskTableTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun newCatalog(): DiskCatalog = DiskCatalog.open(tempFile("kestrel-sql-data"), tempFile("kestrel-sql-wal")) + + private val usersColumns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("name", ColumnType.Text, notNull = true, primaryKey = false), + Column("age", ColumnType.Integer, notNull = false, primaryKey = false) + ) + + private fun row(id: Long, name: String, age: Long?) = + Row(listOf(Value.IntegerValue(id), Value.TextValue(name), age?.let { Value.IntegerValue(it) } ?: Value.NullValue)) + + @Test + fun insertGetAndScanRoundTripValues() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + + val rid = table.insert(txn, row(1, "alice", 30)) + assertEquals(row(1, "alice", 30), table.get(txn, rid)) + assertEquals(listOf(rid to row(1, "alice", 30)), table.scan(txn)) + + catalog.commit(txn) + } + + @Test + fun duplicatePrimaryKeyIsRejectedAndLeavesTheIndexUntouched() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + table.insert(txn, row(1, "alice", 30)) + + assertFailsWith { table.insert(txn, row(1, "bob", 40)) } + + val (_, found) = table.findByPrimaryKey(txn, Value.IntegerValue(1))!! + assertEquals("alice", (found.values[1] as Value.TextValue).value) + catalog.commit(txn) + } + + @Test + fun rangeByPrimaryKeyReturnsOrderedRowsWithinBounds() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + for (id in 1..10L) table.insert(txn, row(id, "user$id", id)) + + val middle = table.rangeByPrimaryKey(txn, Value.IntegerValue(3), true, Value.IntegerValue(6), true) + assertEquals((3..6L).toList(), middle.map { (it.second.values[0] as Value.IntegerValue).value }) + + catalog.commit(txn) + } + + @Test + fun secondaryIndexFindsAllRowsSharingAValue() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_age", "age") + val txn = catalog.beginTransaction() + table.insert(txn, row(1, "alice", 30)) + table.insert(txn, row(2, "bob", 30)) + table.insert(txn, row(3, "carol", 40)) + + val thirtyYearOlds = table.findBySecondary(txn, "idx_age", Value.IntegerValue(30)) + assertEquals(setOf("alice", "bob"), thirtyYearOlds.map { (it.second.values[1] as Value.TextValue).value }.toSet()) + + catalog.commit(txn) + } + + @Test + fun createIndexBackfillsExistingRows() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val setup = catalog.beginTransaction() + table.insert(setup, row(1, "alice", 30)) + table.insert(setup, row(2, "bob", 30)) + catalog.commit(setup) + + catalog.createIndex("users", "idx_age", "age") + + val txn = catalog.beginTransaction() + val found = table.findBySecondary(txn, "idx_age", Value.IntegerValue(30)) + assertEquals(2, found.size) + catalog.commit(txn) + } + + @Test + fun updateMovesPrimaryAndSecondaryIndexEntriesToTheNewValues() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_age", "age") + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + + val newRid = table.update(txn, rid, row(2, "alice", 31)) + + assertNull(table.findByPrimaryKey(txn, Value.IntegerValue(1))) + assertEquals(row(2, "alice", 31), table.findByPrimaryKey(txn, Value.IntegerValue(2))!!.second) + assertTrue(table.findBySecondary(txn, "idx_age", Value.IntegerValue(30)).isEmpty()) + assertEquals(listOf(newRid), table.findBySecondary(txn, "idx_age", Value.IntegerValue(31)).map { it.first }) + + catalog.commit(txn) + } + + @Test + fun updateThatWouldCollideWithAnotherRowsPrimaryKeyIsRejected() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + table.insert(txn, row(2, "bob", 40)) + + assertFailsWith { table.update(txn, rid, row(2, "alice", 30)) } + catalog.commit(txn) + } + + @Test + fun deleteRemovesTheRowAndBothIndexEntries() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_age", "age") + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + + table.delete(txn, rid) + + assertTrue(table.scan(txn).isEmpty()) + assertNull(table.findByPrimaryKey(txn, Value.IntegerValue(1))) + assertTrue(table.findBySecondary(txn, "idx_age", Value.IntegerValue(30)).isEmpty()) + catalog.commit(txn) + } + + @Test + fun updateRelocatesARowThatNoLongerFitsInItsSlotAndIndexesFollowTheNewRecordId() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "a", 1)) + + val bigName = "x".repeat(3000) + val newRid = table.update(txn, rid, row(1, bigName, 1)) + + assertTrue(newRid != rid, "growing the row past its slot capacity should relocate it") + assertEquals(bigName, table.findByPrimaryKey(txn, Value.IntegerValue(1))!!.second.values[1].let { (it as Value.TextValue).value }) + catalog.commit(txn) + } + + @Test + fun abortingATransactionUndoesInsertsAndIndexEntries() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val committed = catalog.beginTransaction() + table.insert(committed, row(1, "alice", 30)) + catalog.commit(committed) + + val doomed = catalog.beginTransaction() + table.insert(doomed, row(2, "bob", 40)) + catalog.abort(doomed) + + val reader = catalog.beginTransaction() + assertEquals(listOf(1L), table.scan(reader).map { (it.second.values[0] as Value.IntegerValue).value }) + assertNull(table.findByPrimaryKey(reader, Value.IntegerValue(2))) + catalog.commit(reader) + } + + @Test + fun notNullConstraintIsEnforcedAtInsert() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + assertFailsWith { + table.insert(txn, Row(listOf(Value.IntegerValue(1), Value.NullValue, Value.NullValue))) + } + catalog.abort(txn) + } + + private val orderLineColumns = listOf( + Column("order_id", ColumnType.Integer, notNull = true, primaryKey = false), + Column("line_no", ColumnType.Integer, notNull = true, primaryKey = false), + Column("sku", ColumnType.Text, notNull = false, primaryKey = false) + ) + + private fun orderLine(orderId: Long, lineNo: Long, sku: String?) = + Row(listOf(Value.IntegerValue(orderId), Value.IntegerValue(lineNo), sku?.let { Value.TextValue(it) } ?: Value.NullValue)) + + @Test + fun compositePrimaryKeyAcceptsDistinctCombinationsAndRejectsDuplicates() { + val catalog = newCatalog() + val table = catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "line_no")) + val txn = catalog.beginTransaction() + table.insert(txn, orderLine(1, 1, "widget")) + table.insert(txn, orderLine(1, 2, "gadget")) + table.insert(txn, orderLine(2, 1, "widget")) + + assertFailsWith { table.insert(txn, orderLine(1, 1, "duplicate")) } + + assertEquals(3, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun compositePrimaryKeyPointLookupFindsTheExactRowByAllColumns() { + val catalog = newCatalog() + val table = catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "line_no")) + val txn = catalog.beginTransaction() + table.insert(txn, orderLine(1, 1, "widget")) + table.insert(txn, orderLine(1, 2, "gadget")) + + val found = table.findByPrimaryKey(txn, listOf(Value.IntegerValue(1), Value.IntegerValue(2)))!! + assertEquals("gadget", (found.second.values[2] as Value.TextValue).value) + assertEquals(null, table.findByPrimaryKey(txn, listOf(Value.IntegerValue(1), Value.IntegerValue(99)))) + catalog.commit(txn) + } + + @Test + fun compositePrimaryKeyUpdateAndDeleteMoveOrRemoveTheIndexEntry() { + val catalog = newCatalog() + val table = catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "line_no")) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, orderLine(1, 1, "widget")) + + val newRid = table.update(txn, rid, orderLine(1, 2, "widget")) + assertNull(table.findByPrimaryKey(txn, listOf(Value.IntegerValue(1), Value.IntegerValue(1)))) + assertEquals("widget", table.findByPrimaryKey(txn, listOf(Value.IntegerValue(1), Value.IntegerValue(2)))!!.second.values[2].let { (it as Value.TextValue).value }) + + table.delete(txn, newRid) + assertNull(table.findByPrimaryKey(txn, listOf(Value.IntegerValue(1), Value.IntegerValue(2)))) + assertTrue(table.scan(txn).isEmpty()) + catalog.commit(txn) + } + + @Test + fun compositePrimaryKeyUpdateThatCollidesWithAnotherRowIsRejected() { + val catalog = newCatalog() + val table = catalog.createTable("order_lines", orderLineColumns, compositePrimaryKey = listOf("order_id", "line_no")) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, orderLine(1, 1, "widget")) + table.insert(txn, orderLine(1, 2, "gadget")) + + assertFailsWith { table.update(txn, rid, orderLine(1, 2, "widget")) } + catalog.commit(txn) + } + + @Test + fun decimalAndDateColumnsRoundTripThroughTheFullDiskBackedStack() { + val catalog = newCatalog() + val columns = listOf( + Column("id", ColumnType.Integer, notNull = true, primaryKey = true), + Column("amount", ColumnType.Decimal, notNull = false, primaryKey = false, precision = 12, scale = 4), + Column("placed", ColumnType.Date, notNull = false, primaryKey = false) + ) + val table = catalog.createTable("orders", columns) + val txn = catalog.beginTransaction() + val amount = BigDecimal("123456.7890") + val placed = LocalDate.of(2024, 1, 15) + val inserted = Row(listOf(Value.IntegerValue(1), Value.DecimalValue(amount), Value.DateValue(placed))) + val rid = table.insert(txn, inserted) + assertEquals(inserted, table.get(txn, rid)) + catalog.commit(txn) + + val reader = catalog.beginTransaction() + assertEquals(inserted, table.get(reader, rid)) + catalog.commit(reader) + } + + // --- UNIQUE secondary index (CREATE UNIQUE INDEX): backed by the same B+Tree secondary + // index shape as a non-unique one, but insert/update reject a value collision. + + @Test + fun uniqueSecondaryIndexRejectsAnInsertSharingAnExistingNonNullValue() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_name", "name", unique = true) + val txn = catalog.beginTransaction() + table.insert(txn, row(1, "alice", 30)) + + assertFailsWith { table.insert(txn, row(2, "alice", 40)) } + + assertEquals(1, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueSecondaryIndexAllowsMultipleNullValues() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_age", "age", unique = true) + val txn = catalog.beginTransaction() + table.insert(txn, row(1, "alice", null)) + table.insert(txn, row(2, "bob", null)) + + assertEquals(2, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueSecondaryIndexRejectsAnUpdateThatWouldCollideWithAnotherRow() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_name", "name", unique = true) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + table.insert(txn, row(2, "bob", 40)) + + assertFailsWith { table.update(txn, rid, row(1, "bob", 30)) } + assertEquals("alice", table.get(txn, rid).values[1].let { (it as Value.TextValue).value }) + catalog.commit(txn) + } + + @Test + fun uniqueSecondaryIndexAllowsAnUpdateThatDoesNotChangeTheConstrainedColumn() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_name", "name", unique = true) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + + table.update(txn, rid, row(1, "alice", 31)) + + assertEquals(31L, (table.get(txn, rid).values[2] as Value.IntegerValue).value) + catalog.commit(txn) + } + + @Test + fun creatingAUniqueIndexOverPreExistingDuplicateValuesFailsAndAttachesNoIndex() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + val txn = catalog.beginTransaction() + table.insert(txn, row(1, "alice", 30)) + table.insert(txn, row(2, "alice", 40)) + catalog.commit(txn) + + assertFailsWith { catalog.createIndex("users", "idx_name", "name", unique = true) } + assertTrue(table.secondaryIndexes.isEmpty()) + } + + // --- composite (multi-column) secondary index: same B+Tree shape as a single-column one, + // keyed by KeyCodec.encodeKey over every listed column rather than a single value's encoding. + + @Test + fun compositeSecondaryIndexFindsRowsByEqualityOnEveryColumn() { + val catalog = newCatalog() + val table = catalog.createTable("users", usersColumns) + catalog.createIndex("users", "idx_both", listOf("name", "age")) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, row(1, "alice", 30)) + table.insert(txn, row(2, "alice", 40)) + + val found = table.findBySecondary(txn, "idx_both", listOf(Value.TextValue("alice"), Value.IntegerValue(30))) + + assertEquals(listOf(rid to row(1, "alice", 30)), found) + catalog.commit(txn) + } + + @Test + fun uniqueCompositeSecondaryIndexRejectsADuplicateColumnCombinationOnInsertButAllowsAPartialOverlap() { + val catalog = newCatalog() + val table = catalog.createTable("registrations", registrationColumns) + catalog.createIndex("registrations", "idx_reg", listOf("event_id", "user_id"), unique = true) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, 100)) + + assertFailsWith { table.insert(txn, registration(1, 100)) } + table.insert(txn, registration(1, 200)) + + assertEquals(2, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueCompositeSecondaryIndexAllowsMultipleRowsWithANullInEitherColumn() { + val catalog = newCatalog() + val table = catalog.createTable("registrations", registrationColumns) + catalog.createIndex("registrations", "idx_reg", listOf("event_id", "user_id"), unique = true) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, null)) + table.insert(txn, registration(1, null)) + table.insert(txn, registration(null, 100)) + table.insert(txn, registration(null, 100)) + + assertEquals(4, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueCompositeSecondaryIndexRejectsAnUpdateThatWouldCollideWithAnotherRowsCombination() { + val catalog = newCatalog() + val table = catalog.createTable("registrations", registrationColumns) + catalog.createIndex("registrations", "idx_reg", listOf("event_id", "user_id"), unique = true) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, 100)) + val rid = table.insert(txn, registration(1, 200)) + + assertFailsWith { table.update(txn, rid, registration(1, 100)) } + catalog.commit(txn) + } + + @Test + fun creatingAUniqueCompositeIndexOverPreExistingDuplicateCombinationsFailsAndAttachesNoIndex() { + val catalog = newCatalog() + val table = catalog.createTable("registrations", registrationColumns) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, 100)) + table.insert(txn, registration(1, 100)) + catalog.commit(txn) + + assertFailsWith { catalog.createIndex("registrations", "idx_reg", listOf("event_id", "user_id"), unique = true) } + assertTrue(table.secondaryIndexes.isEmpty()) + } + + // --- declarative UNIQUE constraints (TableSchema.uniqueConstraints): no backing index, + // enforced by DiskTable with a full heap scan, so a single constraint can span multiple columns. + + private val registrationColumns = listOf( + Column("event_id", ColumnType.Integer, notNull = false, primaryKey = false), + Column("user_id", ColumnType.Integer, notNull = false, primaryKey = false) + ) + + private fun registration(eventId: Long?, userId: Long?) = Row( + listOf(eventId?.let { Value.IntegerValue(it) } ?: Value.NullValue, userId?.let { Value.IntegerValue(it) } ?: Value.NullValue) + ) + + @Test + fun uniqueConstraintRejectsADuplicateColumnCombinationButAllowsPartialOverlap() { + val catalog = newCatalog() + val table = catalog.createTable( + "registrations", registrationColumns, + uniqueConstraints = listOf(UniqueConstraint(null, listOf("event_id", "user_id"))) + ) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, 100)) + + assertFailsWith { table.insert(txn, registration(1, 100)) } + table.insert(txn, registration(1, 200)) + table.insert(txn, registration(2, 100)) + + assertEquals(3, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueConstraintExemptsRowsWhereAnyConstrainedColumnIsNull() { + val catalog = newCatalog() + val table = catalog.createTable( + "registrations", registrationColumns, + uniqueConstraints = listOf(UniqueConstraint(null, listOf("event_id", "user_id"))) + ) + val txn = catalog.beginTransaction() + table.insert(txn, registration(1, null)) + table.insert(txn, registration(1, null)) + + assertEquals(2, table.scan(txn).size) + catalog.commit(txn) + } + + @Test + fun uniqueConstraintRejectsAnUpdateThatWouldCollideWithAnotherRow() { + val catalog = newCatalog() + val table = catalog.createTable( + "registrations", registrationColumns, + uniqueConstraints = listOf(UniqueConstraint(null, listOf("event_id", "user_id"))) + ) + val txn = catalog.beginTransaction() + val rid = table.insert(txn, registration(1, 100)) + table.insert(txn, registration(1, 200)) + + assertFailsWith { table.update(txn, rid, registration(1, 200)) } + catalog.commit(txn) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/DiskTransactionTest.kt b/src/test/kotlin/kestrel/store/sql/DiskTransactionTest.kt new file mode 100644 index 0000000..ff94f10 --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/DiskTransactionTest.kt @@ -0,0 +1,223 @@ +package kestrel.store.sql + +import kestrel.engine.BeginResult +import kestrel.engine.CommitResult +import kestrel.engine.RollbackResult +import kestrel.engine.SelectResult +import kestrel.engine.StatementResult +import kestrel.sql.ast.Statement +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import kestrel.storage.Value +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * Exercises `BEGIN`/`COMMIT`/`ROLLBACK` through [DiskExecutor], parsing real SQL text through the + * same [Lexer]/[Parser] the in-memory engine uses - see [DiskExecutorTest] for that general pattern. + * Unlike [kestrel.TransactionTest] (the in-memory engine's equivalent, backed by a whole-catalog + * snapshot/restore that also undoes DDL), an explicit transaction here is the storage engine's own + * WAL/2PL transaction shared across statements via [DiskExecutor.execute] until `COMMIT`/`ROLLBACK` + * (see [DiskExecutor.executeBegin]) - so it can only wrap DML/SELECT, survives a process restart once + * committed, and actually holds locks across the gap between two SQL statements, which + * [locksAcquiredByAnOpenTransactionBlockAConcurrentWriterUntilItEnds] proves end to end. + */ +class DiskTransactionTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun parse(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram() + + private fun int(value: Long) = Value.IntegerValue(value) + + private class Database(val dataFile: File, val walFile: File) { + var catalog = DiskCatalog.open(dataFile, walFile) + private set + var executor = DiskExecutor(catalog) + private set + + fun run(sql: String): List = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + fun select(sql: String): SelectResult = run(sql).single() as SelectResult + + /** A second executor sharing this database's still-open [catalog] - a second concurrent session. */ + fun secondExecutor(): DiskExecutor = DiskExecutor(catalog) + + /** Closes and reopens the catalog against the same files, simulating a fresh process. */ + fun reopen() { + catalog.close() + catalog = DiskCatalog.open(dataFile, walFile) + executor = DiskExecutor(catalog) + } + } + + private fun newDatabase(): Database = Database(tempFile("kestrel-disk-txn-data"), tempFile("kestrel-disk-txn-wal")) + + private fun Database.seedAccounts() { + run("CREATE TABLE accounts (id INTEGER PRIMARY KEY, balance INTEGER)") + run("INSERT INTO accounts VALUES (1, 100), (2, 200)") + } + + private fun Database.balanceOf(id: Long): Long = + (select("SELECT balance FROM accounts WHERE id = $id").rows.single().single() as Value.IntegerValue).value + + private fun Database.rowCount(sql: String): Int = select(sql).rows.size + + @Test + fun beginReturnsBeginResultAndCommitAppliesEveryStatementInBetween() { + val db = newDatabase() + db.seedAccounts() + + assertEquals(BeginResult, db.run("BEGIN").single()) + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("UPDATE accounts SET balance = 0 WHERE id = 1") + db.run("DELETE FROM accounts WHERE id = 2") + assertEquals(CommitResult, db.run("COMMIT").single()) + + assertEquals(2, db.rowCount("SELECT * FROM accounts")) + assertEquals(0L, db.balanceOf(1)) + } + + @Test + fun rollbackUndoesEveryStatementSinceBegin() { + val db = newDatabase() + db.seedAccounts() + + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("UPDATE accounts SET balance = 0 WHERE id = 1") + db.run("DELETE FROM accounts WHERE id = 2") + assertEquals(RollbackResult, db.run("ROLLBACK").single()) + + assertEquals(2, db.rowCount("SELECT * FROM accounts")) + assertEquals(100L, db.balanceOf(1)) + assertEquals(200L, db.balanceOf(2)) + } + + @Test + fun laterStatementsInTheSameTransactionSeeEarlierUncommittedWrites() { + val db = newDatabase() + db.seedAccounts() + + db.run("BEGIN") + db.run("UPDATE accounts SET balance = 999 WHERE id = 1") + assertEquals(999L, db.balanceOf(1)) + db.run("ROLLBACK") + } + + @Test + fun commitSurvivesClosingAndReopeningTheCatalog() { + val db = newDatabase() + db.seedAccounts() + + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("UPDATE accounts SET balance = 1000 WHERE id = 2") + db.run("COMMIT") + + db.reopen() + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + assertEquals(1000L, db.balanceOf(2)) + } + + @Test + fun commitWithoutBeginIsRejected() { + val db = newDatabase() + db.seedAccounts() + assertFailsWith { db.run("COMMIT") } + } + + @Test + fun rollbackWithoutBeginIsRejected() { + val db = newDatabase() + db.seedAccounts() + assertFailsWith { db.run("ROLLBACK") } + } + + @Test + fun nestedBeginIsRejected() { + val db = newDatabase() + db.seedAccounts() + db.run("BEGIN") + assertFailsWith { db.run("BEGIN") } + db.run("ROLLBACK") + } + + @Test + fun ddlInsideAnOpenTransactionIsRejectedButTheTransactionStaysUsable() { + val db = newDatabase() + db.seedAccounts() + + db.run("BEGIN") + db.run("INSERT INTO accounts VALUES (3, 300)") + assertFailsWith { db.run("CREATE TABLE scratch (a INTEGER)") } + assertFailsWith { db.run("CREATE INDEX idx ON accounts (balance)") } + assertFailsWith { db.run("ALTER TABLE accounts ADD COLUMN note TEXT") } + + // The rejected DDL must not have corrupted or silently closed the still-open transaction. + assertEquals(CommitResult, db.run("COMMIT").single()) + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + } + + @Test + fun everyStatementAutoCommitsImmediatelyOutsideAnExplicitTransaction() { + val db = newDatabase() + db.seedAccounts() + + db.run("INSERT INTO accounts VALUES (3, 300)") + db.run("UPDATE accounts SET balance = 0 WHERE id = 1") + + // Neither statement opened a BEGIN, so each must already be durable on its own - reopening + // the catalog (a fresh process against the same files) has to see both without a COMMIT. + db.reopen() + assertEquals(3, db.rowCount("SELECT * FROM accounts")) + assertEquals(0L, db.balanceOf(1)) + } + + @Test + fun inTransactionReflectsWhetherABeginIsCurrentlyOpen() { + val db = newDatabase() + db.seedAccounts() + assertTrue(!db.executor.inTransaction) + db.run("BEGIN") + assertTrue(db.executor.inTransaction) + db.run("COMMIT") + assertTrue(!db.executor.inTransaction) + } + + /** + * The core new guarantee this session adds: two SQL statements inside one `BEGIN` share a single + * disk transaction, so the exclusive lock taken by the first `UPDATE` (see + * [kestrel.store.engine.TransactionalHeap], locked on the table's header page) is still held while + * a second, concurrent session's `UPDATE` on the very same table blocks on it - across the gap + * between statement 1 and statement 2, not just for the duration of a single statement. Before + * this session's change, `DiskExecutor` had no notion of a transaction spanning more than one + * statement, so this scenario could never even arise: every statement committed (and released its + * locks) the instant it finished. + */ + @Test + fun locksAcquiredByAnOpenTransactionBlockAConcurrentWriterUntilItEnds() { + val db = newDatabase() + db.seedAccounts() + val other = db.secondExecutor() + + db.run("BEGIN") + db.run("UPDATE accounts SET balance = 999 WHERE id = 1") + + val blockedUpdateFinished = java.util.concurrent.atomic.AtomicBoolean(false) + val thread = Thread { + other.execute(parse("UPDATE accounts SET balance = balance + 1 WHERE id = 1").single()) + blockedUpdateFinished.set(true) + } + thread.start() + Thread.sleep(200) + assertTrue(!blockedUpdateFinished.get(), "a concurrent writer must block while the open transaction still holds its lock") + + db.run("COMMIT") + thread.join(2000) + assertTrue(blockedUpdateFinished.get(), "the concurrent writer must proceed once the transaction commits") + assertEquals(1000L, db.balanceOf(1)) + } +} diff --git a/src/test/kotlin/kestrel/store/sql/ProbeTest.kt b/src/test/kotlin/kestrel/store/sql/ProbeTest.kt new file mode 100644 index 0000000..ce536dc --- /dev/null +++ b/src/test/kotlin/kestrel/store/sql/ProbeTest.kt @@ -0,0 +1,25 @@ +package kestrel.store.sql + +import kestrel.sql.lexer.Lexer +import kestrel.sql.parser.Parser +import java.io.File +import java.nio.file.Files +import kotlin.test.Test + +class ProbeTest { + @Test + fun probe() { + val dataFile = Files.createTempFile("probe-data", ".db").toFile().also { it.deleteOnExit() } + val walFile = Files.createTempFile("probe-wal", ".db").toFile().also { it.deleteOnExit() } + val catalog = DiskCatalog.open(dataFile, walFile) + val executor = DiskExecutor(catalog) + fun run(sql: String) = Parser(Lexer(sql).tokenize()).parseProgram().map { executor.execute(it) } + run("CREATE TABLE t (amount DECIMAL(10,2) PRIMARY KEY)") + run("INSERT INTO t VALUES (1.00)") + val table = catalog.table("t") + val before = table.pointLookupCount + val result = run("SELECT * FROM t WHERE amount = 1") + println("RESULT: " + result) + println("pointLookupCount before=$before after=${table.pointLookupCount}") + } +} diff --git a/src/test/kotlin/kestrel/store/txn/LockManagerTest.kt b/src/test/kotlin/kestrel/store/txn/LockManagerTest.kt new file mode 100644 index 0000000..6cca4f1 --- /dev/null +++ b/src/test/kotlin/kestrel/store/txn/LockManagerTest.kt @@ -0,0 +1,117 @@ +package kestrel.store.txn + +import java.util.concurrent.TimeUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class LockManagerTest { + private fun waitUntil(timeoutMs: Long = 2000, predicate: () -> Boolean) { + val deadline = System.currentTimeMillis() + timeoutMs + while (!predicate()) { + if (System.currentTimeMillis() > deadline) throw AssertionError("condition never became true") + Thread.sleep(5) + } + } + + @Test + fun sharedLocksAreCompatibleAcrossTransactions() { + val locks = LockManager() + locks.acquireShared(1L, 100L) + locks.acquireShared(2L, 100L) + assertEquals(mapOf(1L to LockManager.LockMode.SHARED, 2L to LockManager.LockMode.SHARED), locks.holders(100L)) + } + + @Test + fun reacquiringTheSameOrWeakerModeIsANoop() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) + locks.acquireShared(1L, 100L) + assertEquals(mapOf(1L to LockManager.LockMode.EXCLUSIVE), locks.holders(100L)) + } + + @Test + fun upgradeFromSharedToExclusiveSucceedsWhenSoleHolder() { + val locks = LockManager() + locks.acquireShared(1L, 100L) + locks.acquireExclusive(1L, 100L) + assertEquals(mapOf(1L to LockManager.LockMode.EXCLUSIVE), locks.holders(100L)) + } + + @Test + fun exclusiveLockBlocksAnotherTransactionUntilReleased() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) + + val acquired = java.util.concurrent.atomic.AtomicBoolean(false) + val thread = Thread { + locks.acquireExclusive(2L, 100L) + acquired.set(true) + } + thread.start() + waitUntil { locks.waiters(100L).contains(2L) } + assertTrue(!acquired.get(), "txn 2 must not acquire while txn 1 holds the exclusive lock") + + locks.release(1L, 100L) + thread.join(2000) + assertTrue(acquired.get()) + assertEquals(mapOf(2L to LockManager.LockMode.EXCLUSIVE), locks.holders(100L)) + } + + @Test + fun releaseAllDropsEveryLockHeldByATransaction() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) + locks.acquireShared(1L, 200L) + locks.releaseAll(1L) + assertEquals(emptyMap(), locks.holders(100L)) + assertEquals(emptyMap(), locks.holders(200L)) + } + + @Test + fun circularWaitIsDetectedAndRejectedWithoutBlocking() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) // txn 1 holds resource A + locks.acquireExclusive(2L, 200L) // txn 2 holds resource B + + val blocked = java.util.concurrent.atomic.AtomicBoolean(false) + val thread = Thread { + locks.acquireExclusive(1L, 200L) // txn 1 wants B, held by txn 2: blocks + blocked.set(true) + } + thread.start() + waitUntil { locks.waiters(200L).contains(1L) } + + // txn 2 now wants A, held by txn 1, which is already waiting on txn 2: a cycle. + assertFailsWith { locks.acquireExclusive(2L, 100L) } + + // Unblock the background waiter so the test does not leak a thread. + locks.release(2L, 200L) + thread.join(2000) + assertTrue(blocked.get()) + } + + @Test + fun unrelatedWaitersDoNotFalselyTriggerDeadlock() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) + // txn 2 requests an unrelated resource txn 1 does not hold: no cycle, should not throw. + locks.acquireExclusive(2L, 200L) + } + + @Test + fun exclusiveWaiterEventuallyRunsToCompletionAfterHolderReleases() { + val locks = LockManager() + locks.acquireExclusive(1L, 100L) + val latch = java.util.concurrent.CountDownLatch(1) + val thread = Thread { + locks.acquireExclusive(2L, 100L) + latch.countDown() + } + thread.start() + waitUntil { locks.waiters(100L).contains(2L) } + locks.releaseAll(1L) + assertTrue(latch.await(2, TimeUnit.SECONDS)) + } +} diff --git a/src/test/kotlin/kestrel/store/txn/TransactionManagerTest.kt b/src/test/kotlin/kestrel/store/txn/TransactionManagerTest.kt new file mode 100644 index 0000000..bea82d2 --- /dev/null +++ b/src/test/kotlin/kestrel/store/txn/TransactionManagerTest.kt @@ -0,0 +1,88 @@ +package kestrel.store.txn + +import kestrel.store.page.DiskManager +import kestrel.store.page.Page +import java.io.File +import java.nio.file.Files +import kestrel.store.wal.WriteAheadLog +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class TransactionManagerTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".tmp").toFile().also { it.deleteOnExit() } + + private fun pageOf(fill: Int): ByteArray = ByteArray(Page.PAGE_SIZE) { fill.toByte() } + + private class Fixture { + val diskManager = DiskManager(Files.createTempFile("kestrel-txnmgr-data", ".db").toFile().also { it.deleteOnExit() }) + val wal = WriteAheadLog(Files.createTempFile("kestrel-txnmgr-wal", ".log").toFile().also { it.deleteOnExit() }) + val manager = TransactionManager(wal, diskManager) + } + + @Test + fun beginStartsATransactionInActiveStatus() { + val fixture = Fixture() + val txn = fixture.manager.begin() + assertEquals(TransactionManager.Status.ACTIVE, fixture.manager.statusOf(txn)) + } + + @Test + fun commitAppliesLoggedUpdateAndReleasesLocks() { + val fixture = Fixture() + val pageId = fixture.diskManager.allocatePage() + val before = pageOf(0) + fixture.diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val txn = fixture.manager.begin() + fixture.manager.acquireExclusive(txn, pageId) + val after = pageOf(3) + fixture.manager.logUpdate(txn, pageId, before, after) + fixture.diskManager.writePage(Page(pageId).also { System.arraycopy(after, 0, it.data, 0, after.size) }) + fixture.manager.commit(txn) + + assertEquals(TransactionManager.Status.COMMITTED, fixture.manager.statusOf(txn)) + assertContentEquals(after, fixture.diskManager.readPage(pageId).data) + + val otherTxn = fixture.manager.begin() + fixture.manager.acquireExclusive(otherTxn, pageId) // must not block: txn's locks were released on commit + } + + @Test + fun abortRollsBackLoggedUpdatesAndReleasesLocks() { + val fixture = Fixture() + val pageId = fixture.diskManager.allocatePage() + val before = pageOf(1) + fixture.diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val txn = fixture.manager.begin() + fixture.manager.acquireExclusive(txn, pageId) + val after = pageOf(4) + fixture.manager.logUpdate(txn, pageId, before, after) + fixture.diskManager.writePage(Page(pageId).also { System.arraycopy(after, 0, it.data, 0, after.size) }) + fixture.manager.abort(txn) + + assertEquals(TransactionManager.Status.ABORTED, fixture.manager.statusOf(txn)) + assertContentEquals(before, fixture.diskManager.readPage(pageId).data) + + val otherTxn = fixture.manager.begin() + fixture.manager.acquireExclusive(otherTxn, pageId) // must not block: aborted txn released its locks + } + + @Test + fun operationsOnAResolvedTransactionFail() { + val fixture = Fixture() + val txn = fixture.manager.begin() + fixture.manager.commit(txn) + assertFailsWith { fixture.manager.commit(txn) } + assertFailsWith { fixture.manager.abort(txn) } + assertFailsWith { fixture.manager.acquireExclusive(txn, 1L) } + } + + @Test + fun statusOfUnknownTransactionFails() { + val fixture = Fixture() + assertFailsWith { fixture.manager.statusOf(999L) } + } +} diff --git a/src/test/kotlin/kestrel/store/wal/LogRecordCodecTest.kt b/src/test/kotlin/kestrel/store/wal/LogRecordCodecTest.kt new file mode 100644 index 0000000..401fdba --- /dev/null +++ b/src/test/kotlin/kestrel/store/wal/LogRecordCodecTest.kt @@ -0,0 +1,75 @@ +package kestrel.store.wal + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class LogRecordCodecTest { + @Test + fun beginRoundTrips() { + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(LogRecord.Begin(7L))) + assertTrue(decoded is LogRecord.Begin) + assertEquals(7L, decoded.txnId) + } + + @Test + fun commitRoundTrips() { + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(LogRecord.Commit(9L))) + assertTrue(decoded is LogRecord.Commit) + assertEquals(9L, decoded.txnId) + } + + @Test + fun abortRoundTrips() { + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(LogRecord.Abort(3L))) + assertTrue(decoded is LogRecord.Abort) + assertEquals(3L, decoded.txnId) + } + + @Test + fun updateRoundTripsPageImagesAndChainPointer() { + val before = ByteArray(4096) { 1 } + val after = ByteArray(4096) { 2 } + val original = LogRecord.Update(txnId = 5L, prevLsn = 128L, pageId = 42L, before = before, after = after) + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(original)) + assertTrue(decoded is LogRecord.Update) + assertEquals(5L, decoded.txnId) + assertEquals(128L, decoded.prevLsn) + assertEquals(42L, decoded.pageId) + assertContentEquals(before, decoded.before) + assertContentEquals(after, decoded.after) + } + + @Test + fun compensationUpdateRoundTripsUndoImageAndSkipPointer() { + val undoImage = ByteArray(4096) { 3 } + val original = LogRecord.CompensationUpdate( + txnId = 11L, + prevLsn = 256L, + pageId = 17L, + undoImage = undoImage, + undoNextLsn = 64L + ) + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(original)) + assertTrue(decoded is LogRecord.CompensationUpdate) + assertEquals(11L, decoded.txnId) + assertEquals(256L, decoded.prevLsn) + assertEquals(17L, decoded.pageId) + assertContentEquals(undoImage, decoded.undoImage) + assertEquals(64L, decoded.undoNextLsn) + } + + @Test + fun checkpointRoundTripsActiveTxnList() { + val decoded = LogRecordCodec.decode(LogRecordCodec.encode(LogRecord.Checkpoint(listOf(1L, 2L, 3L)))) + assertTrue(decoded is LogRecord.Checkpoint) + assertEquals(listOf(1L, 2L, 3L), decoded.activeTxnIds) + } + + @Test + fun decodingUnknownTypeTagFails() { + assertFailsWith { LogRecordCodec.decode(byteArrayOf(99)) } + } +} diff --git a/src/test/kotlin/kestrel/store/wal/RecoveryManagerTest.kt b/src/test/kotlin/kestrel/store/wal/RecoveryManagerTest.kt new file mode 100644 index 0000000..2b09b98 --- /dev/null +++ b/src/test/kotlin/kestrel/store/wal/RecoveryManagerTest.kt @@ -0,0 +1,333 @@ +package kestrel.store.wal + +import kestrel.store.page.DiskManager +import kestrel.store.page.Page +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class RecoveryManagerTest { + private fun tempFile(prefix: String): File = Files.createTempFile(prefix, ".db").toFile().also { it.deleteOnExit() } + + private fun pageOf(fill: Int): ByteArray = ByteArray(Page.PAGE_SIZE) { fill.toByte() } + + @Test + fun redoRestoresACommittedWriteThatNeverReachedTheDataFile() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val before = pageOf(0) + diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val after = pageOf(7) + wal.logUpdate(txn, pageId, before, after) + wal.commit(txn) + // Simulate a crash: the buffer pool never flushed `after` to the data file, + // but the WAL frame for the update and the commit are durable. + + val result = RecoveryManager(wal, diskManager).recover() + + assertEquals(1, result.redoneUpdates) + assertTrue(result.rolledBackTxns.isEmpty()) + assertContentEquals(after, diskManager.readPage(pageId).data) + } + + @Test + fun undoRestoresAPageWrittenByATransactionThatNeverCommitted() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val before = pageOf(1) + diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val after = pageOf(9) + wal.logUpdate(txn, pageId, before, after) + // Simulate a crash before commit, after the dirty page happened to reach disk + // under a steal buffer-pool policy. + diskManager.writePage(Page(pageId).also { System.arraycopy(after, 0, it.data, 0, after.size) }) + + val result = RecoveryManager(wal, diskManager).recover() + + assertEquals(listOf(txn), result.rolledBackTxns) + assertContentEquals(before, diskManager.readPage(pageId).data) + val abortSeen = wal.readFrom(0L).any { it.second is LogRecord.Abort && (it.second as LogRecord.Abort).txnId == txn } + assertTrue(abortSeen, "expected recovery to append an Abort record resolving the transaction") + } + + @Test + fun checkpointSkipsReplayingUpdatesFromBeforeIt() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageA = diskManager.allocatePage() + val pageB = diskManager.allocatePage() + diskManager.writePage(Page(pageA).also { System.arraycopy(pageOf(0), 0, it.data, 0, Page.PAGE_SIZE) }) + diskManager.writePage(Page(pageB).also { System.arraycopy(pageOf(0), 0, it.data, 0, Page.PAGE_SIZE) }) + + val wal = WriteAheadLog(walFile) + val firstTxn = wal.beginTransaction() + wal.logUpdate(firstTxn, pageA, pageOf(0), pageOf(5)) + wal.commit(firstTxn) + wal.checkpoint() + + val secondTxn = wal.beginTransaction() + wal.logUpdate(secondTxn, pageB, pageOf(0), pageOf(6)) + wal.commit(secondTxn) + + val result = RecoveryManager(wal, diskManager).recover() + + assertEquals(1, result.redoneUpdates, "only the update after the checkpoint should be replayed") + } + + @Test + fun liveRollbackUndoesUpdatesWithoutACrash() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val before = pageOf(2) + diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val after = pageOf(8) + wal.logUpdate(txn, pageId, before, after) + diskManager.writePage(Page(pageId).also { System.arraycopy(after, 0, it.data, 0, after.size) }) + + RecoveryManager(wal, diskManager).rollback(txn) + + assertContentEquals(before, diskManager.readPage(pageId).data) + assertEquals(WriteAheadLog.NO_LSN, wal.lastLsnOf(txn)) + } + + @Test + fun recoverDoesNotResurrectAWriteFromATransactionThatWasAbortedInAnEarlierSession() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val before = pageOf(3) + diskManager.writePage(Page(pageId).also { System.arraycopy(before, 0, it.data, 0, before.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val after = pageOf(4) + wal.logUpdate(txn, pageId, before, after) + RecoveryManager(wal, diskManager).rollback(txn) + // The transaction is now resolved (its Update is followed by an Abort record) + // and the data file already holds `before`, exactly as a live abort left it. + + // A fresh WriteAheadLog reader over the same file, as a new process would open. + val reopenedWal = WriteAheadLog(walFile) + val result = RecoveryManager(reopenedWal, diskManager).recover() + + assertEquals(0, result.redoneUpdates, "an already-aborted transaction's update must not be replayed") + assertTrue(result.rolledBackTxns.isEmpty()) + assertContentEquals(before, diskManager.readPage(pageId).data) + } + + @Test + fun rollingBackATransactionWithNoUpdatesJustMarksItAborted() { + val walFile = tempFile("kestrel-recovery-wal") + val dataFile = tempFile("kestrel-recovery-data") + val wal = WriteAheadLog(walFile) + val diskManager = DiskManager(dataFile) + val txn = wal.beginTransaction() + + RecoveryManager(wal, diskManager).rollback(txn) + + assertEquals(WriteAheadLog.NO_LSN, wal.lastLsnOf(txn)) + val abortSeen = wal.readFrom(0L).any { it.second is LogRecord.Abort && (it.second as LogRecord.Abort).txnId == txn } + assertTrue(abortSeen) + } + + @Test + fun rollingBackAnUnknownTransactionFails() { + val walFile = tempFile("kestrel-recovery-wal") + val dataFile = tempFile("kestrel-recovery-data") + val wal = WriteAheadLog(walFile) + val diskManager = DiskManager(dataFile) + assertFailsWith { RecoveryManager(wal, diskManager).rollback(12345L) } + } + + @Test + fun rollbackToUndoesOnlyUpdatesAfterTheSavepoint() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val v0 = pageOf(0) + diskManager.writePage(Page(pageId).also { System.arraycopy(v0, 0, it.data, 0, v0.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val v1 = pageOf(1) + wal.logUpdate(txn, pageId, v0, v1) + diskManager.writePage(Page(pageId).also { System.arraycopy(v1, 0, it.data, 0, v1.size) }) + val savepointLsn = wal.lastLsnOf(txn) + val v2 = pageOf(2) + wal.logUpdate(txn, pageId, v1, v2) + diskManager.writePage(Page(pageId).also { System.arraycopy(v2, 0, it.data, 0, v2.size) }) + + RecoveryManager(wal, diskManager).rollbackTo(txn, savepointLsn) + + assertContentEquals(v1, diskManager.readPage(pageId).data, "only the write after the savepoint should be undone") + val newTail = wal.lastLsnOf(txn) + val clr = wal.readAt(newTail) + assertTrue(clr is LogRecord.CompensationUpdate, "the transaction's chain tail should land on a new compensation record, not the undone update") + assertEquals(savepointLsn, clr.undoNextLsn, "the compensation record must skip straight back to the savepoint on a later undo walk") + } + + @Test + fun rollbackToLeavesTheTransactionActiveForFurtherWritesOrAFullResolution() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val v0 = pageOf(0) + diskManager.writePage(Page(pageId).also { System.arraycopy(v0, 0, it.data, 0, v0.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val savepointLsn = wal.lastLsnOf(txn) + val v1 = pageOf(1) + wal.logUpdate(txn, pageId, v0, v1) + diskManager.writePage(Page(pageId).also { System.arraycopy(v1, 0, it.data, 0, v1.size) }) + + val recovery = RecoveryManager(wal, diskManager) + recovery.rollbackTo(txn, savepointLsn) + assertContentEquals(v0, diskManager.readPage(pageId).data) + + // The savepoint itself is still usable, and the transaction can still be written to and resolved. + val v3 = pageOf(3) + wal.logUpdate(txn, pageId, v0, v3) + diskManager.writePage(Page(pageId).also { System.arraycopy(v3, 0, it.data, 0, v3.size) }) + recovery.rollback(txn) + + assertContentEquals(v0, diskManager.readPage(pageId).data, "a full rollback after a savepoint rollback must undo everything") + assertEquals(WriteAheadLog.NO_LSN, wal.lastLsnOf(txn)) + } + + /** + * The whole reason [LogRecord.CompensationUpdate] exists: a full rollback that walks back through + * a range already undone by an earlier [RecoveryManager.rollbackTo] must skip it via + * [LogRecord.CompensationUpdate.undoNextLsn] rather than re-undoing it, or it would misapply a + * stale before-image to a page a later write already changed again. + */ + @Test + fun fullRollbackAfterAPartialRollbackDoesNotMisapplyAnAlreadyUndoneBeforeImage() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageA = diskManager.allocatePage() + val pageB = diskManager.allocatePage() + diskManager.writePage(Page(pageA).also { System.arraycopy(pageOf(0), 0, it.data, 0, Page.PAGE_SIZE) }) + diskManager.writePage(Page(pageB).also { System.arraycopy(pageOf(0), 0, it.data, 0, Page.PAGE_SIZE) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + // Write A, then take a savepoint, then write B (on a *different* page than A). + wal.logUpdate(txn, pageA, pageOf(0), pageOf(1)) + diskManager.writePage(Page(pageA).also { System.arraycopy(pageOf(1), 0, it.data, 0, Page.PAGE_SIZE) }) + val savepointLsn = wal.lastLsnOf(txn) + wal.logUpdate(txn, pageB, pageOf(0), pageOf(9)) + diskManager.writePage(Page(pageB).also { System.arraycopy(pageOf(9), 0, it.data, 0, Page.PAGE_SIZE) }) + + val recovery = RecoveryManager(wal, diskManager) + recovery.rollbackTo(txn, savepointLsn) + // Now write B again, with a *different* before/after pair than the undone write had. + wal.logUpdate(txn, pageB, pageOf(0), pageOf(20)) + diskManager.writePage(Page(pageB).also { System.arraycopy(pageOf(20), 0, it.data, 0, Page.PAGE_SIZE) }) + + recovery.rollback(txn) + + assertContentEquals(pageOf(0), diskManager.readPage(pageA).data) + assertContentEquals(pageOf(0), diskManager.readPage(pageB).data) + } + + @Test + fun rollbackToASavepointLsnNotOnTheTransactionsChainFails() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + diskManager.writePage(Page(pageId).also { System.arraycopy(pageOf(0), 0, it.data, 0, Page.PAGE_SIZE) }) + + val wal = WriteAheadLog(walFile) + val otherTxn = wal.beginTransaction() + val foreignLsn = wal.lastLsnOf(otherTxn) + val txn = wal.beginTransaction() + wal.logUpdate(txn, pageId, pageOf(0), pageOf(1)) + + assertFailsWith { RecoveryManager(wal, diskManager).rollbackTo(txn, foreignLsn) } + } + + /** + * A crash right after a live savepoint rollback (before the enclosing transaction resolves) must + * still end up fully rolled back on the next recovery pass, including the write from *before* the + * savepoint - proving the forward-redo-then-backward-undo dance around a + * [LogRecord.CompensationUpdate] reproduces the live session's state exactly, not just the parts + * that happen to already be correct on disk. + */ + @Test + fun recoverFullyUndoesATransactionThatCrashedRightAfterASavepointRollback() { + val dataFile = tempFile("kestrel-recovery-data") + val walFile = tempFile("kestrel-recovery-wal") + + val diskManager = DiskManager(dataFile) + val pageId = diskManager.allocatePage() + val v0 = pageOf(0) + diskManager.writePage(Page(pageId).also { System.arraycopy(v0, 0, it.data, 0, v0.size) }) + + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + val v1 = pageOf(1) + wal.logUpdate(txn, pageId, v0, v1) + diskManager.writePage(Page(pageId).also { System.arraycopy(v1, 0, it.data, 0, v1.size) }) + val savepointLsn = wal.lastLsnOf(txn) + val v2 = pageOf(2) + wal.logUpdate(txn, pageId, v1, v2) + // The buffer pool never flushed v2 to disk before the crash, unlike the earlier tests - + // the data file still shows v1 (from a flush before the savepoint), the WAL is the only + // record that v2 (and its subsequent undo) ever happened. + RecoveryManager(wal, diskManager).rollbackTo(txn, savepointLsn) + // No commit, no abort: simulate a crash right here, mid-transaction. + + val reopenedWal = WriteAheadLog(walFile) + val result = RecoveryManager(reopenedWal, diskManager).recover() + + assertEquals(listOf(txn), result.rolledBackTxns) + assertContentEquals(v0, diskManager.readPage(pageId).data, "the unresolved transaction must be fully undone, including its pre-savepoint write") + } + + @Test + fun redoOfAnUnallocatedPageFailsWithADescriptiveError() { + val walFile = tempFile("kestrel-recovery-wal") + val dataFile = tempFile("kestrel-recovery-data") + val diskManager = DiskManager(dataFile) + val wal = WriteAheadLog(walFile) + val txn = wal.beginTransaction() + wal.logUpdate(txn, 99L, pageOf(0), pageOf(1)) + wal.commit(txn) + + assertFailsWith { RecoveryManager(wal, diskManager).recover() } + } +} diff --git a/src/test/kotlin/kestrel/store/wal/WriteAheadLogTest.kt b/src/test/kotlin/kestrel/store/wal/WriteAheadLogTest.kt new file mode 100644 index 0000000..4efb686 --- /dev/null +++ b/src/test/kotlin/kestrel/store/wal/WriteAheadLogTest.kt @@ -0,0 +1,128 @@ +package kestrel.store.wal + +import java.io.File +import java.io.RandomAccessFile +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class WriteAheadLogTest { + private fun tempFile(): File = Files.createTempFile("kestrel-wal", ".log").toFile().also { it.deleteOnExit() } + + @Test + fun beginAssignsIncreasingTransactionIds() { + val wal = WriteAheadLog(tempFile()) + val a = wal.beginTransaction() + val b = wal.beginTransaction() + assertTrue(b > a) + wal.close() + } + + @Test + fun logUpdateChainsPrevLsnToTheTransactionsPriorRecord() { + val wal = WriteAheadLog(tempFile()) + val txn = wal.beginTransaction() + val beginLsn = wal.lastLsnOf(txn) + val firstUpdateLsn = wal.logUpdate(txn, 0L, ByteArray(1), ByteArray(1)) + val secondUpdateLsn = wal.logUpdate(txn, 0L, ByteArray(1), ByteArray(1)) + + val firstRecord = wal.readAt(firstUpdateLsn) as LogRecord.Update + val secondRecord = wal.readAt(secondUpdateLsn) as LogRecord.Update + assertEquals(beginLsn, firstRecord.prevLsn) + assertEquals(firstUpdateLsn, secondRecord.prevLsn) + wal.close() + } + + @Test + fun commitClearsActiveTrackingForTheTransaction() { + val wal = WriteAheadLog(tempFile()) + val txn = wal.beginTransaction() + wal.commit(txn) + assertEquals(WriteAheadLog.NO_LSN, wal.lastLsnOf(txn)) + wal.close() + } + + @Test + fun loggingUpdateForUnknownTransactionFails() { + val wal = WriteAheadLog(tempFile()) + assertFailsWith { wal.logUpdate(999L, 0L, ByteArray(1), ByteArray(1)) } + wal.close() + } + + @Test + fun checkpointRejectsActiveTransactions() { + val wal = WriteAheadLog(tempFile()) + wal.beginTransaction() + assertFailsWith { wal.checkpoint() } + wal.close() + } + + @Test + fun checkpointIsRecordedAsLastCheckpointLsn() { + val wal = WriteAheadLog(tempFile()) + val txn = wal.beginTransaction() + wal.commit(txn) + val checkpointLsn = wal.checkpoint() + assertEquals(checkpointLsn, wal.lastCheckpointLsn) + wal.close() + } + + @Test + fun readFromReturnsRecordsInLogOrder() { + val wal = WriteAheadLog(tempFile()) + val txn = wal.beginTransaction() + wal.logUpdate(txn, 1L, ByteArray(1), ByteArray(1)) + wal.commit(txn) + val records = wal.readFrom(0L).map { it.second } + assertTrue(records[0] is LogRecord.Begin) + assertTrue(records[1] is LogRecord.Update) + assertTrue(records[2] is LogRecord.Commit) + wal.close() + } + + @Test + fun reopeningResumesTransactionIdsPastThoseAlreadyLogged() { + val file = tempFile() + val first = WriteAheadLog(file) + val txn = first.beginTransaction() + first.commit(txn) + first.close() + + val reopened = WriteAheadLog(file) + val nextTxn = reopened.beginTransaction() + assertTrue(nextTxn > txn) + reopened.close() + } + + @Test + fun reopeningRecoversLastCheckpointLsn() { + val file = tempFile() + val first = WriteAheadLog(file) + val txn = first.beginTransaction() + first.commit(txn) + val checkpointLsn = first.checkpoint() + first.close() + + val reopened = WriteAheadLog(file) + assertEquals(checkpointLsn, reopened.lastCheckpointLsn) + reopened.close() + } + + @Test + fun corruptedFrameFailsChecksumValidation() { + val file = tempFile() + val wal = WriteAheadLog(file) + val txn = wal.beginTransaction() + wal.commit(txn) + wal.close() + + RandomAccessFile(file, "rw").use { raf -> + raf.seek(10) + raf.write(0xFF) + } + + assertFailsWith { WriteAheadLog(file) } + } +}