Add native UPSERT/MERGE functionality with SQL Server, PostgreSQL, MySQL, Oracle, SQLite, DB2, Firebird, Snowflake optimization, INSERT IGNORE support, RawSql UpdateColumns, and comprehensive test coverage

src/FluentMigrator.Abstractions/Builders/Upsert/IUpsertDataOrInSchemaSyntax.cs

@@ -0,0 +1,35 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using FluentMigrator.Infrastructure;
+
+namespace FluentMigrator.Builders.Upsert
+{
+    /// <summary>
+    /// Specify the schema or define match criteria for the upsert operation
+    /// </summary>
+    public interface IUpsertDataOrInSchemaSyntax : IUpsertDataSyntax, IFluentSyntax
+    {
+        /// <summary>
+        /// Specify the schema for the table
+        /// </summary>
+        /// <param name="schemaName">The schema name</param>
+        /// <returns>The next step</returns>
+        IUpsertDataSyntax InSchema(string schemaName);
+    }
+}

src/FluentMigrator.Abstractions/Builders/Upsert/IUpsertDataSyntax.cs

@@ -0,0 +1,35 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using FluentMigrator.Infrastructure;
+
+namespace FluentMigrator.Builders.Upsert
+{
+    /// <summary>
+    /// Specify the match criteria and data for the upsert operation
+    /// </summary>
+    public interface IUpsertDataSyntax : IFluentSyntax
+    {
+        /// <summary>
+        /// Specify the columns to match on for determining if a row exists
+        /// </summary>
+        /// <param name="columnNames">The column names to match on</param>
+        /// <returns>The next step</returns>
+        IUpsertRowSyntax MatchOn(params string[] columnNames);
+    }
+}

src/FluentMigrator.Abstractions/Builders/Upsert/IUpsertExpressionRoot.cs

@@ -0,0 +1,35 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using FluentMigrator.Infrastructure;
+
+namespace FluentMigrator.Builders.Upsert
+{
+    /// <summary>
+    /// The root of the UPSERT expression
+    /// </summary>
+    public interface IUpsertExpressionRoot : IFluentSyntax
+    {
+        /// <summary>
+        /// Specify the table name to upsert data to
+        /// </summary>
+        /// <param name="tableName">The table name</param>
+        /// <returns>The next step</returns>
+        IUpsertDataOrInSchemaSyntax IntoTable(string tableName);
+    }
+}

src/FluentMigrator.Abstractions/Builders/Upsert/IUpsertRowSyntax.cs

@@ -0,0 +1,42 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using FluentMigrator.Infrastructure;
+
+namespace FluentMigrator.Builders.Upsert
+{
+    /// <summary>
+    /// Specify the rows to upsert and optionally the columns to update
+    /// </summary>
+    public interface IUpsertRowSyntax : IFluentSyntax
+    {
+        /// <summary>
+        /// Specify a single row to upsert
+        /// </summary>
+        /// <param name="dataAsAnonymousType">The row data as an anonymous object</param>
+        /// <returns>The next step</returns>
+        IUpsertUpdateColumnsSyntax Row(object dataAsAnonymousType);
+
+        /// <summary>
+        /// Specify multiple rows to upsert with the same structure
+        /// </summary>
+        /// <param name="dataAsAnonymousTypes">The rows data as anonymous objects</param>
+        /// <returns>The next step</returns>
+        IUpsertUpdateColumnsSyntax Rows(params object[] dataAsAnonymousTypes);
+    }
+}

src/FluentMigrator.Abstractions/Builders/Upsert/IUpsertUpdateColumnsSyntax.cs

@@ -0,0 +1,50 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using FluentMigrator.Infrastructure;
+
+namespace FluentMigrator.Builders.Upsert
+{
+    /// <summary>
+    /// Optionally specify which columns to update on match (terminal syntax)
+    /// </summary>
+    public interface IUpsertUpdateColumnsSyntax : IFluentSyntax
+    {
+        /// <summary>
+        /// Specify which columns to update when a match is found. 
+        /// If not called, all columns except match columns will be updated.
+        /// </summary>
+        /// <param name="columnNames">The column names to update on match</param>
+        void UpdateColumns(params string[] columnNames);
+
+        /// <summary>
+        /// Specify the exact values to use when updating matched rows.
+        /// This allows using RawSql expressions and overriding row data values for updates.
+        /// Cannot be used together with UpdateColumns (string array version) or IgnoreInsertIfExists.
+        /// </summary>
+        /// <param name="updateValues">Anonymous object containing column names and their update values, supports RawSql expressions</param>
+        void UpdateColumns(object updateValues);
+
+        /// <summary>
+        /// Configure the upsert to ignore the insert if the row already exists (INSERT IGNORE mode).
+        /// When enabled, existing rows are not updated, only new rows are inserted.
+        /// Cannot be used together with UpdateColumns.
+        /// </summary>
+        void IgnoreInsertIfExists();
+    }
+}

src/FluentMigrator.Abstractions/Expressions/UpsertDataExpression.cs

@@ -0,0 +1,160 @@
+#region License
+//
+// Copyright (c) 2007-2024, Fluent Migrator Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#endregion
+
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+
+using FluentMigrator.Infrastructure;
+using FluentMigrator.Model;
+
+namespace FluentMigrator.Expressions
+{
+    /// <summary>
+    /// Expression to upsert (insert or update) data using SQL MERGE or equivalent
+    /// </summary>
+    public class UpsertDataExpression : IMigrationExpression, ISupportAdditionalFeatures, ISchemaExpression, IValidatableObject
+    {
+        /// <inheritdoc />
+        public string SchemaName { get; set; }
+
+        /// <summary>
+        /// Gets or sets the table name
+        /// </summary>
+        [Required(ErrorMessageResourceType = typeof(ErrorMessages), ErrorMessageResourceName = nameof(ErrorMessages.TableNameCannotBeNullOrEmpty))]
+        public string TableName { get; set; }
+
+        /// <summary>
+        /// Gets the columns to match on for determining if a row exists (merge keys)
+        /// </summary>
+        public List<string> MatchColumns { get; } = new List<string>();
+
+        /// <summary>
+        /// Gets the rows to be upserted
+        /// </summary>
+        public List<InsertionDataDefinition> Rows { get; } = new List<InsertionDataDefinition>();
+
+        /// <summary>
+        /// Gets or sets the columns to update when a match is found (if null, all non-match columns are updated)
+        /// </summary>
+        public List<string> UpdateColumns { get; set; }
+
+        /// <summary>
+        /// Gets or sets the specific values to use when updating matched rows.
+        /// When specified, these values override the values from the row data for update operations.
+        /// This supports RawSql expressions for database-specific functions.
+        /// </summary>
+        public List<KeyValuePair<string, object>> UpdateValues { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether to ignore insert if the row already exists (INSERT IGNORE mode)
+        /// When true, existing rows are not updated, only new rows are inserted
+        /// </summary>
+        public bool IgnoreInsertIfExists { get; set; }
+
+        /// <inheritdoc />
+        public IDictionary<string, object> AdditionalFeatures { get; } = new Dictionary<string, object>();
+
+        /// <inheritdoc />
+        public void ExecuteWith(IMigrationProcessor processor)
+        {
+            processor.Process(this);
+        }
+
+        /// <inheritdoc />
+        public IMigrationExpression Reverse()
+        {
+            // For reversal, we create a delete expression that removes the inserted rows
+            var expression = new DeleteDataExpression
+            {
+                SchemaName = SchemaName,
+                TableName = TableName
+            };
+
+            // Create delete conditions based on the match columns
+            for (var index = Rows.Count - 1; index >= 0; index--)
+            {
+                var dataDefinition = new DeletionDataDefinition();
+                var row = Rows[index];
+
+                // Only include match columns in the delete condition
+                foreach (var matchColumn in MatchColumns)
+                {
+                    var matchingPair = row.FirstOrDefault(kvp => kvp.Key == matchColumn);
+                    if (!matchingPair.Equals(default(KeyValuePair<string, object>)))
+                    {
+                        dataDefinition.Add(matchingPair);
+                    }
+                }
+
+                if (dataDefinition.Count > 0)
+                {
+                    expression.Rows.Add(dataDefinition);
+                }
+            }
+
+            return expression;
+        }
+
+        /// <inheritdoc />
+        public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+        {
+            if (MatchColumns == null || MatchColumns.Count == 0)
+                yield return new ValidationResult("UpsertDataExpression must specify at least one match column");
+
+            if (Rows == null || Rows.Count == 0)
+                yield return new ValidationResult("UpsertDataExpression must specify at least one row to upsert");
+
+            // Validate that all rows contain the match columns
+            foreach (var row in Rows ?? Enumerable.Empty<InsertionDataDefinition>())
+            {
+                foreach (var matchColumn in MatchColumns ?? Enumerable.Empty<string>())
+                {
+                    if (!row.Any(kvp => kvp.Key == matchColumn))
+                        yield return new ValidationResult($"Row data must contain all match columns. Missing column: {matchColumn}");
+                }
+            }
+
+            // Validate that UpdateColumns (if specified) don't include match columns
+            if (UpdateColumns != null)
+            {
+                var invalidUpdateColumns = UpdateColumns.Intersect(MatchColumns ?? Enumerable.Empty<string>()).ToList();
+                if (invalidUpdateColumns.Count > 0)
+                    yield return new ValidationResult($"Update columns cannot include match columns: {string.Join(", ", invalidUpdateColumns)}");
+            }
+
+            // Validate that UpdateColumns and IgnoreInsertIfExists are not both specified
+            if (IgnoreInsertIfExists && UpdateColumns != null && UpdateColumns.Count > 0)
+            {
+                yield return new ValidationResult("UpdateColumns cannot be specified when IgnoreInsertIfExists is true");
+            }
+
+            // Validate that UpdateColumns and UpdateValues are not both specified
+            if (UpdateColumns != null && UpdateColumns.Count > 0 && UpdateValues != null && UpdateValues.Count > 0)
+            {
+                yield return new ValidationResult("UpdateColumns and UpdateValues cannot both be specified. Use either UpdateColumns (column names) or UpdateValues (column name/value pairs)");
+            }
+
+            // Validate that UpdateValues and IgnoreInsertIfExists are not both specified
+            if (IgnoreInsertIfExists && UpdateValues != null && UpdateValues.Count > 0)
+            {
+                yield return new ValidationResult("UpdateValues cannot be specified when IgnoreInsertIfExists is true");
+            }
+        }
+    }
+}

src/FluentMigrator.Abstractions/IMigrationGenerator.cs

@@ -169,6 +169,13 @@ public interface IMigrationGenerator
         /// <returns>The generated SQL</returns>
         string Generate(UpdateDataExpression expression);
 
+        /// <summary>
+        /// Generates an SQL statement to UPSERT (insert or update) data using MERGE or equivalent
+        /// </summary>
+        /// <param name="expression">The expression to create the SQL for</param>
+        /// <returns>The generated SQL</returns>
+        string Generate(UpsertDataExpression expression);
+
         /// <summary>
         /// Generates an SQL statement to move a table from one schema to another
         /// </summary>

src/FluentMigrator.Abstractions/IMigrationProcessor.cs

@@ -198,6 +198,12 @@ public interface IMigrationProcessor : IQuerySchema, IDisposable
         /// <param name="expression">The expression to execute</param>
         void Process(UpdateDataExpression expression);
 
+        /// <summary>
+        /// Executes an SQL expression to UPSERT (insert or update) data using MERGE or equivalent
+        /// </summary>
+        /// <param name="expression">The expression to execute</param>
+        void Process(UpsertDataExpression expression);
+
         /// <summary>
         /// Executes a <c>ALTER SCHEMA</c> SQL expression
         /// </summary>