Console Application Slowing Down Over Time

I'm trying to import around 900K rows of data and map it to my new data model. My problem is that the console application I have build for this import functionality slows down over

Solution 1:

Why don't you use BulkCopy, this code will need amending for your particular tables & columns, but hopefully you get the idea:

using (var bulkCopy = new SqlBulkCopy(_DbContext.Database.Connection.ConnectionString, SqlBulkCopyOptions.TableLock))
            {
               bulkCopy.BulkCopyTimeout = 1200; // 20 minutes
                bulkCopy.BatchSize = 10000;
                bulkCopy.DestinationTableName = "TaxPayer";

                var table = new DataTable();
                var props = TypeDescriptor.GetProperties(typeof(TaxPayer))                                     
                    //Dirty hack to make sure we only have system data types                                      
                    //i.e. filter out the relationships/collections
                    .Cast<PropertyDescriptor>()
                    .Where(propertyInfo => propertyInfo.PropertyType.Namespace.Equals("System"))
                    .ToArray();
                foreach (var propertyInfo in props) 
                { 
                    bulkCopy.ColumnMappings.Add(propertyInfo.Name, propertyInfo.Name); 
                    table.Columns.Add(propertyInfo.Name, Nullable.GetUnderlyingType(propertyInfo.PropertyType) ?? propertyInfo.PropertyType); 
                }

                // need to amend next line to account for the correct number of columns
                var values = new object[props.Length + 1];
                foreach (var item in importDataList) 
                { 
                    for (var i = 0; i < values.Length - 1; i++) 
                    { 
                        ///TODO: Decide which columns need including
                        values[i] = props[i].GetValue(item);
                    }
                    table.Rows.Add(values);
                }

                bulkCopy.WriteToServer(table);
            }