I have an .NET CORE EF application setup as Database first that is supposed to take in a payload containing items to be persisted to my database. They payloads can be huge all containing multiple “Securities” and each “Security” has multiple “Positions/Transactions” to be persisted to the database. Sometimes the time taken to persist can grow exponentially and some of the payloads can take up to 20 minutes to persist. The ultimate goal here is to reduce the amount of time taken to run through this code and persist. The save function is not the issue here (afaik). I think the issue is the nested looping through the securities on the payload and all of their positions and the corresponding LINQ queries that are happening in HandlePostions/HandleTransactions/TryFindSecId functions. Any help cleaning/speeding this up would be welcomed and any help optimizing the queries would be great. Here is my calling function:
private async Task<AccPackBody> Persist(AccPackBody pack) { bool hasModifications = false, hasErrors = false; var secGuidsWithPositions = new List<Guid>(); using (var dbContext = new newportMasterContext(Configs.MasterDbConnString)) { dbContext.Database.SetCommandTimeout(300); foreach(var mappedSecurity in pack.Internal) { mappedSecurity.FirmId = pack.FirmId; SEC_IDENTIFIER existingSecurity = null; try { existingSecurity = await TryFindSecId(mappedSecurity, pack.FirmId.Value, dbContext); } catch(Exception ex) { await Logger.Send(LogType.Error, "Something failed while trying to clone security"); } if(existingSecurity == null) { await Logger.Send(pack, LogType.Error, $ "Missing security for: Identifier: {mappedSecurity.IdentifierName} Value: {mappedSecurity.IdentifierValue}, Firm{pack.FirmId}, Currency:{mappedSecurity.Currency}"); throw new LoggedException(); } else { mappedSecurity.SecurityGuid = existingSecurity.Security_UID; POSITION[] updatedPositions = null; if(mappedSecurity.Postitions != null && mappedSecurity.Postitions.Length > 0) { updatedPositions = HandlePositions(pack,dbContext, pack.State, pack.AccountUID.Value, mappedSecurity, pack.StatementDate.Value); secGuidsWithPositions.Add(mappedSecurity.SecurityGuid.Value); mappedSecurity.Postitions = updatedPositions != null && updatedPositions.Count() > 0 ? updatedPositions : new POSITION[0]; if(updatedPositions != null && updatedPositions.Count() > 0) hasModifications = true; } //EditTransaction TRANSACTION[] updatedTransactions = null; if(mappedSecurity.Transactions != null && mappedSecurity.Transactions.Length > 0) { mappedSecurity.Transactions = updatedTransactions != null && updatedTransactions.Count() > 0 ? updatedTransactions : new TRANSACTION[0]; if(updatedTransactions != null && updatedTransactions.Count() > 0) hasModifications = true; } } } if(!hasErrors && dbContext.ChangeTracker.HasChanges()) { try { await dbContext.SaveChangesAsync(); } catch(Exception ex) { await Logger.Send(pack,LogType.Warning, "Error persisting changes to db, rolling back: " + ex.ToString()); throw new LoggedException(); } } } return pack; }
Here are my three functions that the above method calls – these are where a lot of the slowdowns are occuring:
private async Task<SEC_IDENTIFIER> TryFindSecId(AccMappedSecurity mappedSecurity, long firmId, newportMasterContext dbContext, Guid? packUid = null) { var existingSecurities = dbContext.SEC_IDENTIFIER .Where(x => x.IdentifierName == mappedSecurity.IdentifierName && mappedSecurity.IdentifierValue == x.Identifier && x.FirmID == firmId && (string.IsNullOrEmpty(mappedSecurity.Currency) || x.SECURITY.Currency == mappedSecurity.Currency) && x.SECURITY.ChangeType != "D"); var existingSecurity = existingSecurities.FirstOrDefault(); if(existingSecurity == null) { var cloneResult = await ClonePostition.Clone( new ClonePositionRequest { IdentifierName = mappedSecurity.IdentifierName, IdentifierValue = mappedSecurity.IdentifierValue, NewCurrency = mappedSecurity.Currency, FirmId = firmId }, Logger.LocalLog ); if(cloneResult.Code != HttpStatusCode.InternalServerError) { existingSecurity = dbContext.SEC_IDENTIFIER .FirstOrDefault(x => x.IdentifierName == mappedSecurity.IdentifierName && mappedSecurity.IdentifierValue == x.Identifier && x.FirmID == firmId && (string.IsNullOrEmpty(mappedSecurity.Currency) || x.SECURITY.Currency == mappedSecurity.Currency)); } else { await Logger.Send(LogType.Error, "Internal server error on clone function - clone failed" + cloneResult.Code); } } else { } return existingSecurity; } private POSITION[] HandlePositions(AccPackBody reqPayload, newportMasterContext dbContext, PackState state, Guid accountGuid, AccMappedSecurity mappedSecurity, DateTime statementDate, Guid? packUid = null) { if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "Hit handle positions"); var handledPositions = new List<POSITION>(); int posCnt = mappedSecurity.Postitions.Length; if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "Deserializing mapped positions"); List<POSITION> deserializedPositions = mappedSecurity.Postitions .Select(x => JsonConvert.DeserializeObject<POSITION>(x.ToString(), Configs.JsonCircularRefFixConfig)) .OrderBy(x => x.DateStart) .ToList(); var updatedPosDetails = new List<POSITION_DETAIL>(); POSITION prevailingPosition = null; if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "Checking for prevailing position."); prevailingPosition = dbContext.POSITION .Include(x => x.POSITION_DETAIL) .Include(x => x.POSITION_DETAIL).ThenInclude(x => x.POSITION_BOND) .Include(x => x.POSITION_DETAIL).ThenInclude(x => x.POSITION_FX) .Include(x => x.POSITION_DETAIL).ThenInclude(x => x.POSITION_OPTION) .Include(x => x.POSITION_DETAIL).ThenInclude(x => x.POSITION_PRICE) .Where(x => x.Account_UID == accountGuid && x.Security_UID == mappedSecurity.SecurityGuid && x.SecurityFirmID == mappedSecurity.FirmId && (x.DateStart.Date <= statementDate) && (!x.DateEnd.HasValue || x.DateEnd.Value.Date < statementDate) && x.ChangeType != "D" && x.POSITION_DETAIL.Any(pd => pd.Genesis == "FEED" && pd.ChangeType != "D")) .OrderByDescending(x => x.DateStart) .ToList().FirstOrDefault(); if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "Looping incoming positions."); foreach(var incomingPosition in deserializedPositions) { if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, string.Format("Hit new incoming position:{0}", incomingPosition.Position_UID.ToString())); incomingPosition.Account_UID = accountGuid; incomingPosition.Security_UID = mappedSecurity.SecurityGuid.Value; incomingPosition.SecurityFirmID = mappedSecurity.FirmId.Value; incomingPosition.Position_UID = Guid.NewGuid(); if (prevailingPosition == null) { EntityEntry<POSITION> newPosition = null; dbContext.POSITION.Add(incomingPosition); newPosition = dbContext.Entry(incomingPosition); newPosition.CurrentValues.SetValues(new { ChangeType = "I" }); foreach(var posDetail in incomingPosition.POSITION_DETAIL.Where(x => x.ChangeType.ToUpper() != "D")) { EntityEntry<POSITION_DETAIL> newPositionDetail = null; posDetail.Detail_UID = Guid.NewGuid(); dbContext.POSITION_DETAIL.Add(posDetail); newPositionDetail = dbContext.Entry(posDetail); newPositionDetail.CurrentValues.SetValues(new { ChangeType = "I"}); updatedPosDetails.Add(posDetail); } handledPositions.Add(incomingPosition); if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "Inserted: " + incomingPosition.Position_UID.ToString()); } else { if(prevailingPosition.DateStart.Date == incomingPosition.DateStart.Date) { foreach(var posDetail in prevailingPosition.POSITION_DETAIL.Where(x => x.ChangeType.ToUpper() != "D")) { EntityEntry<POSITION_DETAIL> positionDetailToRemove = dbContext.Entry(posDetail); positionDetailToRemove.CurrentValues.SetValues(new { ChangeDate = DateTime.Now, ChangeType = "D" }); } EntityEntry<POSITION> positionToRemove = dbContext.Entry(prevailingPosition); positionToRemove.CurrentValues.SetValues(new { ChangeDate = DateTime.Now, ChangeType = "D" }); EntityEntry<POSITION> newPosition = null; dbContext.POSITION.Add(incomingPosition); newPosition = dbContext.Entry(incomingPosition); newPosition.CurrentValues.SetValues(new { ChangeType = "I" }); handledPositions.Add(incomingPosition); if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "inserted: " + incomingPosition.Position_UID.ToString()); foreach(var posDetail in incomingPosition.POSITION_DETAIL.Where(x => x.ChangeType.ToUpper() != "D")) { EntityEntry<POSITION_DETAIL> newPositionDetail = null; posDetail.Detail_UID = Guid.NewGuid(); dbContext.POSITION_DETAIL.Add(posDetail); newPositionDetail = dbContext.Entry(posDetail); newPositionDetail.CurrentValues.SetValues(new { ChangeType = "I"}); updatedPosDetails.Add(posDetail); } } else if(prevailingPosition.DateStart.Date < incomingPosition.DateStart.Date && PositionHasChange(prevailingPosition, incomingPosition, packUid)) { EntityEntry<POSITION> newPosition = null; dbContext.POSITION.Add(incomingPosition); newPosition = dbContext.Entry(incomingPosition); newPosition.CurrentValues.SetValues(new { ChangeType = "I" }); handledPositions.Add(incomingPosition); if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "inserted: " + incomingPosition.Position_UID.ToString()); foreach(var posDetail in incomingPosition.POSITION_DETAIL.Where(x => x.ChangeType.ToUpper() != "D")) { EntityEntry<POSITION_DETAIL> newPositionDetail = null; posDetail.Detail_UID = Guid.NewGuid(); dbContext.POSITION_DETAIL.Add(posDetail); newPositionDetail = dbContext.Entry(posDetail); newPositionDetail.CurrentValues.SetValues(new { ChangeType = "I"}); updatedPosDetails.Add(posDetail); } } else if(prevailingPosition.DateStart.Date < incomingPosition.DateStart.Date && !PositionHasChange(prevailingPosition, incomingPosition, packUid)) { } } } return handledPositions.ToArray(); } private TRANSACTION[] HandleTransctions(newportMasterContext dbContext, AccPackBody pack, AccMappedSecurity mappedSecurity, Guid? packUid = null) { object[] transactionsToProcess = new object[0]; var handledTransactions = new List<TRANSACTION>(); foreach(var trans in mappedSecurity.Transactions) { TRANSACTION handledTransaction = null, mappedTransaction = null; mappedTransaction = JsonConvert.DeserializeObject<TRANSACTION>(trans.ToString(), Configs.JsonCircularRefFixConfig); mappedTransaction.Security_UID = mappedSecurity.SecurityGuid.Value; //HACK if(string.IsNullOrWhiteSpace(mappedTransaction.BaseCurrency)) { mappedTransaction.BaseCurrency = "USD"; } //missing transId - drop out if(mappedTransaction.Reversal && string.IsNullOrWhiteSpace(mappedTransaction.TranID)) { Logger.Send(pack, LogType.Warning, "Reversal Transaction is missing a transacation Id"); throw new LoggedException(); } else { //if(mappedTransaction.Reversal && !string.IsNullOrWhiteSpace(mappedTransaction.TranID)) if(mappedTransaction.Reversal) { List<TRANSACTION> transactionsInDb = dbContext.TRANSACTION .Where(x => x.Account_UID == pack.AccountUID && x.Security_UID == mappedSecurity.SecurityGuid && x.SecurityFirmID == mappedSecurity.FirmId && x.TradeDate.Date == mappedTransaction.TradeDate.Date && x.TranID == mappedTransaction.TranID && !x.Reversal && x.ChangeType != "D") .ToList(); if(transactionsInDb.Count == 1) { mappedTransaction.Visible = false; TRANSACTION transactionInDb = transactionsInDb.FirstOrDefault(); transactionInDb.Visible = false; } } mappedTransaction.SecurityFirmID = mappedSecurity.FirmId.Value; mappedTransaction.Account_UID = pack.AccountUID.Value; if(mappedTransaction?.Comment?.Length >= 100) { mappedTransaction.Comment = mappedTransaction.Comment.Substring(0,96) + "..."; } mappedTransaction.Tran_UID = Guid.NewGuid(); var comparer = new TransactionComparer(); EntityEntry<TRANSACTION> transactionToInsert = null; dbContext.TRANSACTION.Add(mappedTransaction); transactionToInsert = dbContext.Entry(mappedTransaction); transactionToInsert.CurrentValues.SetValues(new { ChangeType = "I" }); if(useDiagnosticLogging) LocalLoggingUtilities.WriteDiagnosticLog(_logText, packUid.Value, "inserted: " + mappedTransaction.Tran_UID.ToString()); if(transactionToInsert != null) { transactionToInsert.CurrentValues.SetValues(new { ChangeDate = DateTime.Now }); } handledTransaction = transactionToInsert?.Entity; if(handledTransaction != null) { handledTransactions.Add(handledTransaction); } // } } } return handledTransactions.ToArray(); }
Sample payload:
{ "id": "74a7c410-5b00-4268-bb40-73cf099160ff", "custodianUid": "00000000-0000-0000-0000-000000000000", "datasourceUid": "e4b30a37-d0bf-49d4-bee9-a0cc19959e6d", "token": "@!!!@ec131ebc7537c0f8414447261af2a8ef28414815274ebd4ea0ba33caac231bc8", "statementDate": "2019-01-07T00:00:00Z", "accountUid": "0b251123-b9e5-4f59-be4d-4dde0f0eb23e", "firmId": 490, "state": 3, "exceedsSizeLimit": false, "external": [ { "identifiervalue": "85280", "identifierName": "AssetID", "currency": "USD", "firmId": null, "batchid": null, "securityGuid": null, "previousPositionGuid": "57b63479-9cc1-49b3-9743-8932b1764d1e", "previousPositionQty": 1000000, "previousPositionStartDate": "2019-01-01T00:00:00+00:00", "previousPositionEndDate": null, "transactions": [], "taxlots": [], "positions": [ "1/7/2019 12:00:00 AM|TAA0315||@!!!@ec131ebc7537c0f8414447261af2a8ef28414815274ebd4ea0ba33caac231bc8|Security|Structured Product|85280|0|||||||||0.0000|0.0000||0|0|||0.00000|||1000000.00000|100.00|100.00|84.18000|84.18000|6/8/2021 12:00:00 AM|0.00||0.00|0.00|||841800.00|841800.00|USD|||||0.0000||||||||" ], "customs": [] }, { "identifiervalue": "79428", "identifierName": "AssetID", "currency": "USD", "firmId": null, "batchid": null, "securityGuid": null, "previousPositionGuid": "238c6c1e-0815-4b74-8ac8-d695ddb5ff2e", "previousPositionQty": 1500000, "previousPositionStartDate": "2019-01-01T00:00:00+00:00", "previousPositionEndDate": null, "transactions": [], "taxlots": [], "positions": [ "1/7/2019 12:00:00 AM|TAA0315||@!!!@ec131ebc7537c0f8414447261af2a8ef28414815274ebd4ea0ba33caac231bc8|Security|Structured Product|79428|0|||||||||0.0000|0.0000||0|0|||0.00000|||1500000.00000|100.00|100.00|95.31000|95.31000|1/22/2021 12:00:00 AM|0.00||0.00|0.00|||1429650.00|1429650.00|USD|||||0.0000||||||||" ], "customs": [] } ], "internal": [ { "_name": null, "data": null, "identifiervalue": "85280", "identifierName": "AssetID", "currency": "USD", "previousPositionGuid": null, "previousPositionQty": null, "previousPositionStartDate": null, "previousPositionEndDate": null, "securityGuid": "c260432b-d169-474b-b142-83cd3e2b93e5", "firmId": 490, "positions": [ { "Position_UID": "1e6b6a39-697a-468b-a266-bd5388475ec6", "Account_UID": "0b251123-b9e5-4f59-be4d-4dde0f0eb23e", "Security_UID": "c260432b-d169-474b-b142-83cd3e2b93e5", "SecurityFirmID": 490, "DateStart": "2019-01-07T00:00:00+00:00", "DateEnd": null, "ChangeDate": "2019-04-16T15:40:39.7454448Z", "ChangeType": "I", "BatchID": 2019416151243529, "Account_U": null, "Security": null, "POSITION_DETAIL": [ { "Detail_UID": "39048278-f993-4229-b596-c681315c46b1", "Position_UID": "1e6b6a39-697a-468b-a266-bd5388475ec6", "Type": "BASE", "Quantity": 1000000, "Currency": "USD", "Value": 841800, "LastPrice": 84.18, "Cost": 100, "AccruedInterest": 0, "AccruedDividends": null, "Expenses": null, "UnrealizedGains": null, "RealizedGains": null, "Source_UID": "e4b30a37-d0bf-49d4-bee9-a0cc19959e6d", "Genesis": "Feed", "ChangeDate": "2019-04-16T15:40:39.8235573Z", "ChangeType": "I", "GenesisNavigation": null, "TypeNavigation": null, "POSITION_BOND": null, "POSITION_FX": null, "POSITION_OPTION": null, "POSITION_PRICE": null } ], "POSITION_IDENTIFIER": [], "POSITION_SUBSCRIPTION": [] } ], "transactions": null, "batchid": null }, { "_name": null, "data": null, "identifiervalue": "79428", "identifierName": "AssetID", "currency": "USD", "previousPositionGuid": null, "previousPositionQty": null, "previousPositionStartDate": null, "previousPositionEndDate": null, "securityGuid": "266a3cc3-9f76-4ca1-a1bb-593c55d86e95", "firmId": 490, "positions": [ { "Position_UID": "4d2e6c40-6878-4fab-96df-d62759e95917", "Account_UID": "0b251123-b9e5-4f59-be4d-4dde0f0eb23e", "Security_UID": "266a3cc3-9f76-4ca1-a1bb-593c55d86e95", "SecurityFirmID": 490, "DateStart": "2019-01-07T00:00:00+00:00", "DateEnd": null, "ChangeDate": "2019-04-16T15:40:39.870434Z", "ChangeType": "I", "BatchID": 2019416151243529, "Account_U": null, "Security": null, "POSITION_DETAIL": [ { "Detail_UID": "c894765d-482a-4649-87be-afc6c8092275", "Position_UID": "4d2e6c40-6878-4fab-96df-d62759e95917", "Type": "BASE", "Quantity": 1500000, "Currency": "USD", "Value": 1429650, "LastPrice": 95.31, "Cost": 100, "AccruedInterest": 0, "AccruedDividends": null, "Expenses": null, "UnrealizedGains": null, "RealizedGains": null, "Source_UID": "e4b30a37-d0bf-49d4-bee9-a0cc19959e6d", "Genesis": "Feed", "ChangeDate": "2019-04-16T15:40:39.9798084Z", "ChangeType": "I", "GenesisNavigation": null, "TypeNavigation": null, "POSITION_BOND": null, "POSITION_FX": null, "POSITION_OPTION": null, "POSITION_PRICE": null } ], "POSITION_IDENTIFIER": [], "POSITION_SUBSCRIPTION": [] } ], "transactions": null, "batchid": null } ], "batchid": 2019416151243529 }