B 16906 tac parser

pkg/models/transportation_accounting_code.go

@@ -10,6 +10,8 @@ import (
 )
 
 // TransportationAccountingCode model struct that represents transportation accounting codes
+// TODO: Update this model and internal use to reflect incoming TransportationAccountingCode model updates.
+// Don't forget to update the MakeDefaultTransportationAccountingCode function inside of the testdatagen package.
 type TransportationAccountingCode struct {
 	ID                 uuid.UUID         `json:"id" db:"id"`
 	TAC                string            `json:"tac" db:"tac"`
@@ -52,3 +54,9 @@ func (t *TransportationAccountingCode) Validate(_ *pop.Connection) (*validate.Er
 func (t TransportationAccountingCode) TableName() string {
 	return "transportation_accounting_codes"
 }
+
+func MapTransportationAccountingCodeFileRecordToInternalStruct(tacFileRecord TransportationAccountingCodeTrdmFileRecord) TransportationAccountingCode {
+	return TransportationAccountingCode{
+		TAC: tacFileRecord.TRNSPRTN_ACNT_CD,
+	}
+}

pkg/models/transportation_accounting_code_test.go

@@ -1,10 +1,83 @@
 package models_test
 
 import (
+	"reflect"
+	"testing"
+
 	"github.com/transcom/mymove/pkg/factory"
 	"github.com/transcom/mymove/pkg/models"
 )
 
+// This test will check if any field names are not being mapped properly. This test is not finalized.
+func TestTransportationAccountingCodeMapForUnusedFields(t *testing.T) {
+	t.Skip("Skipping TestTransportationAccountingCodeMapForUnusedFields until the fields and usecase has been finalized.")
+
+	// Example of TransportationAccountingCodeTrdmFileRecord
+	tacFileRecord := models.TransportationAccountingCodeTrdmFileRecord{
+		TRNSPRTN_ACNT_CD:        "4EVR",
+		TAC_SYS_ID:              "3080819",
+		LOA_SYS_ID:              "55555555",
+		TAC_FY_TXT:              "2023",
+		TAC_FN_BL_MOD_CD:        "W",
+		ORG_GRP_DFAS_CD:         "HS",
+		TAC_MVT_DSG_ID:          "",
+		TAC_TY_CD:               "O",
+		TAC_USE_CD:              "N",
+		TAC_MAJ_CLMT_ID:         "012345",
+		TAC_BILL_ACT_TXT:        "123456",
+		TAC_COST_CTR_NM:         "012345",
+		BUIC:                    "",
+		TAC_HIST_CD:             "",
+		TAC_STAT_CD:             "I",
+		TRNSPRTN_ACNT_TX:        "For the purpose of MacDill AFB transporting to Scott AFB",
+		TRNSPRTN_ACNT_BGN_DT:    "2022-10-01 00:00:00",
+		TRNSPRTN_ACNT_END_DT:    "2023-09-30 00:00:00",
+		DD_ACTVTY_ADRS_ID:       "A12345",
+		TAC_BLLD_ADD_FRST_LN_TX: "MacDill",
+		TAC_BLLD_ADD_SCND_LN_TX: "Second Address Line",
+		TAC_BLLD_ADD_THRD_LN_TX: "",
+		TAC_BLLD_ADD_FRTH_LN_TX: "TAMPA FL 33621",
+		TAC_FNCT_POC_NM:         "THISISNOTAREALPERSON@USCG.MIL",
+	}
+
+	mappedStruct := models.MapTransportationAccountingCodeFileRecordToInternalStruct(tacFileRecord)
+
+	reflectedMappedStruct := reflect.TypeOf(mappedStruct)
+	reflectedTacFileRecord := reflect.TypeOf(tacFileRecord)
+
+	// Iterate through each field in the tacRecord struct for the comparison
+	for i := 0; i < reflectedTacFileRecord.NumField(); i++ {
+		fieldName := reflectedTacFileRecord.Field(i).Name
+
+		// Check if this field exists in the reflectedMappedStruct
+		_, exists := reflectedMappedStruct.FieldByName(fieldName)
+
+		// Error if the field isn't found in the reflectedMappedStruct
+		if !exists {
+			t.Errorf("Field '%s' in TransportationAccountingCodeTrdmFileRecord is not used in MapTransportationAccountingCodeFileRecordToInternalStruct function", fieldName)
+		}
+	}
+}
+
+// This function will test the receival of a parsed TAC that has undergone the pipe delimited .txt file parser. It will test
+// that the received values correctly map to our internal TAC struct. For example, our Transporation Accounting Code is called
+// "TAC" in its struct, however when it is received in pipe delimited format it will be received as "TRNSPRTN_ACNT_CD".
+// This function makes sure it gets connected properly.
+func TestTransportationAccountingCodeMapToInternal(t *testing.T) {
+
+	tacFileRecord := models.TransportationAccountingCodeTrdmFileRecord{
+		TRNSPRTN_ACNT_CD: "4EVR",
+	}
+
+	mappedTacFileRecord := models.MapTransportationAccountingCodeFileRecordToInternalStruct(tacFileRecord)
+
+	// Check that the TRNSPRTN_ACNT_CD field in the original struct was correctly
+	// mapped to the TAC field in the resulting struct
+	if mappedTacFileRecord.TAC != tacFileRecord.TRNSPRTN_ACNT_CD {
+		t.Errorf("Expected TAC to be '%s', got '%s'", tacFileRecord.TRNSPRTN_ACNT_CD, mappedTacFileRecord.TAC)
+	}
+}
+
 func (suite *ModelSuite) Test_CanSaveValidTac() {
 	tac := models.TransportationAccountingCode{
 		TAC: "Tac1",

pkg/models/transportation_accounting_code_trdm_file.go

@@ -0,0 +1,32 @@
+package models
+
+// This file declares all of the column values that are received from TRDM in regard to the pipe delimited
+// Transportation_Accounting_Code (TAC) .txt file
+
+// This struct only applies to the received .txt file.
+type TransportationAccountingCodeTrdmFileRecord struct {
+	TAC_SYS_ID              string
+	LOA_SYS_ID              string
+	TRNSPRTN_ACNT_CD        string
+	TAC_FY_TXT              string
+	TAC_FN_BL_MOD_CD        string
+	ORG_GRP_DFAS_CD         string
+	TAC_MVT_DSG_ID          string
+	TAC_TY_CD               string
+	TAC_USE_CD              string
+	TAC_MAJ_CLMT_ID         string
+	TAC_BILL_ACT_TXT        string
+	TAC_COST_CTR_NM         string
+	BUIC                    string
+	TAC_HIST_CD             string
+	TAC_STAT_CD             string
+	TRNSPRTN_ACNT_TX        string
+	TRNSPRTN_ACNT_BGN_DT    string
+	TRNSPRTN_ACNT_END_DT    string
+	DD_ACTVTY_ADRS_ID       string
+	TAC_BLLD_ADD_FRST_LN_TX string
+	TAC_BLLD_ADD_SCND_LN_TX string
+	TAC_BLLD_ADD_THRD_LN_TX string
+	TAC_BLLD_ADD_FRTH_LN_TX string
+	TAC_FNCT_POC_NM         string
+}

pkg/parser/tac/fixtures/Transportation Account.txt

@@ -0,0 +1,6 @@
+Unclassified
+TAC_SYS_ID|LOA_SYS_ID|TRNSPRTN_ACNT_CD|TAC_FY_TXT|TAC_FN_BL_MOD_CD|ORG_GRP_DFAS_CD|TAC_MVT_DSG_ID|TAC_TY_CD|TAC_USE_CD|TAC_MAJ_CLMT_ID|TAC_BILL_ACT_TXT|TAC_COST_CTR_NM|BUIC|TAC_HIST_CD|TAC_STAT_CD|TRNSPRTN_ACNT_TX|TRNSPRTN_ACNT_BGN_DT|TRNSPRTN_ACNT_END_DT|DD_ACTVTY_ADRS_ID|TAC_BLLD_ADD_FRST_LN_TX|TAC_BLLD_ADD_SCND_LN_TX|TAC_BLLD_ADD_THRD_LN_TX|TAC_BLLD_ADD_FRTH_LN_TX|TAC_FNCT_POC_NM
+1234567884061|12345678|0003|2022|3|DF||O|O|USTC||G31M32|||I|FOR MOVEMENT TEST 1|2021-10-01 00:00:00|2022-09-30 00:00:00|F55555|FIRST LINE|SECOND LINE|THIRD LINE|FOURTH LINE|Contact Person Here
+3456789|34567890|ZZQE|2022|M|HS||O|N|018301|051800|018301|||I|FOR MOVEMENT TEST 2|2021-10-01 00:00:00|2022-09-30 00:00:00|Z55555|FIRST LINE|SECOND LINE||TAMPA FL 33621|NotARealPerson@USCG.MIL
+0000000|00000000|ZZQE|2022|W|HS||O|N|018301|051800|018301|||I|FOR MOVEMENT TEST 2|2021-10-01 00:00:00|2022-09-30 00:00:00|Z55555|FIRST LINE|SECOND LINE||TAMPA FL 33621|NotARealPerson@USCG.MIL
+Unclassified

pkg/parser/tac/parse.go

@@ -0,0 +1,234 @@
+package tac
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/transcom/mymove/pkg/models"
+)
+
+// Parse the pipe delimited .txt file with the following assumptions:
+// 1. The first and last lines are the security classification.
+// 2. The second line of the file are the columns that will be a 1:1 match to the TransportationAccountingCodeTrdmFileRecord struct in pipe delimited format.
+// 3. There are 23 values per line, excluding the security classification. Again, to know what these values are refer to note #2.
+// 4. All values are in pipe delimited format.
+// 5. Null values will be present, but are not acceptable for TRNSPRTN_ACNT_CD.
+func Parse(file io.Reader) ([]models.TransportationAccountingCode, error) {
+
+	// Init variables
+	var codes []models.TransportationAccountingCode
+	scanner := bufio.NewScanner(file)
+	var columnHeaders []string
+
+	// Skip first line as it does not hold any necessary data for parsing.
+	// Additionally, this will check if it is empty.
+	if !scanner.Scan() {
+		if err := scanner.Err(); err != nil {
+			return nil, err
+		}
+		return nil, errors.New("empty file")
+	}
+
+	// Read the second line to get the column names from the TRDM file, verify them,
+	// and then proceed with parsing the rest of the file.
+	if scanner.Scan() {
+		columnHeaders = strings.Split(scanner.Text(), "|")
+		ensureFileStructMatchesColumnNames(columnHeaders)
+	}
+
+	// Process the lines of the .txt file into modeled codes
+	codes, err := processLines(scanner, columnHeaders, codes)
+	if err != nil {
+		return nil, err
+	}
+
+	return codes, nil
+}
+
+// Compare a struct's field names to the columns retrieved from the .txt file
+func ensureFileStructMatchesColumnNames(columnNames []string) error {
+	if len(columnNames) == 0 {
+		return errors.New("column names were not parsed properly from the second line of the tac file")
+	}
+	expectedColumnNames := getFieldNames(models.TransportationAccountingCodeTrdmFileRecord{})
+	if !reflect.DeepEqual(columnNames, expectedColumnNames) {
+		return errors.New("column names parsed do not match the expected format of tac file records")
+	}
+	return nil
+}
+
+// This function gathers the struct field names for comparison to
+// line 2 of the .txt file - The columns
+func getFieldNames(obj interface{}) []string {
+	var fieldNames []string
+
+	t := reflect.TypeOf(obj)
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		fieldNames = append(fieldNames, field.Name)
+	}
+
+	return fieldNames
+}
+
+// Removes all TACs with an expiration date in the past
+func PruneExpiredTACs(codes []models.TransportationAccountingCode) []models.TransportationAccountingCode {
+	var pruned []models.TransportationAccountingCode
+
+	// If the expiration date is not before time.Now(), then it is not expired and should be appended to the pruned array
+	for _, code := range codes {
+		if !code.TrnsprtnAcntEndDt.Before(time.Now()) {
+			pruned = append(pruned, code)
+		}
+	}
+
+	return pruned
+}
+
+// Consolidates TACs with the same TAC value. Duplicate "Transaction", aka description & TrnsprtnAcntTx, values are combined with a delimeter of ". Additional description found: "
+func ConsolidateDuplicateTACsDesiredFromTRDM(codes []models.TransportationAccountingCode) []models.TransportationAccountingCode {
+	consolidatedMap := make(map[string]models.TransportationAccountingCode)
+
+	for _, code := range codes {
+		consolidatedMap[code.TAC] = overwriteDuplicateCode(consolidatedMap[code.TAC], code)
+	}
+
+	var consolidated []models.TransportationAccountingCode
+	for _, value := range consolidatedMap {
+		consolidated = append(consolidated, value)
+	}
+
+	return consolidated
+}
+
+// This function checks two TAC codes: one existing and one new to decide which one to keep based on their ExpirationDates
+// If the ExpirationDate is the same, it appends the transactions and maintains the first code found
+// If the ExpirationDate is different, it appends the transactions and maintains the ExpirationDate further in the future
+func overwriteDuplicateCode(existingCode models.TransportationAccountingCode, newCode models.TransportationAccountingCode) models.TransportationAccountingCode {
+
+	// If the existing code has a nil expiry date but the new code does not, the new code is a better representation and should be return
+	if existingCode.TrnsprtnAcntEndDt == nil && newCode.TrnsprtnAcntEndDt != nil {
+		return newCode
+	}
+
+	// If the new code expires later, append its transaction to the existing one (if not empty), and keep the new code
+	if newCode.TrnsprtnAcntEndDt.After(*existingCode.TrnsprtnAcntEndDt) && newCode.TrnsprtnAcntEndDt != nil {
+		if existingCode.TrnsprtnAcntTx != nil && *existingCode.TrnsprtnAcntTx != "" {
+			*newCode.TrnsprtnAcntTx = *existingCode.TrnsprtnAcntTx + ". Additional description found: " + *newCode.TrnsprtnAcntTx
+		}
+		return newCode
+	}
+
+	// If the new code expires at the same time or earlier compared to the existing code,
+	// append its transaction to the existing code (if not empty) because this one expires earlier or is already expired.
+	// A separate function handles the pruning of expired codes, not this one
+	// Additionally, the new codes end date must not be nil
+	if newCode.TrnsprtnAcntEndDt.Before(*existingCode.TrnsprtnAcntEndDt) || newCode.TrnsprtnAcntEndDt.Equal(*existingCode.TrnsprtnAcntEndDt) && newCode.TrnsprtnAcntEndDt != nil {
+		if existingCode.TrnsprtnAcntTx != nil && *existingCode.TrnsprtnAcntTx != "" {
+			*existingCode.TrnsprtnAcntTx = *existingCode.TrnsprtnAcntTx + ". Additional description found: " + *newCode.TrnsprtnAcntTx
+		} else {
+			existingCode.TrnsprtnAcntTx = newCode.TrnsprtnAcntTx
+		}
+	}
+
+	return existingCode
+}
+
+// This function handles the heavy lifting for the main parse function. It handles the scanning of every line and conversion into the TransportationAccountingCode model.
+func processLines(scanner *bufio.Scanner, columnHeaders []string, codes []models.TransportationAccountingCode) ([]models.TransportationAccountingCode, error) {
+	// Scan every line and parse into Transportation Accounting Codes
+	for scanner.Scan() {
+		line := scanner.Text()
+		var tacFyTxt int
+		var tacSysId int
+		var loaSysID int
+		var err error
+
+		// This check will skip the last line of the file.
+		if line == "Unclassified" {
+			break
+		}
+
+		// Gather values from the pipe delimited line
+		values := strings.Split(line, "|")
+		if len(values) != len(columnHeaders) {
+			return nil, errors.New("malformed line in the provided tac file: " + line)
+		}
+
+		// Skip the entry if the TAC value is empty
+		if values[2] == "" {
+			continue
+		}
+
+		// If TacSysID is not empty, convert to int
+		if values[0] != "" {
+			tacSysId, err = strconv.Atoi(values[0])
+			if err != nil {
+				return nil, errors.New("malformed tac_sys_id in the provided tac file: " + line)
+			}
+		}
+
+		// If LoaSysId is not empty, convert to int
+		if values[1] != "" {
+			loaSysID, err = strconv.Atoi(values[1])
+			if err != nil {
+				return nil, errors.New("malformed loa_sys_id in the provided tac file: " + line)
+			}
+		}
+
+		// Check if fiscal year text is not empty, convert to int
+		if values[3] != "" {
+			tacFyTxt, err = strconv.Atoi(values[3])
+			if err != nil {
+				return nil, fmt.Errorf("malformed tac_fy_txt in the provided tac file: %s", err)
+			}
+		}
+
+		effectiveDate, err := time.Parse("2006-01-02 15:04:05", values[16])
+		if err != nil {
+			return nil, fmt.Errorf("malformed effective date in the provided tac file: %s", err)
+		}
+
+		expiredDate, err := time.Parse("2006-01-02 15:04:05", values[17])
+		if err != nil {
+			return nil, fmt.Errorf("malformed expiration date in the provided tac file: %s", err)
+		}
+
+		code := models.TransportationAccountingCode{
+			TacSysID:           &tacSysId,
+			LoaSysID:           &loaSysID,
+			TAC:                values[2],
+			TacFyTxt:           &tacFyTxt,
+			TacFnBlModCd:       &values[4],
+			OrgGrpDfasCd:       &values[5],
+			TacMvtDsgID:        &values[6],
+			TacTyCd:            &values[7],
+			TacUseCd:           &values[8],
+			TacMajClmtID:       &values[9],
+			TacBillActTxt:      &values[10],
+			TacCostCtrNm:       &values[11],
+			Buic:               &values[12],
+			TacHistCd:          &values[13],
+			TacStatCd:          &values[14],
+			TrnsprtnAcntTx:     &values[15],
+			TrnsprtnAcntBgnDt:  &effectiveDate,
+			TrnsprtnAcntEndDt:  &expiredDate,
+			DdActvtyAdrsID:     &values[18],
+			TacBlldAddFrstLnTx: &values[19],
+			TacBlldAddScndLnTx: &values[20],
+			TacBlldAddThrdLnTx: &values[21],
+			TacBlldAddFrthLnTx: &values[22],
+			TacFnctPocNm:       &values[23],
+		}
+
+		codes = append(codes, code)
+	}
+
+	return codes, scanner.Err()
+}