Merge branch 'secure-params-fix' of github.com:ericholguin/trafficcon…

…trol into secure-params-fix

.asf.yaml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# ATC Collaborators for July 2023
+# ATC Collaborators for August 2023
 # Collaborators are contributors, other than committers, who have had 2 or more Issue-closing Pull Requests merged
 # in the past 31 days. If you want to be an Apache Traffic Control collaborator:
 # 1. Read our contribution guidelines at https://github.com/apache/trafficcontrol/blob/master/CONTRIBUTING.md
@@ -26,7 +26,6 @@
 github:
   collaborators:
   - jagan-parthiban
-  - heneriknguyen
   description: Apache Traffic Control is an Open Source implementation of a Content
     Delivery Network
   enabled_merge_buttons:

.github/actions/tpv2-integration-tests/config.json

@@ -2,6 +2,7 @@
   "insecure": true,
   "port": 4200,
   "trafficOps": "https://localhost:6443",
+  "tpv1Url": "https://localhost",
   "useSSL": false,
   "browserFolder": "$GITHUB_WORKSPACE/$BROWSER_FOLDER"
 }

CHANGELOG.md

@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - [#7619](https://github.com/apache/trafficcontrol/pull/7619) Traffic Ops* added optional field `oauth_user_attribute` for OAuth login credentials
 - [#7641](https://github.com/apache/trafficcontrol/pull/7641) *Traffic Router* Added further optimization to TR's algorithm of figuring out the zone for an incoming request.
 - [#7646](https://github.com/apache/trafficcontrol/pull/7646) *Traffic Portal* Add the ability to delete a cert.
+- [#7652](https://github.com/apache/trafficcontrol/pull/7652) *t3c* added rpmdb checks and use package data from t3c-apply-metadata.json if rpmdb is corrupt
 
 ### Changed
 - [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent.
@@ -65,6 +66,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - [#7469](https://github.com/apache/trafficcontrol/pull/7469) *Traffic Ops* Changed logic to not report empty or missing cookies into TO error.log.
 - [#7586](https://github.com/apache/trafficcontrol/pull/7586) *Traffic Ops* Add permission to Operations Role to read from dnsseckeys endpoint.
 - [#7600](https://github.com/apache/trafficcontrol/pull/7600) *t3c* changed default go-direct command line arg to be old to avoid unexpected config changes upon upgrade.
+- [##7605](https://github.com/apache/trafficcontrol/pull/#7605) *Traffic Ops* Fixes `cachegroups_request_comments` v5 apis to respond with `RFC3339` date/time Format.
 - [#7621](https://github.com/apache/trafficcontrol/pull/7621) *Traffic Ops* Use ID token for OAuth authentication, not Access Token
 
 ### Fixed
@@ -138,6 +140,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - [#7575](https://github.com/apache/trafficcontrol/pull/7575) *Traffic Ops* Fixes `types` v5 apis to respond with `RFC3339` date/time Format.
 - [#7628](https://github.com/apache/trafficcontrol/pull/7628) *Traffic Ops* Fixes an issue where certificate chain validation failed based on leading or trailing whitespace.
 - [#7596](https://github.com/apache/trafficcontrol/pull/7596) *Traffic Ops* Fixes `federation_resolvers` v5 apis to respond with `RFC3339` date/time Format.
+- [#7660](https://github.com/apache/trafficcontrol/pull/7660) *Traffic Ops* Fixes `deliveryServices` v5 apis to respond with `RFC3339` date/time Format.
 - [#7686](https://github.com/apache/trafficcontrol/pull/7686) *Traffic Ops* Fixes secured parameters being visible when role has proper permissions.
 
 ### Removed

cache-config/t3c-apply/config/config.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -81,6 +82,7 @@ type Cfg struct {
 	SvcManagement       SvcManagement
 	Retries             int
 	ReverseProxyDisable bool
+	RpmDBOk             bool
 	SkipOSCheck         bool
 	UseStrategies       t3cutil.UseStrategiesFlag
 	TOInsecure          bool
@@ -188,6 +190,29 @@ func directoryExists(dir string) (bool, os.FileInfo) {
 	return info.IsDir(), info
 }
 
+const rpmDir = "/var/lib/rpm"
+
+// verifies the rpm database files. if there is any database corruption
+// it will return false
+func verifyRpmDB() bool {
+	exclude := regexp.MustCompile(`(^\.|^__)`)
+	dbFiles, err := os.ReadDir(rpmDir)
+	if err != nil {
+		return false
+	}
+	for _, file := range dbFiles {
+		if exclude.Match([]byte(file.Name())) {
+			continue
+		}
+		cmd := exec.Command("/usr/lib/rpm/rpmdb_verify", rpmDir+"/"+file.Name())
+		err := cmd.Run()
+		if err != nil || cmd.ProcessState.ExitCode() > 0 {
+			return false
+		}
+	}
+	return true
+}
+
 // derives the ATS Installation directory from
 // the rpm config file list.
 func GetTSPackageHome() string {
@@ -322,10 +347,11 @@ If any of the related flags are also set, they override the mode's default behav
 	// so we want to log what flags the mode set here, to aid debugging.
 	// But we can't do that until the loggers are initialized.
 	modeLogStrs := []string{}
+	fatalLogStrs := []string{}
 	if getopt.IsSet(runModeFlagName) {
 		runMode := t3cutil.StrToMode(*runModePtr)
 		if runMode == t3cutil.ModeInvalid {
-			return Cfg{}, errors.New(*runModePtr + " is an invalid mode.")
+			fatalLogStrs = append(fatalLogStrs, *runModePtr+" is an invalid mode.")
 		}
 		modeLogStrs = append(modeLogStrs, "t3c-apply is running in "+runMode.String()+" mode")
 		switch runMode {
@@ -411,7 +437,7 @@ If any of the related flags are also set, they override the mode's default behav
 	}
 
 	if *verbosePtr > 2 {
-		return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
+		fatalLogStrs = append(fatalLogStrs, "Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
 	}
 
 	var cacheHostName string
@@ -420,7 +446,7 @@ If any of the related flags are also set, they override the mode's default behav
 	} else {
 		cacheHostName, err = os.Hostname()
 		if err != nil {
-			return Cfg{}, errors.New("Could not get the hostname from the O.S., please supply a hostname: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "Could not get the hostname from the O.S., please supply a hostname: "+err.Error())
 		}
 		// strings.Split always returns a slice with at least 1 element, so we don't need a len check
 		cacheHostName = strings.Split(cacheHostName, ".")[0]
@@ -429,7 +455,7 @@ If any of the related flags are also set, they override the mode's default behav
 	useGit := StrToUseGitFlag(*useGitStr)
 
 	if useGit == UseGitInvalid {
-		return Cfg{}, errors.New("Invalid git flag '" + *useGitStr + "'. Valid options are yes, no, auto.")
+		fatalLogStrs = append(fatalLogStrs, "Invalid git flag '"+*useGitStr+"'. Valid options are yes, no, auto.")
 	}
 
 	retries := *retriesPtr
@@ -471,6 +497,17 @@ If any of the related flags are also set, they override the mode's default behav
 		os.Setenv("TO_PASS", toPass)
 	}
 
+	rpmDBisOk := verifyRpmDB()
+
+	if *installPackagesPtr && !rpmDBisOk {
+		if t3cutil.StrToMode(*runModePtr) == t3cutil.ModeBadAss {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue in badass mode")
+		} else {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue")
+		}
+	}
+
+	toInfoLog = append(toInfoLog, fmt.Sprintf("rpm database is ok: %t", rpmDBisOk))
 	// set TSHome
 	var tsHome = ""
 	if *tsHomePtr != "" {
@@ -481,13 +518,13 @@ If any of the related flags are also set, they override the mode's default behav
 		tsHome = os.Getenv("TS_HOME") // check for the environment variable.
 		if tsHome != "" {
 			toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from TS_HOME environment variable '%s'\n", TSHome))
-		} else { // finally check using the config file listing from the rpm package.
+		} else if rpmDBisOk { // check using the config file listing from the rpm package if rpmdb is ok.
 			tsHome = GetTSPackageHome()
 			if tsHome != "" {
 				toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from the RPM config file  list '%s'\n", TSHome))
-			} else {
-				toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 			}
+		} else if tsHome == "" {
+			toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 		}
 	}
 
@@ -503,23 +540,23 @@ If any of the related flags are also set, they override the mode's default behav
 	if *useLocalATSVersionPtr {
 		atsVersionStr, err = GetATSVersionStr(tsHome)
 		if err != nil {
-			return Cfg{}, errors.New("getting local ATS version: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "getting local ATS version: "+err.Error())
 		}
 	}
 	toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr))
 
 	usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache"
 	if strings.TrimSpace(toURL) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-url or TO_URL environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toUser) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-user or TO_USER environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-user or TO_USER environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toPass) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-password or TO_PASS environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-password or TO_PASS environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(cacheHostName) == "" {
-		return Cfg{}, errors.New("Missing required argument --cache-host-name. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --cache-host-name. "+usageStr)
 	}
 
 	toURLParsed, err := url.Parse(toURL)
@@ -540,6 +577,7 @@ If any of the related flags are also set, they override the mode's default behav
 		CacheHostName:               cacheHostName,
 		SvcManagement:               svcManagement,
 		Retries:                     retries,
+		RpmDBOk:                     rpmDBisOk,
 		ReverseProxyDisable:         reverseProxyDisable,
 		SkipOSCheck:                 skipOsCheck,
 		UseStrategies:               useStrategies,
@@ -580,6 +618,13 @@ If any of the related flags are also set, they override the mode's default behav
 		return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
 	}
 
+	if len(fatalLogStrs) > 0 {
+		for _, str := range fatalLogStrs {
+			str = strings.TrimSpace(str)
+			log.Errorln(str)
+		}
+		return Cfg{}, errors.New("fatal error has occurred")
+	}
 	for _, str := range modeLogStrs {
 		str = strings.TrimSpace(str)
 		if str == "" {

cache-config/t3c-apply/t3c-apply.go

@@ -22,7 +22,6 @@ package main
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -94,8 +93,8 @@ func Main() int {
 	var lock util.FileLock
 	cfg, err := config.GetCfg(Version, GitRevision)
 	if err != nil {
-		fmt.Println(err)
-		fmt.Println(FailureExitMsg)
+		log.Infoln(err)
+		log.Errorln(FailureExitMsg)
 		return ExitCodeConfigError
 	} else if cfg == (config.Cfg{}) { // user used the --help option
 		return ExitCodeSuccess
@@ -261,7 +260,7 @@ func Main() int {
 		// make sure we got the data necessary to check packages
 		log.Infoln("======== Didn't get all files, no package processing needed or possible ========")
 		metaData.InstalledPackages = oldMetaData.InstalledPackages
-	} else {
+	} else if cfg.RpmDBOk {
 		log.Infoln("======== Start processing packages  ========")
 		err = trops.ProcessPackages()
 		if err != nil {
@@ -276,6 +275,9 @@ func Main() int {
 			log.Errorf("Error verifying system services: %s\n", err.Error())
 			return GitCommitAndExit(ExitCodeServicesError, FailureExitMsg, cfg, metaData, oldMetaData)
 		}
+	} else {
+		log.Warnln("======== RPM DB checks failed, package processing not possible, using installed packages from  metadata if available========")
+		trops.ProcessPackagesWithMetaData(oldMetaData.InstalledPackages)
 	}
 
 	log.Debugf("Preparing to fetch the config files for %s, files: %s, syncdsUpdate: %s\n", cfg.CacheHostName, cfg.Files, syncdsUpdate)

cache-config/t3c-apply/torequest/torequest.go

@@ -587,10 +587,14 @@ func (r *TrafficOpsReq) CheckSystemServices() error {
 func (r *TrafficOpsReq) IsPackageInstalled(name string) bool {
 	for k, v := range r.Pkgs {
 		if strings.HasPrefix(k, name) {
+			log.Infof("Found in cache for '%s'", k)
 			return v
 		}
 	}
-
+	if !r.Cfg.RpmDBOk {
+		log.Warnf("RPM DB is corrupted cannot run IsPackageInstalled for '%s' and package metadata is unavailable", name)
+		return false
+	}
 	log.Infof("IsPackageInstalled '%v' not found in cache, querying rpm", name)
 	pkgArr, err := util.PackageInfo("pkg-query", name)
 	if err != nil {
@@ -1030,6 +1034,47 @@ func (r *TrafficOpsReq) ProcessPackages() error {
 	return nil
 }
 
+func pkgMetaDataToMap(pmd []string) map[string]bool {
+	pkgMap := map[string]bool{}
+	for _, pkg := range pmd {
+		pkgMap[pkg] = true
+	}
+	return pkgMap
+}
+
+func pkgMatch(pkgMetaData []string, pk string) bool {
+	for _, pkg := range pkgMetaData {
+		if strings.Contains(pk, pkg) {
+			return true
+		}
+	}
+	return false
+
+}
+
+// ProcessPackagesWithMetaData will attempt to get installed package data from
+// t3c-apply-metadata.json and log the results.
+func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error {
+	pkgs, err := getPackages(r.Cfg)
+	pkgMdataMap := pkgMetaDataToMap(packageMetaData)
+	if err != nil {
+		return fmt.Errorf("getting packages: %w", err)
+	}
+	for _, pkg := range pkgs {
+		fullPackage := pkg.Name + "-" + pkg.Version
+		if pkgMdataMap[fullPackage] {
+			log.Infof("package %s is assumed to be installed according to metadata file", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else if pkgMatch(packageMetaData, pkg.Name) {
+			log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else {
+			log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version)
+		}
+	}
+	return nil
+}
+
 func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
 	updateStatus, err := r.CheckRevalidateState(true)
 	if err != nil {

docs/source/api/v5/cachegroups.rst

@@ -71,7 +71,7 @@ Response Structure
 :fallbacks:                     An array of strings that are :ref:`Cache Group names <cache-group-name>` that are registered as :ref:`cache-group-fallbacks` for this :term:`Cache Group`\ [#fallbacks]_
 :fallbackToClosest:             A boolean value that defines the :ref:`cache-group-fallback-to-closest` behavior of this :term:`Cache Group`\ [#fallbacks]_
 :id:                            An integer that is the :ref:`cache-group-id` of the :term:`Cache Group`
-:lastUpdated:                   The time and date at which this entry was last updated in :ref:`non-rfc-datetime`
+:lastUpdated:                   The time and date at which this entry was last updated in :rfc:`3339`
 :latitude:                      A floating-point :ref:`cache-group-latitude` for the :term:`Cache Group`
 :localizationMethods:           An array of :ref:`cache-group-localization-methods` as strings
 :longitude:                     A floating-point :ref:`cache-group-longitude` for the :term:`Cache Group`
@@ -116,7 +116,7 @@ Response Structure
 			"localizationMethods": [],
 			"typeName": "EDGE_LOC",
 			"typeId": 23,
-			"lastUpdated": "2018-11-07 14:45:43+00",
+			"lastUpdated": "2023-05-30T19:52:58.183642+00:00",
 			"fallbacks": []
 		}
 	]}
@@ -182,7 +182,7 @@ Response Structure
 :fallbacks:                     An array of strings that are :ref:`Cache Group names <cache-group-name>` that are registered as :ref:`cache-group-fallbacks` for this :term:`Cache Group`\ [#fallbacks]_
 :fallbackToClosest:             A boolean value that defines the :ref:`cache-group-fallback-to-closest` behavior of this :term:`Cache Group`\ [#fallbacks]_
 :id:                            An integer that is the :ref:`cache-group-id` of the :term:`Cache Group`
-:lastUpdated:                   The time and date at which this entry was last updated in :ref:`non-rfc-datetime`
+:lastUpdated:                   The time and date at which this entry was last updated in :rfc:`3339`
 :latitude:                      A floating-point :ref:`cache-group-latitude` for the :term:`Cache Group`
 :localizationMethods:           An array of :ref:`cache-group-localization-methods` as strings
 :longitude:                     A floating-point :ref:`cache-group-longitude` for the :term:`Cache Group`
@@ -207,7 +207,7 @@ Response Structure
 	Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly
 	Whole-Content-Sha512: YvZlh3rpfl3nBq6SbNVhbkt3IvckbB9amqGW2JhLxWK9K3cxjBq5J2sIHBUhrLKUhE9afpxtvaYrLRxjt1/YMQ==
 	X-Server-Name: traffic_ops_golang/
-	Date: Wed, 07 Nov 2018 22:11:50 GMT
+	Date: Wed, 07 Nov 2018 19:46:36 GMT
 	Content-Length: 379
 
 	{ "alerts": [
@@ -234,7 +234,7 @@ Response Structure
 		],
 		"typeName": "EDGE_LOC",
 		"typeId": 23,
-		"lastUpdated": "2019-12-02 22:21:08+00",
+		"lastUpdated": "2023-05-30T19:52:58.183642+00:00",
 		"fallbacks": []
 	}}
 

docs/source/api/v5/cachegroups_id.rst

@@ -84,7 +84,7 @@ Response Structure
 :fallbacks:                     An array of strings that are :ref:`Cache Group names <cache-group-name>` that are registered as :ref:`cache-group-fallbacks` for this :term:`Cache Group`\ [#fallbacks]_
 :fallbackToClosest:             A boolean value that defines the :ref:`cache-group-fallback-to-closest` behavior of this :term:`Cache Group`\ [#fallbacks]_
 :id:                            An integer that is the :ref:`cache-group-id` of the :term:`Cache Group`
-:lastUpdated:                   The time and date at which this entry was last updated in :ref:`non-rfc-datetime`
+:lastUpdated:                   The time and date at which this entry was last updated in :rfc:`3339`
 :latitude:                      A floating-point :ref:`cache-group-latitude` for the :term:`Cache Group`
 :localizationMethods:           An array of :ref:`cache-group-localization-methods` as strings
 :longitude:                     A floating-point :ref:`cache-group-longitude` for the :term:`Cache Group`
@@ -135,7 +135,7 @@ Response Structure
 		],
 		"typeName": "EDGE_LOC",
 		"typeId": 23,
-		"lastUpdated": "2018-11-14 19:14:28+00"
+		"lastUpdated": "2023-05-30T19:52:58.183642+00:00"
 	}}
 
 
@@ -181,7 +181,7 @@ Response Structure
 	Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly
 	Whole-Content-Sha512: 5jZBgO7h1eNF70J/cmlbi3Hf9KJPx+WLMblH/pSKF3FWb/10GUHIN35ZOB+lN5LZYCkmk3izGbTFkiruG8I41Q==
 	X-Server-Name: traffic_ops_golang/
-	Date: Wed, 14 Nov 2018 20:31:04 GMT
+	Date: Wed, 14 Nov 2018 19:14:28 GMT
 	Content-Length: 57
 
 	{ "alerts": [