improved backflow algorithm, added (Created) to README, included some…

… extra functions needed by Wizard

README.md

@@ -89,6 +89,13 @@ Workflow:
     Doing: Doing  
     Done: Done
 
+Sometimes Jira issues are created with a certain status, and there is no event that corresponds to a move into that status, so there is no date at which the work item entered the corresponding workflow stage. You can designate that an item be created in a certain workflow stage by adding (Created) to the list of Jira statuses. For example, in the previous example if you wanted to designate that items enter the ToDo workflow stage when they are created, you would change the workflow section of the config file as follows:
+
+Workflow:  
+    ToDo: ToDo, (Created)
+    Doing: Doing  
+    Done: Done
+
 Again, please refer to the sample config file for an example of what the workflow section looks like. 
 
 **NOTE**:  The Workflow section requires a minimum of TWO (2) workflow steps, and the names of the workflow steps must be specified in the order you want them to appear in the output CSV.  The expectation is that all Jira issue types that are requested will follow the exact same workflow steps in the exact same order.

config.go

@@ -14,13 +14,16 @@ var criteriaKeys = []string{"Types", "Projects", "Filters"}
 var attributeFields = []string{"status", "issuetype", "priority", "resolution", "project",
 	"labels", "fixVersions", "components"}
 
+const URL_SUFFIX = "/rest/api/latest"
+
 type ConfigAttr struct {
 	ColumnName  string // CSV column name
 	FieldName   string // Jira field from attributeFields above or a customfield_...
 	ContentName string // Struct field to pull the value from (default to "value", n/a for arrays)
 }
 
 type Config struct {
+
 	// connection stuff
 	Domain   string
 	UrlRoot  string
@@ -50,6 +53,22 @@ func (c *Config) GetCredentials() (credentials string, err error) {
 	return
 }
 
+func CreateConnectionConfig(domain, username, password string) *Config {
+	return &Config{
+		Domain:   domain,
+		UrlRoot:  domain + URL_SUFFIX,
+		Username: username,
+		Password: password,
+		StageMap: make(map[string]int),
+	}
+}
+
+func CreateProjectConfig(domain, username, password, project string) (config *Config) {
+	config = CreateConnectionConfig(domain, username, password)
+	config.ProjectNames = []string{project}
+	return config
+}
+
 func LoadConfigFromLines(lines []string) (*Config, error) {
 	config := Config{StageMap: make(map[string]int), CreateInFirstStage: false}
 
@@ -138,7 +157,7 @@ func LoadConfigFromLines(lines []string) (*Config, error) {
 	if len(config.Domain) == 0 {
 		return nil, fmt.Errorf("Config file has no property \"Domain\"")
 	}
-	config.UrlRoot = config.Domain + "/rest/api/latest/search"
+	config.UrlRoot = config.Domain + URL_SUFFIX
 
 	// extract required username and optional password
 	config.Username = properties["Username"]

items.go

@@ -7,10 +7,10 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 )
 
 const PRINT_JSON = false
@@ -42,6 +42,26 @@ type JiraItem struct {
 	ToString string
 }
 
+// for extracting project names
+type JiraProjectList []JiraProject
+
+type JiraProject struct {
+	Key  string
+	Name string
+}
+
+// for extracting workflows
+type JiraWorkflowList []JiraWorkflow
+
+type JiraWorkflow struct {
+	Name     string
+	Statuses []JiraStatus
+}
+
+type JiraStatus struct {
+	Name string
+}
+
 // collect work items for writing to file
 type Item struct {
 	Id         string
@@ -134,7 +154,6 @@ func getStages(query string, config *Config) (scores Scores, e error) {
 
 // returns items resulting from supplied query. Use getQuery() to get the query
 func getItems(query string, config *Config) (items []*Item, unusedStages map[string]int, e error) {
-
 	// fetch the issues
 	issues, err := getIssues(query, config)
 	if err != nil {
@@ -157,15 +176,15 @@ func getItems(query string, config *Config) (items []*Item, unusedStages map[str
 		// accumulate out-of-order events so we can handle backward flow
 		events := make([][]string, len(config.StageNames))
 		if config.CreateInFirstStage {
-			creationDate := strings.SplitN(fields["created"].(string), "T", 2)[0]
+			creationDate := fields["created"].(string)
 			events[0] = append(events[0], creationDate)
 		}
 		for _, history := range issue.Changelog.Histories {
 			for _, jItem := range history.Items {
 				if jItem.Field == "status" {
 					stageName := jItem.ToString
 					if stageIndex, found := config.StageMap[strings.ToUpper(stageName)]; found {
-						date := strings.SplitN(history.Created, "T", 2)[0]
+						date := history.Created
 						events[stageIndex] = append(events[stageIndex], date)
 					} else {
 						count := unusedStages[jItem.ToString]
@@ -175,24 +194,31 @@ func getItems(query string, config *Config) (items []*Item, unusedStages map[str
 			}
 		}
 
-		// for each stage use min date that is >= max date from previous stages
-		previousMaxDate := ""
-		for stageIndex := range config.StageNames {
-			stageBestDate := ""
-			stageMaxDate := ""
-			for _, date := range events[stageIndex] {
-				if date >= previousMaxDate && (stageBestDate == "" || date < stageBestDate) {
-					stageBestDate = date
-				}
-				if date > stageMaxDate {
-					stageMaxDate = date
+		// backflow occurs when a stage has multiple dates. Note the latest date in the stage,
+		// but keep only the earliest date. Then traverse subsequent stages deleting all dates
+		// that are on or before that noted latest date.
+		latestValidDate := ""
+		for stageIndex, stageDates := range events {
+
+			// first filter out dates in this stage that are before latestValidDate
+			stageDates = removeDatesBefore(stageDates, latestValidDate)
+
+			// now handle backflow
+			if len(stageDates) > 1 {
+				sort.Strings(stageDates)                       // from earliest to latest
+				latestInStage := stageDates[len(stageDates)-1] // note the latest
+				stageDates = stageDates[:1]                    // keep only the earliest
+
+				// traverse subsequent stages, removing dates before latest
+				for j := stageIndex + 1; j < len(events); j++ {
+					events[j] = removeDatesBefore(events[j], latestInStage)
 				}
 			}
-			if stageBestDate != "" {
-				item.StageDates[stageIndex] = stageBestDate
-			}
-			if stageMaxDate != "" && stageMaxDate > previousMaxDate {
-				previousMaxDate = stageMaxDate
+
+			// at this point 0 or 1 date remains
+			if len(stageDates) > 0 {
+				latestValidDate = stageDates[0]
+				item.StageDates[stageIndex] = strings.SplitN(latestValidDate, "T", 2)[0]
 			}
 		}
 
@@ -225,10 +251,10 @@ func getItems(query string, config *Config) (items []*Item, unusedStages map[str
 					item.Attributes[i] = getValue(fields, "project", "name")
 				case "labels":
 					item.Attributes[i] = getValue(fields, "labels", "")
-				case "fixVersion":
-					item.Attributes[i] = getValue(fields, "fixVersion", "")
+				case "fixVersions":
+					item.Attributes[i] = getValue(fields, "fixVersions", "name")
 				case "components":
-					item.Attributes[i] = getValue(fields, "components", "")
+					item.Attributes[i] = getValue(fields, "components", "name")
 				}
 			}
 		}
@@ -239,6 +265,18 @@ func getItems(query string, config *Config) (items []*Item, unusedStages map[str
 	return items, unusedStages, nil
 }
 
+// performed in place
+func removeDatesBefore(dates []string, date string) []string {
+	keepIndex := 0
+	for _, d := range dates {
+		if d >= date {
+			dates[keepIndex] = d
+			keepIndex++
+		}
+	}
+	return dates[:keepIndex]
+}
+
 func getQuery(startIndex int, batchSize int, config *Config) string {
 	var clauses []string
 	if len(config.ProjectNames) == 1 {
@@ -255,12 +293,99 @@ func getQuery(startIndex int, batchSize int, config *Config) string {
 	jql := strings.Join(clauses, " AND ") + " order by key"
 
 	return config.UrlRoot +
-		"?jql=" + url.QueryEscape(jql) +
+		"/search?jql=" + url.QueryEscape(jql) +
 		"&startAt=" + strconv.Itoa(startIndex) +
 		"&maxResults=" + strconv.Itoa(batchSize) +
 		"&expand=changelog"
 }
 
+func getProjects(config *Config) (result JiraProjectList, e error) {
+
+	// get credentials
+	credentials, err := config.GetCredentials()
+	if err != nil {
+		return result, err
+	}
+
+	// send the request
+	client := http.Client{}
+	url := config.UrlRoot + "/project"
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", "Basic "+credentials)
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// process the response
+	if resp.StatusCode == 200 { // OK
+
+		if PRINT_JSON {
+			var indented bytes.Buffer
+			bodyBytes, _ := ioutil.ReadAll(resp.Body)
+			json.Indent(&indented, bodyBytes, "", "\t")
+			indented.WriteTo(os.Stdout)
+		}
+
+		// decode json
+		var list JiraProjectList
+		json.NewDecoder(resp.Body).Decode(&list)
+		result = list
+
+	} else {
+		e = fmt.Errorf("Error: %v failed", url)
+	}
+
+	return result, e
+}
+
+func getWorkflows(config *Config) (result JiraWorkflowList, e error) {
+
+	// get credentials
+	credentials, err := config.GetCredentials()
+	if err != nil {
+		return result, err
+	}
+
+	// extract the project
+	project := strings.Trim(config.ProjectNames[0], "\"")
+
+	// send the request
+	client := http.Client{}
+	url := config.UrlRoot + "/project/" + project + "/statuses"
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", "Basic "+credentials)
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// process the response
+	if resp.StatusCode == 200 { // OK
+
+		if PRINT_JSON {
+			var indented bytes.Buffer
+			bodyBytes, _ := ioutil.ReadAll(resp.Body)
+			json.Indent(&indented, bodyBytes, "", "\t")
+			indented.WriteTo(os.Stdout)
+		}
+
+		// decode json
+		var list JiraWorkflowList
+		json.NewDecoder(resp.Body).Decode(&list)
+		result = list
+
+	} else {
+		e = fmt.Errorf("Error: %v failed", url)
+	}
+
+	return result, e
+}
+
 // returns items resulting from supplied query. Use getQuery() to get the query
 func getIssues(query string, config *Config) (result []JiraIssue, e error) {
 
@@ -271,28 +396,25 @@ func getIssues(query string, config *Config) (result []JiraIssue, e error) {
 	}
 
 	// send the request
-	client := http.Client{Timeout: time.Duration(60 * time.Second)}
+	//client := http.Client{Timeout: time.Duration(10*time.Millisecond)}
+	client := http.Client{}
 	req, _ := http.NewRequest("GET", query, nil)
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("Authorization", "Basic "+credentials)
 	var resp *http.Response
 	if resp, err = client.Do(req); err != nil {
-		return nil, fmt.Errorf("Couldn't connect to %s\n"+
-			"Possible causes:\n"+
-			" - Misspelled domain\n"+
-			" - No network connection\n",
-			config.Domain)
+		return nil, fmt.Errorf("No such host: %v", err)
 	}
 	defer resp.Body.Close()
 
 	// process the response
 	if resp.StatusCode == 200 { // OK
 
 		if PRINT_JSON {
+			var indented bytes.Buffer
 			bodyBytes, _ := ioutil.ReadAll(resp.Body)
-			bodyString := string(bodyBytes)
-			fmt.Println(bodyString)
-			return result, e
+			json.Indent(&indented, bodyBytes, "", "\t")
+			indented.WriteTo(os.Stdout)
 		}
 
 		// decode json
@@ -407,6 +529,9 @@ func (item *Item) toCSV(config *Config, writeLink bool) string {
 	for _, stageDate := range item.StageDates {
 		buffer.WriteString("," + stageDate)
 	}
+	if len(config.Types) > 1 {
+		buffer.WriteString("," + item.Type)
+	}
 	for _, value := range item.Attributes {
 		buffer.WriteString("," + cleanString(value))
 	}