Merge branch 'master' into 4.x

.github/workflows/main.yml

@@ -12,10 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.16
-      uses: actions/setup-go@v1
+    - name: Set up Go
+      uses: actions/setup-go@v3
       with:
-        go-version: 1.16
+        go-version: 1.21
       id: go
 
     - name: Check out code into the Go module directory

.goreleaser.yml

@@ -93,6 +93,11 @@ nfpms:
       - deb
       - rpm
 
+    conflicts:
+      - dkron-pro
+    replaces:
+      - dkron-pro
+
     # Override default /usr/local/bin destination for binaries
     bindir: /usr/bin
 
@@ -109,11 +114,6 @@ nfpms:
       - dst: /var/log/dkron
         type: dir
 
-    overrides:
-      rpm:
-        replacements:
-          amd64: x86_64
-
 snapshot:
   name_template: "{{ .Tag }}-next"
 

CHANGELOG.md

@@ -6,6 +6,88 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [3.2.7] - 2023-12-09
+
+### Changes
+
+- Replace dependent class library github.com/struCoder/pidusage with github.com/shirou/gopsutil/v3/process
+- pkg import only once https://github.com/distribworks/dkron/pull/1406
+- Bump deps
+- Use Go 1.21
+
+### Features
+
+- add TLS options for kafka executor plugin https://github.com/distribworks/dkron/pull/1384
+- Resource usage from child processes spawned by the task startup process.
+- Introduce two new metrics, dkron_job_execution_time_seconds and dkron_job_execution_done_count, for improved monitoring and alerting capabilities.
+
+### Fixes
+
+- cmd.Process.Kill() does not clean up the started process, causing the job to remain running. https://github.com/distribworks/dkron/pull/1385
+- The metric dkron_job_mem_usage_kb now exclusively calculates the memory usage of the task startup process, excluding 
+- Should output.TotalWritten be judged after cmd.Wait() is executed? https://github.com/distribworks/dkron/pull/1319
+- The metrics exposed by dkron are only of summary type and cannot cover related analysis for specific jobs. https://github.com/distribworks/dkron/pull/1310
+
+## [3.2.6] - 2023-09-02
+
+### Changes
+
+- Bump deps & update docs
+
+## [3.2.5] - 2023-06-15
+
+### Changes
+
+- Bump deps
+
+## [3.2.4] - 2023-05-29
+
+### Fixes
+
+- Do not retry job when unknown status [#1335](https://github.com/distribworks/dkron/pull/1335)
+
+### Features
+
+- Prometheus metrics for shell executor [#1330](https://github.com/distribworks/dkron/pull/1330)
+
+### Changes
+
+- Bump deps
+- Fix linting issues [#1336](https://github.com/distribworks/dkron/pull/1336)
+
+## [3.2.3] - 2023-03-17
+
+### Fixes
+
+- Fix non existing dependent job (https://github.com/distribworks/dkron/pull/1297)
+### Changes
+
+- Bump deps
+
+## [3.2.2] - 2023-03-11
+
+### Changes
+
+- Bump deps
+- Update docusaurus
+
+## [3.2.1] - 2022-09-18
+
+### Features
+
+- Optionally set an alternative raftStore
+
+### Fixes
+
+- Re-add arm-v7 docker build
+- Minor fixes
+
+### Changes
+
+- Add golintci config
+- More variety to testing infra
+- Bump deps
+
 ## [3.2.0] - 2022-06-05
 
 ### Fixes

Makefile

@@ -5,7 +5,7 @@ $(LINUX_PKGS):
 	fury push --as distribworks $@
 
 PACKAGE_NAME          := github.com/distribworks/dkron
-GOLANG_CROSS_VERSION  ?= v1.18.1
+GOLANG_CROSS_VERSION  ?= v1.21.5
 
 .PHONY: release-dry-run
 release-dry-run:
@@ -22,7 +22,7 @@ release-dry-run:
 		-v `pwd`:/go/src/$(PACKAGE_NAME) \
 		-w /go/src/$(PACKAGE_NAME) \
 		goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
-		--rm-dist --skip-validate --skip-publish --timeout=1h
+		--rm-dist --skip-validate --skip-publish --timeout=1h --parallelism=1
 
 .PHONY: release
 release:

README.md

@@ -10,7 +10,7 @@ Dkron is a distributed cron service, easy to setup and fault tolerant with focus
 
 - Easy: Easy to use with a great UI
 - Reliable: Completely fault tolerant
-- High scalable: Able to handle high volumes of scheduled jobs and thousands of nodes
+- Highly scalable: Able to handle high volumes of scheduled jobs and thousands of nodes
 
 Dkron is written in Go and leverage the power of the Raft protocol and Serf for providing fault tolerance, reliability and scalability while keeping simple and easily installable.
 
@@ -22,7 +22,7 @@ You can use Dkron to run the most important part of your company, scheduled jobs
 
 ## Installation
 
-[Installation instructions](https://dkron.io/basics/installation/)
+[Installation instructions](https://dkron.io/docs/basics/installation)
 
 Full, comprehensive documentation is viewable on the [Dkron website](http://dkron.io)
 
@@ -75,15 +75,17 @@ https://github.com/bozerkins/terraform-provider-dkron
 Manage and run jobs in Dkron from your django project
 https://github.com/surface-security/django-dkron
 
+## Contributors
+
+<a href="https://github.com/distribworks/dkron/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=distribworks/dkron" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).
+
 ## Get in touch
 
 - Twitter: [@distribworks](https://twitter.com/distribworks)
 - Chat: https://gitter.im/distribworks/dkron
 - Email: victor at distrib.works
 
-# Sponsor
-
-This project is possible thanks to the Support of Jobandtalent
-
-![](https://upload.wikimedia.org/wikipedia/en/d/db/Jobandtalent_logo.jpg)
-

builtin/bins/dkron-executor-grpc/grpc.go

@@ -4,6 +4,10 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/armon/circbuf"
 	dkplugin "github.com/distribworks/dkron/v3/plugin"
 	dktypes "github.com/distribworks/dkron/v3/plugin/types"
@@ -15,9 +19,6 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 	reflectpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha"
-	"strconv"
-	"strings"
-	"time"
 )
 
 const (
@@ -95,6 +96,8 @@ func (g *GRPC) ExecuteImpl(args *dktypes.ExecuteRequest) ([]byte, error) {
 	if grpcDialErr != nil {
 		return output.Bytes(), grpcDialErr
 	}
+	defer cc.Close() // nolint:errcheck
+
 	md := grpcurl.MetadataFromHeaders([]string{})
 	refCtx := metadata.NewOutgoingContext(ctx, md)
 	refClient = grpcreflect.NewClient(refCtx, reflectpb.NewServerReflectionClient(cc))

builtin/bins/dkron-executor-http/http.go

@@ -76,6 +76,7 @@ func (s *HTTP) ExecuteImpl(args *types.ExecuteRequest) ([]byte, error) {
 	if err != nil {
 		return output.Bytes(), err
 	}
+	req.Close = true
 
 	var headers []string
 	if args.Config["headers"] != "" {

builtin/bins/dkron-executor-http/testdata/badssl.com-client.pem

@@ -1,13 +1,13 @@
 Bag Attributes
-    localKeyID: 77 23 D2 3A C0 2C 87 E2 AD 98 3F 06 68 F2 54 33 B6 05 0E FE 
+    localKeyID: 57 CC 99 3F 7C 3E D2 81 83 06 43 E2 9A 5E 12 06 9F 13 82 D0 
 subject=/C=US/ST=California/L=San Francisco/O=BadSSL/CN=BadSSL Client Certificate
 issuer=/C=US/ST=California/L=San Francisco/O=BadSSL/CN=BadSSL Client Root Certificate Authority
 -----BEGIN CERTIFICATE-----
-MIIEnTCCAoWgAwIBAgIJAPYAapdmy98xMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
+MIIEnTCCAoWgAwIBAgIJAMn8nqcEWekMMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
 BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp
 c2NvMQ8wDQYDVQQKDAZCYWRTU0wxMTAvBgNVBAMMKEJhZFNTTCBDbGllbnQgUm9v
-dCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMjExMjA0MDAwODE5WhcNMjMxMjA0
-MDAwODE5WjBvMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQG
+dCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMjMxMTI5MjIzNDAzWhcNMjUxMTI4
+MjIzNDAzWjBvMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQG
 A1UEBwwNU2FuIEZyYW5jaXNjbzEPMA0GA1UECgwGQmFkU1NMMSIwIAYDVQQDDBlC
 YWRTU0wgQ2xpZW50IENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
 MIIBCgKCAQEAxzdfEeseTs/rukjly6MSLHM+Rh0enA3Ai4Mj2sdl31x3SbPoen08
@@ -17,15 +17,48 @@ DeZTf2ES66abA5pOp60Q6OEdwg/vCUJfarhKDpi9tj3P6qToy9Y4DiBUhOct4MG8
 w5XwmKAC+Vfm8tb7tMiUoU0yvKKOcL6YXBXxB2kPcOYxYNobXavfVBEdwSrjQ7i/
 s3o6hkGQlm9F7JPEuVgbl/Jdwa64OYIqjQIDAQABoy0wKzAJBgNVHRMEAjAAMBEG
 CWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQELBQADggIB
-ABlLNovFvSrULgLvJmKX/boSWQOhWE0HDX6bVKyTs48gf7y3DXSOD+bHkBNHL0he
-m4HRFSarj+x389oiPEti5i12Ng9OLLHwSHK+7AfnrkhLHA8ML3NWw0GBr5DgdsIv
-7MJdGIrXPQwTN5j++ICyY588TfGHH8vU5qb5PrSqClLZSSHU05FTr/Dc1B8hKjjl
-d/FKOidLo1YDLFUjaB9x1mZPUic/C489lyPfWqPqoMRd5i/XShST5FPvfGuKRd5q
-XKDkrn+GaQ/4iDDdCgekDCCPhOwuulavNxBDjShwZt1TeUrZNSM3U4GeZfyrVBIu
-Tr+gBK4IkD9d/vP7sa2NQszF0wRQt3m1wvSWxPz91eH+MQU1dNPzg1hnQgKKIrUC
-NTab/CAmSQfKC1thR15sPg5bE0kwJd1AJ1AqTrYxI0VITUV8Gka3tSAp3aKZ2LBg
-gYHLI2Rv9jXe5Yx5Dckf3l+YSFp/3dSDkFOgEuZm2FfZl4vNBR+coohpB9+2jRWL
-K+4fIkCJba+Y2cEd5usJE18MTH9FU/JKDwzC+eO9SNLFUw3zGUsSwgZsBHP6kiQN
-suia9q4M5f+68kzM4+0NU8HwwyzZEtmTBhktKHijExixdvjlMAZ8hAOsFifsevI0
-02dUYvtxoHaeXh4jpYHVNnsIf/74uLagiPHtVf7+9UZV
+ACh1uH1cx+ThMJJscvhUAKFCseowrjh6msHGyNApZH6cN3F9XCXroHIE7l0/Qnzc
+2AwIYLVeuo7mxNCxCN7aj16nC+rkmcP7j5Si9LQi6f0mi+XXuL+6Wx4wve3D6fua
+vxuAvLxVxOH02zNzBvfTxhGasjnxwoOA4bKQr2/2LG+AKNpWNWBI7ufTJiWzJH3T
+u+MRWsHRfNjbgSnOCqnL2r147Ry/L4H15qQwtRErUCZ6oaJbM1zvawvRFZBJ3RYF
+WynmTQlq84OzWqy091tbR6WEsA5MY12tjRCSwTJd665ZxiGV4Hu5pZgX5iNlheII
+TPoZw1fLc4KvQxYUAmDzo3XDUJA18+t2neb1UGB1kQNQVckLwwlpBln7CFoJSisM
+gezc4baqRNIFQG19M9uBRpIyq3cucXzbnniLaVSMveY98Wp9VXvC4lNvRKuhxceD
+WKnZSHkaBd7wH8IuH2/EvVn3OYkYQutOFs9tMHfchVTLqI5XLVvYD6zsxIvs2w0K
+GsxMqeXoS8m+R/GLmppp2NTQRj4ZhuM+bKq2ywISkTEhEQzkL8IjAEv6Pfd+LLWh
+wJhbVEwWduH9cR3KNlS4OGtg0EZaln3giGu2LUIOkiBsdM5yZlhQadMdsDZriIkW
+FjcUKwwbLq/omV8WIlu0HA/klNxwnjihNcMbSKzECtEJ
 -----END CERTIFICATE-----
+Bag Attributes
+    localKeyID: 57 CC 99 3F 7C 3E D2 81 83 06 43 E2 9A 5E 12 06 9F 13 82 D0 
+Key Attributes: <No Attributes>
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQIL+MtOq0M5E8CAggA
+MBQGCCqGSIb3DQMHBAg8iC4Os1mfsgSCBMifxplyMA/zVyiT/q7wHs0JyxEHKiEA
+KrjXsc61qMTyfAaRQwr5V8Hsy2ZrwTZkwPCOWgUzFm0nTDJl3HZEgfMrHU6VnY5+
+wCc4IiXohG/D+dmaEAvOQ+ukEC5wESWmXhJcEhL8jnZEtKvos51R5MJP2YmDbN+t
+ZjM0YLwZzV8Sn/xA115Izmx831epjJYbz2aLpKVD2QlLlGiD9+XQfqet+JO0pFqF
+L+24DkSzWT8M09T0Prn208h8xPudGxlCUB9QyLHp3NLKzI1YarWSEoYt5Q2Jm6yB
+oEZwywEUQyIRmlkbKMKo6X0kxiZk+LaUeLVX2/0XYVj23rzuVuDxnTDeA6DotQGa
+kOLjzynLxyLw6ZtgeIpmyqlzAe1wGdUKe+WLQ2Km9HhSQxTfHRTKg1/F2kf9C4FX
+L2Fs/pbJyLQW5ax8YUeicmGnsb4Ge77HHxNGLvblnPCFDh1bqxacJhkdpbDFP7YC
+UewAZM24i1vpd7p0UsMlZnGula04rW4R4HJ9sIrMnVZJPgtUsku9uNr9+GdOfHLc
+lUd4M3Cnv/Hx5wdFiDBXYVVWpc21Sey6cRmqofw23ZdKlSioBvk4bpTaJxG+PDq1
+tLnGRyUPa4lbHuh1nXXQ0hSazdQ50aCMbMrjnMZ7nlnqKDeHj2mkx1gnatCOfgX8
+9DfFp/f6pv+m7iRfA5/AbT8Eu790wFbqAiouS33dDKgscoOb8Ko42nDL0uFEx32x
+i3EBCU1l78zAbYnjJi+++PeMlT20Vna/7k6pgY6Z3jnakZc0Gn0yII9FnmUKvPvz
+BIY/Wvgj05DTi2EUVdIXIGqGbsyacLvlYwm1jOo0ZFqgtRvWPVXRvjdapSF0JwSo
+c3R43HkF7E0NadTdD1E94Z/oeu+IFUzuvkbJv4XeVwOVd0jEs17l54CJeub+d4V0
+NSAmCgRqwu90YPn/2vJ3WwuX5kclcDeqy49vm/wuVzy8kLEzF1timqU4hANqP2t3
+3u9rMgMyhGhDE3FdjqPQ93Osvol61tZ9HzYFMzPRAyG6tR/7cK2Tc1WNpquciWRi
+hnMactgA+c8bINHuwzAIYiTH5e048Oe/D4Khw1uUWlyg8JRxIhpZP3zgJfpmJ9vc
++/h9HyLWD/wY96bYUDM/glYULa/WRA0NsQKgDFLxjFpjX0+liVFfcQZpqp5vjZoE
+v0LQmjkz4+9mcbPwbefcACIIxZbVySlMjNADouKhg2fQki/BUPDJJ+M3VXM7lktZ
+TqLklz1stoPMOQUXkXD56S7NPVSCBxOOtbSwicaxDGdm0YM0IQJ10uiUVyTLz34M
+12EL5WkWcxl2ywUjEFR6qbGDvG0NoAHiUA0+phxNLO6aqUag054G2WwhesLEXm/W
+V5HJ5L379YQNgxS4MkNTcGRy3WoRW5QNnfBYtsKWmI+U0Yz8/8zFbzstM0r1IlEo
+Fk/lzQcOnLJTFrepqJPNAP3wzuoMO1tGyat//mchLx5XdXmGEBhYBWokvpr71auz
+eD8u94U7pllJc0MJYqYzvdi0ohbGU1Gn8t4y9fWw7w3pzzymoamD2ytnojPjFH7y
+qsjE34t7ZQrraAcCN3p3fpMo+66xToRDgjMNStsWlybhslAx3oDfAcMWgXWm9oFg
+SdE=
+-----END ENCRYPTED PRIVATE KEY-----

builtin/bins/dkron-executor-kafka/kafka.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"crypto/tls"
 	"errors"
 	"log"
 	"strings"
@@ -66,6 +67,15 @@ func (s *Kafka) ExecuteImpl(args *dktypes.ExecuteRequest) ([]byte, error) {
 	config.Producer.Return.Successes = true
 	config.Producer.Return.Errors = true
 
+	if args.Config["tlsEnable"] == "true" {
+		config.Net.TLS.Enable = true
+
+		config.Net.TLS.Config = &tls.Config{}
+		if args.Config["tlsInsecureSkipVerify"] == "true" {
+			config.Net.TLS.Config.InsecureSkipVerify = true
+		}
+	}
+
 	brokers := strings.Split(args.Config["brokerAddress"], ",")
 	producer, err := sarama.NewSyncProducer(brokers, config)
 	if err != nil {

cmd/dkron.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/distribworks/dkron/v3/dkron"
+	"github.com/distribworks/dkron/v3/logging"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -58,6 +59,9 @@ func initConfig() {
 	viper.SetEnvKeyReplacer(replacer)
 	viper.AutomaticEnv() // read in environment variables that match
 
+	// Add hook to set error logs to stderr and regular logs to stdout
+	logrus.AddHook(&logging.LogSplitter{})
+
 	err := viper.ReadInConfig() // Find and read the config file
 	if err != nil {             // Handle errors reading the config file
 		logrus.WithError(err).Info("No valid config found: Applying default values.")