From c0b3fffec989093f8bcf9f619813a035bc78a786 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 18 Jul 2017 09:20:43 +0200 Subject: [PATCH 01/15] [skip-ci] WIP --- engine/api/businesscontext/context.go | 9 +- engine/api/cache/cache.go | 31 +++++++ engine/api/cache/interface.go | 126 -------------------------- engine/api/cache/local.go | 46 ++++++++++ engine/api/cache/redis.go | 34 ++++++- engine/api/main_routes.go | 3 + engine/api/router.go | 16 ++++ engine/api/sse.go | 95 +++++++++++++++++++ 8 files changed, 228 insertions(+), 132 deletions(-) delete mode 100644 engine/api/cache/interface.go create mode 100644 engine/api/sse.go diff --git a/engine/api/businesscontext/context.go b/engine/api/businesscontext/context.go index 8b26bdb77a..3a24c06836 100644 --- a/engine/api/businesscontext/context.go +++ b/engine/api/businesscontext/context.go @@ -6,8 +6,9 @@ import ( // Ctx gather information about http call origin type Ctx struct { - Agent string - User *sdk.User - Worker *sdk.Worker - Hatchery *sdk.Hatchery + Agent string + User *sdk.User + Worker *sdk.Worker + Hatchery *sdk.Hatchery + LastUpdateChan chan string } diff --git a/engine/api/cache/cache.go b/engine/api/cache/cache.go index d9b0bd32e6..bdfdc40c74 100644 --- a/engine/api/cache/cache.go +++ b/engine/api/cache/cache.go @@ -3,6 +3,7 @@ package cache import ( "container/list" "context" + "fmt" "strings" "sync" @@ -12,6 +13,10 @@ import ( //Status : local ok redis var Status string +type PubSub interface { + Unsubscribe(channels ...string) error +} + //Key make a key as expected func Key(args ...string) string { return strings.Join(args, ":") @@ -28,6 +33,9 @@ type Store interface { Dequeue(queueName string, value interface{}) DequeueWithContext(c context.Context, queueName string, value interface{}) QueueLen(queueName string) int + Publish(queueName string, value interface{}) + Subscribe(queueName string) PubSub + GetMessage(pb PubSub) (string, error) } //Initialize the global cache in memory, or redis @@ -128,3 +136,26 @@ func QueueLen(queueName string) int { } return s.QueueLen(queueName) } + +func Publish(queueName string, value interface{}) { + if s == nil { + return + } + s.Publish(queueName, value) +} + +func Subscribe(queueName string) PubSub { + if s == nil { + return nil + } + return s.Subscribe(queueName) +} + + +func GetMessage(pb PubSub) (string, error) { + if s == nil { + return "", fmt.Errorf("Cache > Client store is nil") + } + return s.GetMessage(pb) +} + diff --git a/engine/api/cache/interface.go b/engine/api/cache/interface.go deleted file mode 100644 index 634da5036f..0000000000 --- a/engine/api/cache/interface.go +++ /dev/null @@ -1,126 +0,0 @@ -package cache - -import ( - "time" - - "github.com/go-redis/redis" -) - -type redisClient interface { - Append(key, value string) *redis.IntCmd - BLPop(timeout time.Duration, keys ...string) *redis.StringSliceCmd - BRPop(timeout time.Duration, keys ...string) *redis.StringSliceCmd - BRPopLPush(source, destination string, timeout time.Duration) *redis.StringCmd - Decr(key string) *redis.IntCmd - DecrBy(key string, decrement int64) *redis.IntCmd - Del(keys ...string) *redis.IntCmd - Exists(key ...string) *redis.IntCmd - Expire(key string, expiration time.Duration) *redis.BoolCmd - ExpireAt(key string, tm time.Time) *redis.BoolCmd - FlushDb() *redis.StatusCmd - Get(key string) *redis.StringCmd - GetBit(key string, offset int64) *redis.IntCmd - GetRange(key string, start, end int64) *redis.StringCmd - GetSet(key string, value interface{}) *redis.StringCmd - HDel(key string, fields ...string) *redis.IntCmd - HExists(key, field string) *redis.BoolCmd - HGet(key, field string) *redis.StringCmd - HGetAll(key string) *redis.StringStringMapCmd - HIncrBy(key, field string, incr int64) *redis.IntCmd - HKeys(key string) *redis.StringSliceCmd - HLen(key string) *redis.IntCmd - HMGet(key string, fields ...string) *redis.SliceCmd - HMSet(key string, fields map[string]interface{}) *redis.StatusCmd - HSet(key, field string, value interface{}) *redis.BoolCmd - HSetNX(key, field string, value interface{}) *redis.BoolCmd - HVals(key string) *redis.StringSliceCmd - Incr(key string) *redis.IntCmd - IncrBy(key string, value int64) *redis.IntCmd - Info(section ...string) *redis.StringCmd - Keys(pattern string) *redis.StringSliceCmd - LIndex(key string, index int64) *redis.StringCmd - LInsert(key, op string, pivot, value interface{}) *redis.IntCmd - LInsertAfter(key string, pivot, value interface{}) *redis.IntCmd - LInsertBefore(key string, pivot, value interface{}) *redis.IntCmd - LLen(key string) *redis.IntCmd - LPop(key string) *redis.StringCmd - LPush(key string, values ...interface{}) *redis.IntCmd - LPushX(key string, value interface{}) *redis.IntCmd - LRange(key string, start, stop int64) *redis.StringSliceCmd - LRem(key string, count int64, value interface{}) *redis.IntCmd - LSet(key string, index int64, value interface{}) *redis.StatusCmd - LTrim(key string, start, stop int64) *redis.StatusCmd - MGet(keys ...string) *redis.SliceCmd - MSet(pairs ...interface{}) *redis.StatusCmd - MSetNX(pairs ...interface{}) *redis.BoolCmd - PExpire(key string, expiration time.Duration) *redis.BoolCmd - PExpireAt(key string, tm time.Time) *redis.BoolCmd - Ping() *redis.StatusCmd - PTTL(key string) *redis.DurationCmd - Persist(key string) *redis.BoolCmd - Pipeline() redis.Pipeliner - PubSubChannels(pattern string) *redis.StringSliceCmd - PubSubNumPat() *redis.IntCmd - Publish(channel, message string) *redis.IntCmd - RPop(key string) *redis.StringCmd - RPopLPush(source, destination string) *redis.StringCmd - RPush(key string, values ...interface{}) *redis.IntCmd - RPushX(key string, value interface{}) *redis.IntCmd - Rename(key, newkey string) *redis.StatusCmd - RenameNX(key, newkey string) *redis.BoolCmd - SAdd(key string, members ...interface{}) *redis.IntCmd - SCard(key string) *redis.IntCmd - SDiff(keys ...string) *redis.StringSliceCmd - SDiffStore(destination string, keys ...string) *redis.IntCmd - SInter(keys ...string) *redis.StringSliceCmd - SInterStore(destination string, keys ...string) *redis.IntCmd - SIsMember(key string, member interface{}) *redis.BoolCmd - SMembers(key string) *redis.StringSliceCmd - SMove(source, destination string, member interface{}) *redis.BoolCmd - SPop(key string) *redis.StringCmd - SPopN(key string, count int64) *redis.StringSliceCmd - SRandMember(key string) *redis.StringCmd - SRandMemberN(key string, count int64) *redis.StringSliceCmd - SRem(key string, members ...interface{}) *redis.IntCmd - SUnion(keys ...string) *redis.StringSliceCmd - SUnionStore(destination string, keys ...string) *redis.IntCmd - Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd - SetBit(key string, offset int64, value int) *redis.IntCmd - SetNX(key string, value interface{}, expiration time.Duration) *redis.BoolCmd - SetRange(key string, offset int64, value string) *redis.IntCmd - SetXX(key string, value interface{}, expiration time.Duration) *redis.BoolCmd - Sort(key string, sort redis.Sort) *redis.StringSliceCmd - StrLen(key string) *redis.IntCmd - TTL(key string) *redis.DurationCmd - Type(key string) *redis.StatusCmd - ZAdd(key string, members ...redis.Z) *redis.IntCmd - ZAddCh(key string, members ...redis.Z) *redis.IntCmd - ZAddNX(key string, members ...redis.Z) *redis.IntCmd - ZAddNXCh(key string, members ...redis.Z) *redis.IntCmd - ZAddXX(key string, members ...redis.Z) *redis.IntCmd - ZAddXXCh(key string, members ...redis.Z) *redis.IntCmd - ZCard(key string) *redis.IntCmd - ZCount(key, min, max string) *redis.IntCmd - ZIncr(key string, member redis.Z) *redis.FloatCmd - ZIncrBy(key string, increment float64, member string) *redis.FloatCmd - ZIncrNX(key string, member redis.Z) *redis.FloatCmd - ZIncrXX(key string, member redis.Z) *redis.FloatCmd - ZInterStore(destination string, store redis.ZStore, keys ...string) *redis.IntCmd - ZRange(key string, start, stop int64) *redis.StringSliceCmd - ZRangeByLex(key string, opt redis.ZRangeBy) *redis.StringSliceCmd - ZRangeByScore(key string, opt redis.ZRangeBy) *redis.StringSliceCmd - ZRangeByScoreWithScores(key string, opt redis.ZRangeBy) *redis.ZSliceCmd - ZRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd - ZRank(key, member string) *redis.IntCmd - ZRem(key string, members ...interface{}) *redis.IntCmd - ZRemRangeByRank(key string, start, stop int64) *redis.IntCmd - ZRemRangeByScore(key, min, max string) *redis.IntCmd - ZRevRange(key string, start, stop int64) *redis.StringSliceCmd - ZRevRangeByLex(key string, opt redis.ZRangeBy) *redis.StringSliceCmd - ZRevRangeByScore(key string, opt redis.ZRangeBy) *redis.StringSliceCmd - ZRevRangeByScoreWithScores(key string, opt redis.ZRangeBy) *redis.ZSliceCmd - ZRevRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd - ZRevRank(key, member string) *redis.IntCmd - ZScore(key, member string) *redis.FloatCmd - ZUnionStore(dest string, store redis.ZStore, keys ...string) *redis.IntCmd -} diff --git a/engine/api/cache/local.go b/engine/api/cache/local.go index 21aacdc8b5..22cfb6a2a4 100644 --- a/engine/api/cache/local.go +++ b/engine/api/cache/local.go @@ -9,6 +9,7 @@ import ( "time" "github.com/ovh/cds/sdk/log" + "fmt" ) var s Store @@ -185,3 +186,48 @@ func (s *LocalStore) DequeueWithContext(c context.Context, queueName string, val }) return } + +type LocalPubSub struct { + queueName string +} + +func (s *LocalPubSub) Unsubscribe(channels ...string) error { + return nil +} + +// Publish a msg in a queue +func (s *LocalStore) Publish(channel string, value interface{}) { + s.Mutex.Lock() + l := s.Queues[channel] + if l == nil { + s.Queues[channel] = &list.List{} + l = s.Queues[channel] + } + s.Mutex.Unlock() + b, err := json.Marshal(value) + if err != nil { + return + } + s.Mutex.Lock() + l.PushBack(b) + s.Mutex.Unlock() +} + +// Subscribe to a channel +func (s *LocalStore) Subscribe(channel string) PubSub { + return &LocalPubSub{ + queueName: channel, + } +} + +// GetMessage from a queue +func (s *LocalStore) GetMessage(pb PubSub) (string, error) { + lps, ok := pb.(*LocalPubSub) + if !ok { + return "", fmt.Errorf("GetMessage> PubSub is not a LocalPubSub. Got %T", pb) + } + var msg string + Dequeue(lps.queueName, &msg) + return msg, nil +} + diff --git a/engine/api/cache/redis.go b/engine/api/cache/redis.go index a46ad71e50..abd846d943 100644 --- a/engine/api/cache/redis.go +++ b/engine/api/cache/redis.go @@ -16,12 +16,12 @@ import ( //RedisStore a redis client and a default ttl type RedisStore struct { ttl int - Client redisClient + Client *redis.Client } //NewRedisStore initiate a new redisStore func NewRedisStore(host, password string, ttl int) (*RedisStore, error) { - var client redisClient + var client *redis.Client //if host is line master@localhost:26379,localhost:26380 => it's a redis sentinel cluster if strings.Contains(host, "@") && strings.Contains(host, ",") { @@ -210,3 +210,33 @@ func (s *RedisStore) DequeueWithContext(c context.Context, queueName string, val json.Unmarshal(b, value) } } + +// Publish a msg in a channel +func (s *RedisStore) Publish(channel string, value interface{}) { + msg, err := json.Marshal(value) + if err != nil { + log.Warning("redis.Publish> Cannot push in channel %s: %v, %s", channel, value, err) + return + } + s.Client.Publish(channel, string(msg)) +} + +// Subscribe to a channel +func (s *RedisStore) Subscribe(channel string) PubSub { + return s.Client.Subscribe(channel) +} + +// GetMessage from a redis PubSub +func (s *RedisStore) GetMessage(pb PubSub) (string, error) { + rps, ok := pb.(*redis.PubSub) + if !ok { + return "", fmt.Errorf("GetMessage> PubSub is not a redis.PubSub. Got %T", pb) + } + + redisMsg, err := rps.ReceiveMessage() + if err != nil { + log.Warning("redis.GetMessage> Cannot receive message: %s", err) + return "", err + } + return redisMsg.Payload, nil +} diff --git a/engine/api/main_routes.go b/engine/api/main_routes.go index 816cd2bc93..4f29904e33 100644 --- a/engine/api/main_routes.go +++ b/engine/api/main_routes.go @@ -5,6 +5,7 @@ import ( "path" "github.com/spf13/viper" + "github.com/ovh/cds/engine/api/cache" ) func (router *Router) init() { @@ -282,6 +283,8 @@ func (router *Router) init() { router.Handle("/worker/model/capability/type", GET(getWorkerModelCapaTypes)) router.Handle("/worker/model/{permModelID}/capability/{capa}", PUT(updateWorkerModelCapa), DELETE(deleteWorkerModelCapa)) + router.Handle("/mon/lastupdates/events", GET(lastUpdateBroker.ServeHTTP)) + //Not Found handler router.mux.NotFoundHandler = http.HandlerFunc(notFoundHandler) } diff --git a/engine/api/router.go b/engine/api/router.go index 27c2ab42e2..037aa05ba4 100644 --- a/engine/api/router.go +++ b/engine/api/router.go @@ -20,6 +20,7 @@ import ( "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" + "context" ) var ( @@ -27,6 +28,7 @@ var ( panicked bool nbPanic int lastPanic *time.Time + lastUpdateBroker *Broker ) const nbPanicsBeforeFail = 50 @@ -50,6 +52,17 @@ type routerConfig struct { needWorker bool } +func init() { + lastUpdateBroker = &Broker{ + make(map[chan string]bool), + make(chan (chan string)), + make(chan (chan string)), + make(chan string), + } + // Start processing events + lastUpdateBroker.Start() +} + // ServeAbsoluteFile Serve file to download func (r *Router) ServeAbsoluteFile(uri, path, filename string) { f := func(w http.ResponseWriter, r *http.Request) { @@ -151,6 +164,8 @@ func (r *Router) Handle(uri string, handlers ...RouterConfigParam) { } f := func(w http.ResponseWriter, req *http.Request) { + req = req.WithContext(context.Background()) + // Close indicates to close the connection after replying to this request req.Close = true // Authorization @@ -287,6 +302,7 @@ func (r *Router) Handle(uri string, handlers ...RouterConfigParam) { log.Debug("%-7s | %13v | %v", req.Method, latency, req.URL) }() + c.LastUpdateChan = lastUpdateBroker.messages if req.Method == "GET" && rc.get != nil { if err := rc.get(w, req, db, c); err != nil { WriteError(w, req, err) diff --git a/engine/api/sse.go b/engine/api/sse.go new file mode 100644 index 0000000000..bcfbafd5db --- /dev/null +++ b/engine/api/sse.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "net/http" + + "github.com/go-gorp/gorp" + + "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" + "context" +) + +type BrokerSubscribe struct { + ID int + Queue chan string +} +// Broker keep connected client of the current route, +type Broker struct { + clients map[int]chan string + newClients chan BrokerSubscribe + exitClients chan int + messages chan string +} + +func LastUpdateEventBus(ctx context.Context) { + pubSub := cache.Subscribe("lastUpdate") + if pubSub == nil { + return + } + + for { + select { + case ctx.Done(): + // TODO Stop all + + } + } +} + +// Start the broker +func (b *Broker) Start() { + go func() { + for { + select { + case s := <-b.newClients: + b.clients[s.ID] = s.Queue + case s := <-b.exitClients: + delete(b.clients, s) + close(s) + case msg := <-b.messages: + for s := range b.clients { + s <- msg + } + } + } + }() +} + +func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error{ + + // Make sure that the writer supports flushing. + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return nil + } + messageChan := make(chan string) + + // Add this client to the map of those that should receive updates + b.newClients <- messageChan + + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + for { + + select { + case <- r.Context().Done(): + // Close + b.exitClients <- messageChan + case msg, open := <-messageChan: + if !open { + break + } + + // Message must start with data: https://developer.mozilla.org/fr/docs/Server-sent_events/Using_server-sent_events + fmt.Fprintf(w, "data: %s", msg) + f.Flush() + } + } + return nil +} From 0aea43d54bcb234ea28a30cdbd5d666977e06257 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 10:26:38 +0200 Subject: [PATCH 02/15] fix(api): clean update last modified --- engine/api/application.go | 15 +- engine/api/application/application_delete.go | 12 +- engine/api/application/application_group.go | 2 +- .../api/application/application_variable.go | 5 +- engine/api/application/dao.go | 2 +- engine/api/application_variable.go | 1 + engine/api/businesscontext/context.go | 1 - engine/api/cache/cache.go | 10 +- engine/api/cache/local.go | 6 +- engine/api/cache/redis.go | 30 +++- engine/api/environment.go | 7 +- engine/api/environment/environment.go | 7 +- .../api/environment/environment_variable.go | 33 +---- engine/api/environment_group.go | 39 +++++- engine/api/environment_import.go | 4 + engine/api/environment_variable.go | 19 ++- engine/api/group.go | 50 +++++++ engine/api/group/environment_group.go | 41 +----- engine/api/group/project_group.go | 22 --- engine/api/lastupdate.go | 132 ++++++++++++++++++ engine/api/main_routes.go | 1 - engine/api/pipeline.go | 47 ++++--- engine/api/pipeline/pipeline.go | 49 +------ engine/api/pipeline/pipeline_importer.go | 2 +- engine/api/pipeline_parameter.go | 32 +++-- engine/api/project/dao.go | 18 +++ engine/api/project_group.go | 3 +- engine/api/repositories_manager.go | 75 ++++++++-- engine/api/repositoriesmanager/dao.go | 66 ++------- engine/api/router.go | 13 -- engine/api/sse.go | 95 ------------- 31 files changed, 440 insertions(+), 399 deletions(-) create mode 100644 engine/api/lastupdate.go delete mode 100644 engine/api/sse.go diff --git a/engine/api/application.go b/engine/api/application.go index d34f2911ef..d701c7fe54 100644 --- a/engine/api/application.go +++ b/engine/api/application.go @@ -399,6 +399,11 @@ func deleteApplicationHandler(w http.ResponseWriter, r *http.Request, db *gorp.D cache.DeleteAll(cache.Key("application", projectKey, "*")) cache.DeleteAll(cache.Key("pipeline", projectKey, "*")) + proj, errP := project.Load(db, projectKey, c.User) + if errP != nil { + return sdk.WrapError(errP, "deleteApplicationHandler> Cannot load project") + } + app, err := application.LoadByName(db, projectKey, applicationName, c.User) if err != nil { if err != sdk.ErrApplicationNotFound { @@ -425,14 +430,16 @@ func deleteApplicationHandler(w http.ResponseWriter, r *http.Request, db *gorp.D } defer tx.Rollback() - err = application.DeleteApplication(tx, app.ID) - if err != nil { + if err := application.DeleteApplication(tx, app.ID); err != nil { log.Warning("deleteApplicationHandler> Cannot delete application: %s\n", err) return err } - err = tx.Commit() - if err != nil { + if err := project.UpdateLastModified(tx, c.User, proj); err != nil { + return sdk.WrapError(err, "deleteApplicationHandler> Cannot update project last modified date") + } + + if err := tx.Commit(); err != nil { log.Warning("deleteApplicationHandler> Cannot commit transaction: %s\n", err) return err } diff --git a/engine/api/application/application_delete.go b/engine/api/application/application_delete.go index 5256dd3310..cb0a6a2806 100644 --- a/engine/api/application/application_delete.go +++ b/engine/api/application/application_delete.go @@ -93,15 +93,5 @@ func DeleteApplication(db gorp.SqlExecutor, applicationID int64) error { log.Warning("DeleteApplication> Cannot delete application: %s\n", err) return err } - - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE id IN ( - select project_id from application where id = $1 - ) - ` - _, err = db.Exec(query, applicationID) - return err + return nil } diff --git a/engine/api/application/application_group.go b/engine/api/application/application_group.go index 22e7ba5700..5c172c84c0 100644 --- a/engine/api/application/application_group.go +++ b/engine/api/application/application_group.go @@ -130,7 +130,7 @@ func AddGroup(db gorp.SqlExecutor, proj *sdk.Project, a *sdk.Application, u *sdk return sdk.WrapError(err, "AddGroup> Cannot add group %s in pipeline %s", g.Name, p.Pipeline.Name) } - if err := pipeline.UpdateLastModified(db, u, &p.Pipeline); err != nil { + if err := pipeline.UpdatePipelineLastModified(db, proj, &p.Pipeline, u); err != nil { return sdk.WrapError(err, "AddGroup> Cannot update pipeline %s", p.Pipeline.Name) } } diff --git a/engine/api/application/application_variable.go b/engine/api/application/application_variable.go index cf0e5e8607..d340cb94a0 100644 --- a/engine/api/application/application_variable.go +++ b/engine/api/application/application_variable.go @@ -391,10 +391,7 @@ func DeleteAllVariable(db gorp.SqlExecutor, applicationID int64) error { if err != nil { return err } - - query = "UPDATE application SET last_modified = current_timestamp WHERE id=$1" - _, err = db.Exec(query, applicationID) - return err + return nil } // AddKeyPairToApplication generate a ssh key pair and add them as application variables diff --git a/engine/api/application/dao.go b/engine/api/application/dao.go index 38c06ec734..c1052ce67b 100644 --- a/engine/api/application/dao.go +++ b/engine/api/application/dao.go @@ -196,7 +196,7 @@ func Update(db gorp.SqlExecutor, app *sdk.Application, u *sdk.User) error { // UpdateLastModified Update last_modified column in application table func UpdateLastModified(db gorp.SqlExecutor, app *sdk.Application, u *sdk.User) error { query := ` - UPDATE application SET last_modified=current_timestamp WHERE id = $1 RETURNING last_modified + UPDATE application SET last_modified = current_timestamp WHERE id = $1 RETURNING last_modified ` var lastModified time.Time err := db.QueryRow(query, app.ID).Scan(&lastModified) diff --git a/engine/api/application_variable.go b/engine/api/application_variable.go index 62d1a3daee..81db875d38 100644 --- a/engine/api/application_variable.go +++ b/engine/api/application_variable.go @@ -31,6 +31,7 @@ func getVariablesAuditInApplicationHandler(w http.ResponseWriter, r *http.Reques return WriteJSON(w, r, audits, http.StatusOK) } +// Must be deprecated have on handler to restore 1 variable at once func restoreAuditHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { vars := mux.Vars(r) key := vars["key"] diff --git a/engine/api/businesscontext/context.go b/engine/api/businesscontext/context.go index 3a24c06836..3d079004ec 100644 --- a/engine/api/businesscontext/context.go +++ b/engine/api/businesscontext/context.go @@ -10,5 +10,4 @@ type Ctx struct { User *sdk.User Worker *sdk.Worker Hatchery *sdk.Hatchery - LastUpdateChan chan string } diff --git a/engine/api/cache/cache.go b/engine/api/cache/cache.go index bdfdc40c74..949d563932 100644 --- a/engine/api/cache/cache.go +++ b/engine/api/cache/cache.go @@ -35,7 +35,7 @@ type Store interface { QueueLen(queueName string) int Publish(queueName string, value interface{}) Subscribe(queueName string) PubSub - GetMessage(pb PubSub) (string, error) + GetMessageFromSubscription(c context.Context, pb PubSub) (string, error) } //Initialize the global cache in memory, or redis @@ -137,6 +137,7 @@ func QueueLen(queueName string) int { return s.QueueLen(queueName) } +// Publish a message on a channel func Publish(queueName string, value interface{}) { if s == nil { return @@ -144,6 +145,7 @@ func Publish(queueName string, value interface{}) { s.Publish(queueName, value) } +// Subscribe to a channel func Subscribe(queueName string) PubSub { if s == nil { return nil @@ -151,11 +153,11 @@ func Subscribe(queueName string) PubSub { return s.Subscribe(queueName) } - -func GetMessage(pb PubSub) (string, error) { +// GetMessageFromSubscription Get a message from a subscription +func GetMessageFromSubscription(pb PubSub, c context.Context) (string, error) { if s == nil { return "", fmt.Errorf("Cache > Client store is nil") } - return s.GetMessage(pb) + return s.GetMessageFromSubscription(c, pb) } diff --git a/engine/api/cache/local.go b/engine/api/cache/local.go index f2f0e0cca7..13b604ba0c 100644 --- a/engine/api/cache/local.go +++ b/engine/api/cache/local.go @@ -221,14 +221,14 @@ func (s *LocalStore) Subscribe(channel string) PubSub { } } -// GetMessage from a queue -func (s *LocalStore) GetMessage(pb PubSub) (string, error) { +// GetMessageFromSubscription from a queue +func (s *LocalStore) GetMessageFromSubscription(c context.Context, pb PubSub) (string, error) { lps, ok := pb.(*LocalPubSub) if !ok { return "", fmt.Errorf("GetMessage> PubSub is not a LocalPubSub. Got %T", pb) } var msg string - Dequeue(lps.queueName, &msg) + DequeueWithContext(c, lps.queueName, &msg) return msg, nil } diff --git a/engine/api/cache/redis.go b/engine/api/cache/redis.go index abd846d943..5616ca5ff8 100644 --- a/engine/api/cache/redis.go +++ b/engine/api/cache/redis.go @@ -226,17 +226,33 @@ func (s *RedisStore) Subscribe(channel string) PubSub { return s.Client.Subscribe(channel) } -// GetMessage from a redis PubSub -func (s *RedisStore) GetMessage(pb PubSub) (string, error) { +// GetMessageFromSubscription from a redis PubSub +func (s *RedisStore) GetMessageFromSubscription(c context.Context, pb PubSub) (string, error) { rps, ok := pb.(*redis.PubSub) if !ok { - return "", fmt.Errorf("GetMessage> PubSub is not a redis.PubSub. Got %T", pb) + return "", fmt.Errorf("redis.GetMessage> PubSub is not a redis.PubSub. Got %T", pb) } + rps.ReceiveTimeout(200 * time.Millisecond) - redisMsg, err := rps.ReceiveMessage() - if err != nil { - log.Warning("redis.GetMessage> Cannot receive message: %s", err) - return "", err + var redisMsg *redis.Message + ticker := time.NewTicker(250 * time.Second).C + for redisMsg == nil { + select { + case <-ticker: + msg, err := rps.ReceiveTimeout(200 * time.Millisecond) + if err != nil { + log.Warning("redis.GetMessage> Cannot receive message: %s", err) + return "", err + } + var ok bool + redisMsg, ok = msg.(*redis.Message) + if !ok { + continue + } + + case <-c.Done(): + return "", nil + } } return redisMsg.Payload, nil } diff --git a/engine/api/environment.go b/engine/api/environment.go index 6464003e6d..6105ee9059 100644 --- a/engine/api/environment.go +++ b/engine/api/environment.go @@ -109,8 +109,7 @@ func updateEnvironmentsHandler(w http.ResponseWriter, r *http.Request, db *gorp. return sdk.ErrGroupNeedWrite } - err = group.DeleteAllGroupFromEnvironment(tx, env.ID) - if err != nil { + if err := group.DeleteAllGroupFromEnvironment(tx, env.ID); err != nil { log.Warning("updateEnvironmentsHandler> Cannot delete groups from environment %s for update: %s\n", env.Name, err) return err } @@ -416,6 +415,10 @@ func updateEnvironmentHandler(w http.ResponseWriter, r *http.Request, db *gorp.D } } + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "updateEnvironmentHandler> Cannot update environment last modified date") + } + if err := project.UpdateLastModified(tx, c.User, p); err != nil { log.Warning("updateEnvironmentHandler> Cannot update last modified date: %s\n", err) return err diff --git a/engine/api/environment/environment.go b/engine/api/environment/environment.go index 271edbf8a6..2b210765e4 100644 --- a/engine/api/environment/environment.go +++ b/engine/api/environment/environment.go @@ -195,13 +195,10 @@ func InsertEnvironment(db gorp.SqlExecutor, env *sdk.Environment) error { // UpdateEnvironment Update an environment func UpdateEnvironment(db gorp.SqlExecutor, environment *sdk.Environment) error { - var lastModified time.Time - query := `UPDATE environment SET name=$1, last_modified=current_timestamp WHERE id=$2 RETURNING last_modified` - err := db.QueryRow(query, environment.Name, environment.ID).Scan(&lastModified) - if err != nil { + query := `UPDATE environment SET name=$1 WHERE id=$2` + if _, err := db.Exec(query, environment.Name, environment.ID); err != nil { return err } - environment.LastModified = lastModified.Unix() return nil } diff --git a/engine/api/environment/environment_variable.go b/engine/api/environment/environment_variable.go index bb5b73bc51..00edcdca76 100644 --- a/engine/api/environment/environment_variable.go +++ b/engine/api/environment/environment_variable.go @@ -332,14 +332,7 @@ func InsertVariable(db gorp.SqlExecutor, environmentID int64, variable *sdk.Vari if err := InsertAudit(db, eva); err != nil { return sdk.WrapError(err, "InsertVariable> Cannot add audit") } - - query = ` - UPDATE environment - SET last_modified = current_timestamp - WHERE id=$1 - ` - _, err = db.Exec(query, environmentID) - return err + return nil } // UpdateVariable Update a variable in the given environment @@ -387,13 +380,7 @@ func UpdateVariable(db gorp.SqlExecutor, envID int64, variable *sdk.Variable, u if err := InsertAudit(db, eva); err != nil { return sdk.WrapError(err, "UpdateVariable> Cannot add audit") } - - query = ` - UPDATE environment - SET last_modified = current_timestamp - WHERE id=$1` - _, err = db.Exec(query, envID) - return err + return nil } // DeleteVariable Delete a variable from the given pipeline @@ -420,13 +407,7 @@ func DeleteVariable(db gorp.SqlExecutor, envID int64, variable *sdk.Variable, u if err := InsertAudit(db, eva); err != nil { return sdk.WrapError(err, "DeleteVariable> Cannot add audit") } - - query = ` - UPDATE environment - SET last_modified = current_timestamp - WHERE id = $1` - _, err = db.Exec(query, envID) - return err + return nil } // DeleteAllVariable Delete all variables from the given pipeline @@ -437,13 +418,7 @@ func DeleteAllVariable(db gorp.SqlExecutor, environmentID int64) error { if err != nil { return err } - - query = ` - UPDATE environment - SET last_modified = current_timestamp - WHERE id=$1` - _, err = db.Exec(query, environmentID) - return err + return nil } // InsertAudit Insert an audit for an environment variable diff --git a/engine/api/environment_group.go b/engine/api/environment_group.go index 2db2d7cb8e..37e7c26b3d 100644 --- a/engine/api/environment_group.go +++ b/engine/api/environment_group.go @@ -136,6 +136,10 @@ func addGroupsInEnvironmentHandler(w http.ResponseWriter, r *http.Request, db *g return sdk.WrapError(errP, "addGroupsInEnvironmentHandler: Cannot load project %s", env.Name) } + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "addGroupsInEnvironmentHandler: Cannot update environment last modified date") + } + if err := project.UpdateLastModified(tx, c.User, p); err != nil { return sdk.WrapError(errP, "addGroupsInEnvironmentHandler: Cannot update project %s", p.Key) } @@ -201,10 +205,37 @@ func deleteGroupFromEnvironmentHandler(w http.ResponseWriter, r *http.Request, d envName := vars["permEnvironmentName"] groupName := vars["group"] - err := group.DeleteGroupFromEnvironment(db, key, envName, groupName) - if err != nil { - log.Warning("deleteGroupFromEnvironmentHandler: Cannot delete group %s from pipeline %s: %s\n", groupName, envName, err) - return err + proj, errP := project.Load(db, key, c.User) + if errP != nil { + return sdk.WrapError(errP, "deleteGroupFromEnvironmentHandler: Cannot load project") + } + + env, errE := environment.LoadEnvironmentByName(db, key, envName) + if errE != nil { + return sdk.WrapError(errE, "deleteGroupFromEnvironmentHandler: Cannot load environment") + } + + tx, errT := db.Begin() + if errT != nil { + return sdk.WrapError(errT, "deleteGroupFromEnvironmentHandler: Cannot start transaction") } + defer tx.Rollback() + + if err := group.DeleteGroupFromEnvironment(tx, key, envName, groupName); err != nil { + return sdk.WrapError(err, "deleteGroupFromEnvironmentHandler: Cannot delete group %s from pipeline %s", groupName, envName) + } + + if err := project.UpdateLastModified(tx, c.User, proj); err != nil { + return sdk.WrapError(err, "deleteGroupFromEnvironmentHandler: Cannot update project last modified date") + } + + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "deleteGroupFromEnvironmentHandler: Cannot update environment last modified date") + } + + if err := tx.Commit(); err != nil { + return sdk.WrapError(errT, "deleteGroupFromEnvironmentHandler: Cannot commit transaction") + } + return nil } diff --git a/engine/api/environment_import.go b/engine/api/environment_import.go index aade15ede4..02f8cf23d2 100644 --- a/engine/api/environment_import.go +++ b/engine/api/environment_import.go @@ -215,6 +215,10 @@ func importIntoEnvironmentHandler(w http.ResponseWriter, r *http.Request, db *go return err } + if err := project.UpdateLastModified(db, c.User, proj); err != nil { + return sdk.WrapError(err, "importIntoEnvironmentHandler> Cannot update project last modified date") + } + close(msgChan) <-done diff --git a/engine/api/environment_variable.go b/engine/api/environment_variable.go index 448d9d36fa..96e801ca0f 100644 --- a/engine/api/environment_variable.go +++ b/engine/api/environment_variable.go @@ -30,6 +30,7 @@ func getEnvironmentsAuditHandler(w http.ResponseWriter, r *http.Request, db *gor return WriteJSON(w, r, audits, http.StatusOK) } +//Deprecated func restoreEnvironmentAuditHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { vars := mux.Vars(r) key := vars["key"] @@ -212,15 +213,19 @@ func deleteVariableFromEnvironmentHandler(w http.ResponseWriter, r *http.Request } // clear passwordfor audit - varToDelete, errV := environment.GetVariable(db, key, envName, varName, environment.WithClearPassword()) + varToDelete, errV := environment.GetVariable(tx, key, envName, varName, environment.WithClearPassword()) if errV != nil { return sdk.WrapError(errV, "deleteVariableFromEnvironmentHandler> Cannot load variable %s", varName) } - if err := environment.DeleteVariable(db, env.ID, varToDelete, c.User); err != nil { + if err := environment.DeleteVariable(tx, env.ID, varToDelete, c.User); err != nil { return sdk.WrapError(err, "deleteVariableFromEnvironmentHandler: Cannot delete %s", varName) } + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "deleteVariableFromEnvironmentHandler> Cannot update environment last modified date") + } + if err := project.UpdateLastModified(tx, c.User, p); err != nil { return sdk.WrapError(err, "deleteVariableFromEnvironmentHandler> Cannot update last modified date") } @@ -281,10 +286,14 @@ func updateVariableInEnvironmentHandler(w http.ResponseWriter, r *http.Request, return err } - if err := environment.UpdateVariable(db, env.ID, &newVar, c.User); err != nil { + if err := environment.UpdateVariable(tx, env.ID, &newVar, c.User); err != nil { return sdk.WrapError(err, "updateVariableInEnvironmentHandler: Cannot update variable %s for environment %s", varName, envName) } + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "updateVariableInEnvironmentHandler: Cannot update environment last modified date") + } + if err := project.UpdateLastModified(tx, c.User, p); err != nil { return sdk.WrapError(err, "updateVariableInEnvironmentHandler: Cannot update last modified date") } @@ -364,6 +373,10 @@ func addVariableInEnvironmentHandler(w http.ResponseWriter, r *http.Request, db return sdk.WrapError(errInsert, "addVariableInEnvironmentHandler: Cannot add variable %s in environment %s", varName, envName) } + if err := environment.UpdateLastModified(tx, c.User, env); err != nil { + return sdk.WrapError(err, "addVariableInEnvironmentHandler> Cannot update environment last modified date") + } + if err := project.UpdateLastModified(tx, c.User, p); err != nil { return sdk.WrapError(err, "addVariableInEnvironmentHandler: Cannot update last modified date") } diff --git a/engine/api/group.go b/engine/api/group.go index 55491cb839..252b9c2c86 100644 --- a/engine/api/group.go +++ b/engine/api/group.go @@ -11,6 +11,10 @@ import ( "github.com/ovh/cds/engine/api/user" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" + "github.com/ovh/cds/engine/api/project" + "github.com/ovh/cds/engine/api/application" + "github.com/ovh/cds/engine/api/pipeline" + "github.com/ovh/cds/engine/api/environment" ) func getGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { @@ -40,6 +44,22 @@ func deleteGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, return sdk.WrapError(errl, "deleteGroupHandler: Cannot load %s", name) } + if err := project.LoadPermissions(db, g); err != nil { + return sdk.WrapError(err, "deleteGroupHandler: Cannot load projects for group") + } + + if err := application.LoadPermissions(db, g); err != nil { + return sdk.WrapError(err, "deleteGroupHandler: Cannot load application for group") + } + + if err := pipeline.LoadPipelineByGroup(db, g); err != nil { + return sdk.WrapError(err, "deleteGroupHandler: Cannot load pipeline for group") + } + + if err := environment.LoadEnvironmentByGroup(db, g); err != nil { + return sdk.WrapError(err, "deleteGroupHandler: Cannot load environment for group") + } + tx, errb := db.Begin() if errb != nil { return sdk.WrapError(errb, "deleteGroupHandler> cannot start transaction") @@ -50,6 +70,36 @@ func deleteGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, return sdk.WrapError(err, "deleteGroupHandler> cannot delete group") } + for _, pg := range g.ProjectGroups { + if err := project.UpdateLastModified(tx, c.User, &pg.Project); err != nil { + return sdk.WrapError(err, "deleteGroupHandler> Cannot update project last modified date") + } + } + + for _, pg := range g.ApplicationGroups { + if err := application.UpdateLastModified(tx, &pg.Application, c.User); err != nil { + return sdk.WrapError(err, "deleteGroupHandler> Cannot update application last modified date") + } + } + + + for _, pg := range g.PipelineGroups { + p := &sdk.Project{ + Key: pg.Pipeline.ProjectKey, + } + if err := pipeline.UpdatePipelineLastModified(tx, p, &pg.Pipeline, c.User); err != nil { + return sdk.WrapError(err, "deleteGroupHandler> Cannot update pipeline last modified date") + } + } + + + for _, pg := range g.EnvironmentGroups { + if err := environment.UpdateLastModified(tx, c.User, &pg.Environment); err != nil { + return sdk.WrapError(err, "deleteGroupHandler> Cannot update environment last modified date") + } + } + + if err := tx.Commit(); err != nil { return sdk.WrapError(err, "deleteGroupHandler> cannot commit transaction") } diff --git a/engine/api/group/environment_group.go b/engine/api/group/environment_group.go index 5bb98b05b9..1ffcfc6f1f 100644 --- a/engine/api/group/environment_group.go +++ b/engine/api/group/environment_group.go @@ -71,34 +71,13 @@ func UpdateGroupRoleInEnvironment(db gorp.SqlExecutor, key, envName, groupName s if _, err := db.Exec(query, role, envName, key, groupName); err != nil { return err } - - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE id IN ( - SELECT id - FROM project - WHERE projectKey = $1 - ) - ` - _, err := db.Exec(query, key) - return err + return nil } // DeleteAllGroupFromEnvironment remove all group from the given environment func DeleteAllGroupFromEnvironment(db gorp.SqlExecutor, environmentID int64) error { - // Update environment - query := ` - UPDATE environment - SET last_modified = current_timestamp - WHERE id = $1 - ` - if _, err := db.Exec(query, environmentID); err != nil { - return err - } //Delete association - query = `DELETE FROM environment_group + query := `DELETE FROM environment_group WHERE environment_id=$1` _, err := db.Exec(query, environmentID) return err @@ -106,21 +85,7 @@ func DeleteAllGroupFromEnvironment(db gorp.SqlExecutor, environmentID int64) err // DeleteGroupFromEnvironment removes access to environment to group members func DeleteGroupFromEnvironment(db gorp.SqlExecutor, key, envName, groupName string) error { - // Update project - query := ` - UPDATE project - SET last_modified = current_timestamp - WHERE id IN ( - SELECT id - FROM project - WHERE projectKey = $1 - ) - ` - if _, err := db.Exec(query, key); err != nil { - return err - } - - query = `DELETE FROM environment_group + query := `DELETE FROM environment_group USING environment, project, "group" WHERE environment.id = environment_group.environment_id AND environment.project_id = project.id AND "group".id = environment_group.group_id AND environment.name = $1 AND project.projectKey = $2 AND "group".name = $3` diff --git a/engine/api/group/project_group.go b/engine/api/group/project_group.go index a6f341f280..2d10d84808 100644 --- a/engine/api/group/project_group.go +++ b/engine/api/group/project_group.go @@ -53,34 +53,12 @@ func InsertGroupInProject(db gorp.SqlExecutor, projectID, groupID int64, role in func DeleteGroupProjectByProject(db gorp.SqlExecutor, projectID int64) error { query := `DELETE FROM project_group WHERE project_id=$1` _, err := db.Exec(query, projectID) - if err != nil { - return err - } - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE id=$1 - ` - _, err = db.Exec(query, projectID) return err } func deleteGroupProjectByGroup(db gorp.SqlExecutor, group *sdk.Group) error { query := `DELETE FROM project_group WHERE group_id=$1` _, err := db.Exec(query, group.ID) - if err != nil { - return err - } - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE id in ( - select project_id from project_group where group_id=$1 - ) - ` - _, err = db.Exec(query, group.ID) return err } diff --git a/engine/api/lastupdate.go b/engine/api/lastupdate.go new file mode 100644 index 0000000000..f3d8f61409 --- /dev/null +++ b/engine/api/lastupdate.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/go-gorp/gorp" + + "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +// LastUpdateBrokerSubscribe is the information need to to subscribe +type LastUpdateBrokerSubscribe struct { + User *sdk.User + Queue chan string +} + +// LastUpdateBroker keeps connected client of the current route, +type LastUpdateBroker struct { + clients map[chan string]*sdk.User + newClients chan LastUpdateBrokerSubscribe + exitClients chan chan string + messages chan string +} + +var lastUpdateBroker *LastUpdateBroker +var cacheQueueName = "lastUpdates" + +func Initialize(c context.Context, DBFunc func() *gorp.DbMap) { + lastUpdateBroker = &LastUpdateBroker{ + make(map[chan string]*sdk.User), + make(chan LastUpdateBrokerSubscribe), + make(chan (chan string)), + make(chan string), + } + + // Start cache Subscription + go CacheSubscribe(c, lastUpdateBroker.messages) + + // Start processing events + go lastUpdateBroker.Start(c, DBFunc()) +} + +// CacheSubscribe subscribe to a channel and push received message in a channel +func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { + pubSub := cache.Subscribe(cacheQueueName) + tick := time.NewTicker(250 * time.Millisecond).C + for { + select { + case <-c.Done(): + if c.Err() != nil { + log.Warning("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) + return + } + case <-tick: + msg, err := cache.GetMessageFromSubscription(pubSub, c) + if err != nil { + log.Warning("lastUpdate.CacheSubscribe> Cannot get message") + time.Sleep(5 * time.Second) + continue + } + cacheMsgChan <- msg + } + } +} + +// Start the broker +func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { + + for { + select { + case s := <-b.newClients: + // Register new client + b.clients[s.Queue] = s.User + case s := <-b.exitClients: + // Deleting client + delete(b.clients, s) + close(s) + case msg := <-b.messages: + //Receive new message + for s := range b.clients { + s <- msg + } + } + } + +} + +func (b *LastUpdateBroker) ServeHTTP(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { + + // Make sure that the writer supports flushing. + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return nil + } + messageChan := LastUpdateBrokerSubscribe{ + User: c.User, + Queue: make(chan string), + } + + // Add this client to the map of those that should receive updates + b.newClients <- messageChan + + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + for { + + select { + case <-r.Context().Done(): + // Close + b.exitClients <- messageChan.Queue + case msg, open := <-messageChan.Queue: + if !open { + break + } + + // Message must start with data: https://developer.mozilla.org/fr/docs/Server-sent_events/Using_server-sent_events + fmt.Fprintf(w, "data: %s", msg) + f.Flush() + } + } + return nil +} diff --git a/engine/api/main_routes.go b/engine/api/main_routes.go index 4f29904e33..410920d485 100644 --- a/engine/api/main_routes.go +++ b/engine/api/main_routes.go @@ -5,7 +5,6 @@ import ( "path" "github.com/spf13/viper" - "github.com/ovh/cds/engine/api/cache" ) func (router *Router) init() { diff --git a/engine/api/pipeline.go b/engine/api/pipeline.go index f300c8a318..c156874800 100644 --- a/engine/api/pipeline.go +++ b/engine/api/pipeline.go @@ -499,6 +499,11 @@ func updatePipelineHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMa key := vars["key"] name := vars["permPipelineKey"] + proj, errP := project.Load(db, key, c.User) + if errP != nil { + return sdk.WrapError(errP, "updatePipelineHandler> Cannot load project") + } + var p sdk.Pipeline if err := UnmarshalBody(r, &p); err != nil { return sdk.WrapError(err, "updatePipelineHandler> Cannot read body") @@ -528,10 +533,8 @@ func updatePipelineHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMa return sdk.WrapError(err, "updatePipelineHandler> cannot update pipeline %s", name) } - // Update project - proj, errP := project.Load(tx, key, c.User) - if errP != nil { - return sdk.WrapError(errP, "updatePipelineHandler> cannot load project %s", key) + if err := pipeline.UpdatePipelineLastModified(tx, proj, pipelineDB, c.User); err != nil { + return sdk.WrapError(err, "updatePipelineHandler> Cannot update pipeline last modified date") } if err := project.UpdateLastModified(tx, c.User, proj); err != nil { @@ -618,7 +621,7 @@ func addPipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *busi defer tx.Rollback() p.ProjectID = proj.ID - if err := pipeline.InsertPipeline(tx, &p, c.User); err != nil { + if err := pipeline.InsertPipeline(tx, proj, &p, c.User); err != nil { return sdk.WrapError(err, "Cannot insert pipeline") } @@ -766,40 +769,42 @@ func deletePipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *b projectKey := vars["key"] pipelineName := vars["permPipelineKey"] + proj, errP := project.Load(db, projectKey, c.User) + if errP != nil { + return sdk.WrapError(errP, "deletePipeline> Cannot load project") + } + + p, err := pipeline.LoadPipeline(db, projectKey, pipelineName, false) if err != nil { - if err != sdk.ErrPipelineNotFound { - log.Warning("deletePipeline> Cannot load pipeline %s: %s\n", pipelineName, err) - } - return err + return sdk.WrapError(err, "deletePipeline> Cannot load pipeline %s", pipelineName) } used, err := application.CountPipeline(db, p.ID) if err != nil { - log.Warning("deletePipeline> Cannot check if pipeline is used by an application: %s\n", err) - return err + return sdk.WrapError(err, "deletePipeline> Cannot check if pipeline is used by an application") } if used { - log.Warning("deletePipeline> Cannot delete a pipeline used by at least 1 application\n") - return sdk.ErrPipelineHasApplication + return sdk.WrapError(sdk.ErrPipelineHasApplication, "deletePipeline> Cannot delete a pipeline used by at least 1 application") } - tx, err := db.Begin() - if err != nil { - log.Warning("deletePipeline> Cannot begin transaction: %s\n", err) - return err + tx, errT := db.Begin() + if errT != nil { + return sdk.WrapError(errT, "deletePipeline> Cannot begin transaction") } defer tx.Rollback() - err = pipeline.DeletePipeline(tx, p.ID, c.User.ID) - if err != nil { + if err := pipeline.DeletePipeline(tx, p.ID, c.User.ID); err != nil { log.Warning("deletePipeline> Cannot delete pipeline %s: %s\n", pipelineName, err) return err } - err = tx.Commit() - if err != nil { + if err := project.UpdateLastModified(db, c.User, proj); err != nil { + return sdk.WrapError(err, "deletePipeline> Cannot update project last modified date") + } + + if err := tx.Commit(); err != nil { log.Warning("deletePipeline> Cannot commit transaction: %s\n", err) return err } diff --git a/engine/api/pipeline/pipeline.go b/engine/api/pipeline/pipeline.go index eca167dbe0..48f91ab1f0 100644 --- a/engine/api/pipeline/pipeline.go +++ b/engine/api/pipeline/pipeline.go @@ -114,21 +114,8 @@ func DeletePipeline(db gorp.SqlExecutor, pipelineID int64, userID int64) error { return err } - // Update project - query := ` - UPDATE project - SET last_modified = current_timestamp - WHERE id in ( - SELECT project_id from pipeline WHERE id = $1 - ) - ` - - if _, err := db.Exec(query, pipelineID); err != nil { - return err - } - // Delete pipeline groups - query = `DELETE FROM pipeline_group WHERE pipeline_id = $1` + query := `DELETE FROM pipeline_group WHERE pipeline_id = $1` if _, err := db.Exec(query, pipelineID); err != nil { return err } @@ -316,42 +303,16 @@ func LoadGroupByPipeline(db gorp.SqlExecutor, pipeline *sdk.Pipeline) error { return nil } -// UpdateLastModified updates last_modified on pipeline -func UpdateLastModified(db gorp.SqlExecutor, u *sdk.User, pip *sdk.Pipeline) error { - if u != nil { - cache.SetWithTTL(cache.Key("lastModified", pip.ProjectKey, "pipeline", pip.Name), sdk.LastModification{ - Name: pip.Name, - Username: u.Username, - LastModified: time.Now().Unix(), - }, 0) - } - - query := `UPDATE pipeline SET last_modified = current_timestamp WHERE id=$1` - _, err := db.Exec(query, pip.ID) - return err -} - // UpdatePipeline update the pipeline func UpdatePipeline(db gorp.SqlExecutor, p *sdk.Pipeline) error { - // Update project - query := ` - UPDATE project - SET last_modified = current_timestamp - WHERE id IN (SELECT project_id from pipeline WHERE id = $1) - ` - _, err := db.Exec(query, p.ID) - if err != nil { - return err - } - //Update pipeline - query = `UPDATE pipeline SET name=$1, type=$2, last_modified = current_timestamp WHERE id=$3` - _, err = db.Exec(query, p.Name, string(p.Type), p.ID) + query := `UPDATE pipeline SET name=$1, type=$2 WHERE id=$3` + _, err := db.Exec(query, p.Name, string(p.Type), p.ID) return err } // InsertPipeline inserts pipeline informations in database -func InsertPipeline(db gorp.SqlExecutor, p *sdk.Pipeline, u *sdk.User) error { +func InsertPipeline(db gorp.SqlExecutor, proj *sdk.Project, p *sdk.Pipeline, u *sdk.User) error { query := `INSERT INTO pipeline (name, project_id, type, last_modified) VALUES ($1,$2,$3, current_timestamp) RETURNING id` if p.Name == "" { @@ -376,7 +337,7 @@ func InsertPipeline(db gorp.SqlExecutor, p *sdk.Pipeline, u *sdk.User) error { } } - return UpdateLastModified(db, u, p) + return UpdatePipelineLastModified(db, proj, p, u) } // ExistPipeline Check if the given pipeline exist in database diff --git a/engine/api/pipeline/pipeline_importer.go b/engine/api/pipeline/pipeline_importer.go index ea73d968f8..a8ff5683d6 100644 --- a/engine/api/pipeline/pipeline_importer.go +++ b/engine/api/pipeline/pipeline_importer.go @@ -251,7 +251,7 @@ func Import(db gorp.SqlExecutor, proj *sdk.Project, pip *sdk.Pipeline, msgChan c func importNew(db gorp.SqlExecutor, proj *sdk.Project, pip *sdk.Pipeline, u *sdk.User) error { log.Debug("pipeline.importNew> Creating pipeline %s", pip.Name) //Insert pipeline - if err := InsertPipeline(db, pip, u); err != nil { + if err := InsertPipeline(db, proj, pip, u); err != nil { return err } diff --git a/engine/api/pipeline_parameter.go b/engine/api/pipeline_parameter.go index 0266d93290..0e60da5c7c 100644 --- a/engine/api/pipeline_parameter.go +++ b/engine/api/pipeline_parameter.go @@ -11,6 +11,7 @@ import ( "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" + "github.com/ovh/cds/engine/api/application" ) func getParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { @@ -86,6 +87,12 @@ func updateParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, d key := vars["key"] pipelineName := vars["permPipelineKey"] + + proj, errP := project.Load(db, key, c.User) + if errP != nil { + return sdk.WrapError(errP, "updateParametersInPipelineHandler: Cannot load project") + } + var pipParams []sdk.Parameter if err := UnmarshalBody(r, &pipParams); err != nil { return err @@ -163,26 +170,21 @@ func updateParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, d } } - query := ` - UPDATE application - SET last_modified = current_timestamp - FROM application_pipeline - WHERE application_pipeline.application_id = application.id - AND application_pipeline.pipeline_id = $1 - ` - if _, err := tx.Exec(query, pip.ID); err != nil { - log.Warning("UpdatePipelineParameters> Cannot update linked application [%d]: %s", pip.ID, err) + if err := pipeline.UpdatePipelineLastModified(tx, proj, pip, c.User); err != nil { + + log.Warning("UpdatePipelineParameters> Cannot update pipeline last_modified date: %s", err) return err } - proj, errproj := project.Load(db, key, c.User) - if errproj != nil { - return sdk.WrapError(errproj, "UpdatePipelineParameters> unable to load project") + apps, errA := application.LoadByPipeline(tx, pip.ID, c.User) + if errA != nil { + return sdk.WrapError(errA, "UpdatePipelineParameters> Cannot load applications using pipeline") } - if err := pipeline.UpdatePipelineLastModified(tx, proj, pip, c.User); err != nil { - log.Warning("UpdatePipelineParameters> Cannot update pipeline last_modified date: %s", err) - return err + for _, app := range apps { + if err := application.UpdateLastModified(tx, &app, c.User); err != nil { + return sdk.WrapError(errA, "UpdatePipelineParameters> Cannot update application last modified date") + } } if err := tx.Commit(); err != nil { diff --git a/engine/api/project/dao.go b/engine/api/project/dao.go index ecf0f408c3..2b403a1580 100644 --- a/engine/api/project/dao.go +++ b/engine/api/project/dao.go @@ -12,6 +12,7 @@ import ( "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" + "encoding/json" ) // LoadAll returns all projects @@ -149,6 +150,18 @@ func UpdateLastModified(db gorp.SqlExecutor, u *sdk.User, proj *sdk.Project) err _, err := db.Exec("update project set last_modified = $2 where projectkey = $1", proj.Key, t) proj.LastModified = t + + projLastUpdate := sdk.ProjectLastUpdates{ + LastModification: sdk.LastModification{ + Name: proj.Key, + LastModified: t.Unix(), + Username: u.Username, + }, + } + b, errP := json.Marshal(projLastUpdate) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } return err } @@ -205,6 +218,11 @@ var LoadOptions = struct { WithApplicationVariables: &loadApplicationVariables, } +// LoadByID returns a project with all its variables and applications given a user. It can also returns pipelines, environments, groups, permission, and repositorires manager. See LoadOptions +func LoadByID(db gorp.SqlExecutor, id int64, u *sdk.User, opts ...LoadOptionFunc) (*sdk.Project, error) { + return load(db, u, opts, "select * from project where id = $1", id) +} + // Load returns a project with all its variables and applications given a user. It can also returns pipelines, environments, groups, permission, and repositorires manager. See LoadOptions func Load(db gorp.SqlExecutor, key string, u *sdk.User, opts ...LoadOptionFunc) (*sdk.Project, error) { return load(db, u, opts, "select * from project where projectkey = $1", key) diff --git a/engine/api/project_group.go b/engine/api/project_group.go index e6d58cc490..629caeab3e 100644 --- a/engine/api/project_group.go +++ b/engine/api/project_group.go @@ -155,8 +155,7 @@ func updateGroupsInProject(w http.ResponseWriter, r *http.Request, db *gorp.DbMa } defer tx.Rollback() - err = group.DeleteGroupProjectByProject(tx, p.ID) - if err != nil { + if err := group.DeleteGroupProjectByProject(tx, p.ID); err != nil { return sdk.WrapError(err, "updateGroupsInProject: Cannot delete groups from project %s", p.Name) } diff --git a/engine/api/repositories_manager.go b/engine/api/repositories_manager.go index 9dfb771707..1caf9c0c85 100644 --- a/engine/api/repositories_manager.go +++ b/engine/api/repositories_manager.go @@ -23,6 +23,7 @@ import ( "github.com/ovh/cds/engine/api/poller" "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/engine/api/repositoriesmanager" + "github.com/ovh/cds/engine/api/user" "github.com/ovh/cds/engine/api/workflow" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" @@ -96,6 +97,11 @@ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *go projectKey := vars["permProjectKey"] rmName := vars["name"] + proj, errP := project.Load(db, projectKey, c.User) + if errP != nil { + return sdk.WrapError(errP, "repositoriesManagerAuthorize> Cannot load project") + } + //Load the repositories manager from the DB rm, err := repositoriesmanager.LoadForProject(db, projectKey, rmName) var lastModified time.Time @@ -116,13 +122,15 @@ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *go } defer tx.Rollback() - var errI error - lastModified, errI = repositoriesmanager.InsertForProject(tx, rm, projectKey) - if errI != nil { + if errI := repositoriesmanager.InsertForProject(tx, rm, projectKey); errI != nil { log.Warning("repositoriesManagerAuthorize> error while inserting repositories manager for project %s: %s\n", projectKey, errI) return errI } + if err := project.UpdateLastModified(tx, c.User, proj); err != nil { + return sdk.WrapError(err, "repositoriesManagerAuthorize> Cannot update project last modified") + } + if err := tx.Commit(); err != nil { log.Warning("repositoriesManagerAuthorize> Cannot commit transaction %s\n", err) return err @@ -146,6 +154,7 @@ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *go "repositories_manager": rmName, "url": url, "request_token": token, + "username": c.User.Username, } cache.Set(cache.Key("reposmanager", "oauth", token), data) @@ -171,6 +180,17 @@ func repositoriesManagerOAuthCallbackHandler(w http.ResponseWriter, r *http.Requ cache.Get(cache.Key("reposmanager", "oauth", state), &data) projectKey := data["project_key"] rmName := data["repositories_manager"] + username := data["username"] + + u, errU := user.LoadUserWithoutAuth(db, username) + if errU != nil { + return sdk.WrapError(errU, "repositoriesManagerAuthorizeCallback> Cannot load user %s", username) + } + + proj, errP := project.Load(db, projectKey, u) + if errP != nil { + return sdk.WrapError(errP, "repositoriesManagerAuthorizeCallback> Cannot load project") + } //Load the repositories manager from the DB rm, err := repositoriesmanager.LoadForProject(db, projectKey, rmName) @@ -195,13 +215,27 @@ func repositoriesManagerOAuthCallbackHandler(w http.ResponseWriter, r *http.Requ "access_token_secret": accessTokenSecret, } - if err := repositoriesmanager.SaveDataForProject(db, rm, projectKey, result); err != nil { + tx, errT := db.Begin() + if errT != nil { + return sdk.WrapError(errT, "repositoriesManagerAuthorizeCallback> Cannot start transaction") + } + defer tx.Rollback() + + if err := repositoriesmanager.SaveDataForProject(tx, rm, projectKey, result); err != nil { log.Warning("repositoriesManagerAuthorizeCallback> Error with SaveDataForProject: %s", err) return err } + if err := project.UpdateLastModified(tx, u, proj); err != nil { + return sdk.WrapError(err, "repositoriesManagerAuthorizeCallback> Cannot update project last modified date") + } + + if err := tx.Commit(); err != nil { + return sdk.WrapError(errT, "repositoriesManagerAuthorizeCallback> Cannot commit transaction") + } + //Redirect on UI advanced project page - url := fmt.Sprintf("%s/#/project/%s?tab=advanced", baseURL, projectKey) + url := fmt.Sprintf("%s/project/%s?tab=advanced", baseURL, projectKey) http.Redirect(w, r, url, http.StatusTemporaryRedirect) return nil @@ -283,7 +317,7 @@ func deleteRepositoriesManagerHandler(w http.ResponseWriter, r *http.Request, db } - //Load the repositories manager from the DB + // Load the repositories manager from the DB rm, err := repositoriesmanager.LoadForProject(db, projectKey, rmName) if err != nil { log.Warning("deleteRepositoriesManagerHandler> error loading %s-%s: %s\n", projectKey, rmName, err) @@ -305,8 +339,11 @@ func deleteRepositoriesManagerHandler(w http.ResponseWriter, r *http.Request, db } - err = tx.Commit() - if err != nil { + if err := project.UpdateLastModified(tx, c.User, p); err != nil { + return sdk.WrapError(err, "deleteRepositoriesManagerHandler> Cannot update project last modified date") + } + + if err := tx.Commit(); err != nil { log.Warning("deleteRepositoriesManagerHandler> Cannot commit transaction: %s\n", err) return err @@ -415,12 +452,26 @@ func attachRepositoriesManager(w http.ResponseWriter, r *http.Request, db *gorp. app.RepositoriesManager = rm app.RepositoryFullname = fullname - if err := repositoriesmanager.InsertForApplication(db, app, projectKey); err != nil { + tx, errT := db.Begin() + if errT != nil { + return sdk.WrapError(errT, "attachRepositoriesManager> Cannot start transaction") + } + defer tx.Rollback() + + if err := repositoriesmanager.InsertForApplication(tx, app, projectKey); err != nil { log.Warning("attachRepositoriesManager> Cannot insert for application: %s", err) return err } + if err := application.UpdateLastModified(tx, app, c.User); err != nil { + return sdk.WrapError(err, "attachRepositoriesManager> Cannot update application last modified date") + } + + if err := tx.Commit(); err != nil { + return sdk.WrapError(err, "attachRepositoriesManager> Cannot commit transaction") + } + return WriteJSON(w, r, app, http.StatusOK) } @@ -447,7 +498,7 @@ func detachRepositoriesManager(w http.ResponseWriter, r *http.Request, db *gorp. tx, err := db.Begin() defer tx.Rollback() - if err := repositoriesmanager.DeleteForApplication(tx, projectKey, app); err != nil { + if err := repositoriesmanager.DeleteForApplication(tx, app); err != nil { log.Warning("detachRepositoriesManager> Cannot delete for application: %s", err) return err @@ -475,6 +526,10 @@ func detachRepositoriesManager(w http.ResponseWriter, r *http.Request, db *gorp. } + if err := application.UpdateLastModified(tx, app, c.User); err != nil { + return sdk.WrapError(err, "detachRepositoriesManager> Cannot update application last modified date") + } + if err := tx.Commit(); err != nil { log.Warning("detachRepositoriesManager> Cannot commit transaction: %s", err) return err diff --git a/engine/api/repositoriesmanager/dao.go b/engine/api/repositoriesmanager/dao.go index f254e1987c..1a37738e19 100644 --- a/engine/api/repositoriesmanager/dao.go +++ b/engine/api/repositoriesmanager/dao.go @@ -4,11 +4,9 @@ import ( "database/sql" "encoding/json" "strings" - "time" "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" ) @@ -175,8 +173,7 @@ func Update(db gorp.SqlExecutor, rm *sdk.RepositoriesManager) error { } //InsertForProject associates a repositories manager with a project -func InsertForProject(db gorp.SqlExecutor, rm *sdk.RepositoriesManager, projectKey string) (time.Time, error) { - var lastModified time.Time +func InsertForProject(db gorp.SqlExecutor, rm *sdk.RepositoriesManager, projectKey string) error { query := `INSERT INTO repositories_manager_project (id_repositories_manager, id_project) VALUES ( @@ -185,19 +182,7 @@ func InsertForProject(db gorp.SqlExecutor, rm *sdk.RepositoriesManager, projectK )` _, err := db.Exec(query, rm.ID, projectKey) - if err != nil { - return lastModified, err - } - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE projectkey = $1 RETURNING last_modified - ` - if err = db.QueryRow(query, projectKey).Scan(&lastModified); err != nil { - return lastModified, err - } - return lastModified, nil + return err } //DeleteForProject removes association between a repositories manager and a project @@ -213,17 +198,6 @@ func DeleteForProject(db gorp.SqlExecutor, rm *sdk.RepositoriesManager, project if err != nil { return err } - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE projectkey = $1 RETURNING last_modified - ` - var lastModified time.Time - if err = db.QueryRow(query, project.Key).Scan(&lastModified); err != nil { - return err - } - project.LastModified = lastModified return nil } @@ -241,15 +215,6 @@ func SaveDataForProject(db gorp.SqlExecutor, rm *sdk.RepositoriesManager, projec if err != nil { return err } - // Update project - query = ` - UPDATE project - SET last_modified = current_timestamp - WHERE projectkey = $1 - ` - if _, err = db.Exec(query, projectKey); err != nil { - return err - } return nil } @@ -291,46 +256,31 @@ func InsertForApplication(db gorp.SqlExecutor, app *sdk.Application, projectKey query := `UPDATE application SET repositories_manager_id = $1, - repo_fullname = $2, - last_modified = current_timestamp + repo_fullname = $2 WHERE id = $3 - RETURNING last_modified ` - var lastModified time.Time - err := db.QueryRow(query, app.RepositoriesManager.ID, app.RepositoryFullname, app.ID).Scan(&lastModified) - if err != nil { + if _, err := db.Exec(query, app.RepositoriesManager.ID, app.RepositoryFullname, app.ID); err != nil { return err } - app.LastModified = lastModified - - k := cache.Key("application", projectKey, "*"+app.Name+"*") - cache.DeleteAll(k) return nil } //DeleteForApplication removes association between a repositories manager and an application //it deletes the corresponding line in repositories_manager_project -func DeleteForApplication(db gorp.SqlExecutor, projectKey string, app *sdk.Application) error { +func DeleteForApplication(db gorp.SqlExecutor, app *sdk.Application) error { query := `UPDATE application SET repositories_manager_id = NULL, - repo_fullname = '', - last_modified = current_timestamp + repo_fullname = '' WHERE id = $1 - RETURNING last_modified ` - var lastModified time.Time - err := db.QueryRow(query, app.ID).Scan(&lastModified) - if err != nil { + + if _, err := db.Exec(query, app.ID); err != nil { return err } - app.LastModified = lastModified - - k := cache.Key("application", projectKey, "*"+app.Name+"*") - cache.DeleteAll(k) return nil } diff --git a/engine/api/router.go b/engine/api/router.go index 037aa05ba4..dc715adb03 100644 --- a/engine/api/router.go +++ b/engine/api/router.go @@ -28,7 +28,6 @@ var ( panicked bool nbPanic int lastPanic *time.Time - lastUpdateBroker *Broker ) const nbPanicsBeforeFail = 50 @@ -52,17 +51,6 @@ type routerConfig struct { needWorker bool } -func init() { - lastUpdateBroker = &Broker{ - make(map[chan string]bool), - make(chan (chan string)), - make(chan (chan string)), - make(chan string), - } - // Start processing events - lastUpdateBroker.Start() -} - // ServeAbsoluteFile Serve file to download func (r *Router) ServeAbsoluteFile(uri, path, filename string) { f := func(w http.ResponseWriter, r *http.Request) { @@ -302,7 +290,6 @@ func (r *Router) Handle(uri string, handlers ...RouterConfigParam) { log.Debug("%-7s | %13v | %v", req.Method, latency, req.URL) }() - c.LastUpdateChan = lastUpdateBroker.messages if req.Method == "GET" && rc.get != nil { if err := rc.get(w, req, db, c); err != nil { WriteError(w, req, err) diff --git a/engine/api/sse.go b/engine/api/sse.go deleted file mode 100644 index bcfbafd5db..0000000000 --- a/engine/api/sse.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/businesscontext" - "github.com/ovh/cds/engine/api/cache" - "context" -) - -type BrokerSubscribe struct { - ID int - Queue chan string -} -// Broker keep connected client of the current route, -type Broker struct { - clients map[int]chan string - newClients chan BrokerSubscribe - exitClients chan int - messages chan string -} - -func LastUpdateEventBus(ctx context.Context) { - pubSub := cache.Subscribe("lastUpdate") - if pubSub == nil { - return - } - - for { - select { - case ctx.Done(): - // TODO Stop all - - } - } -} - -// Start the broker -func (b *Broker) Start() { - go func() { - for { - select { - case s := <-b.newClients: - b.clients[s.ID] = s.Queue - case s := <-b.exitClients: - delete(b.clients, s) - close(s) - case msg := <-b.messages: - for s := range b.clients { - s <- msg - } - } - } - }() -} - -func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error{ - - // Make sure that the writer supports flushing. - f, ok := w.(http.Flusher) - if !ok { - http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) - return nil - } - messageChan := make(chan string) - - // Add this client to the map of those that should receive updates - b.newClients <- messageChan - - // Set the headers related to event streaming. - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - - for { - - select { - case <- r.Context().Done(): - // Close - b.exitClients <- messageChan - case msg, open := <-messageChan: - if !open { - break - } - - // Message must start with data: https://developer.mozilla.org/fr/docs/Server-sent_events/Using_server-sent_events - fmt.Fprintf(w, "data: %s", msg) - f.Flush() - } - } - return nil -} From 42602b4d2f0d32674c1e80223231cb4a7af04ce9 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 11:09:56 +0200 Subject: [PATCH 03/15] feat (ui): last update by SSE --- engine/api/application/dao.go | 12 ++++++ engine/api/lastupdate.go | 52 ++++++++++++++++++++++-- engine/api/pipeline/pipeline.go | 13 ++++++ engine/api/project/dao.go | 16 ++++---- engine/api/workflow/run_workflow_test.go | 2 +- sdk/project.go | 9 ++++ 6 files changed, 91 insertions(+), 13 deletions(-) diff --git a/engine/api/application/dao.go b/engine/api/application/dao.go index c1052ce67b..b5cb5bd6c4 100644 --- a/engine/api/application/dao.go +++ b/engine/api/application/dao.go @@ -2,6 +2,7 @@ package application import ( "database/sql" + "encoding/json" "fmt" "time" @@ -212,6 +213,17 @@ func UpdateLastModified(db gorp.SqlExecutor, app *sdk.Application, u *sdk.User) }, 0) } + updates := sdk.LastModification{ + Key: app.ProjectKey, + Name: app.Name, + LastModified: lastModified.Unix(), + Username: u.Username, + Type: sdk.ApplicationLastModificationType, + } + b, errP := json.Marshal(updates) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } return sdk.WrapError(err, "application.UpdateLastModified %s(%d)", app.Name, app.ID) } diff --git a/engine/api/lastupdate.go b/engine/api/lastupdate.go index f3d8f61409..aaa75eb4c0 100644 --- a/engine/api/lastupdate.go +++ b/engine/api/lastupdate.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "net/http" "time" @@ -29,7 +30,6 @@ type LastUpdateBroker struct { } var lastUpdateBroker *LastUpdateBroker -var cacheQueueName = "lastUpdates" func Initialize(c context.Context, DBFunc func() *gorp.DbMap) { lastUpdateBroker = &LastUpdateBroker{ @@ -48,7 +48,7 @@ func Initialize(c context.Context, DBFunc func() *gorp.DbMap) { // CacheSubscribe subscribe to a channel and push received message in a channel func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { - pubSub := cache.Subscribe(cacheQueueName) + pubSub := cache.Subscribe("lastupdates") tick := time.NewTicker(250 * time.Millisecond).C for { select { @@ -74,6 +74,16 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { for { select { + case <-c.Done(): + // Close all channels + for c := range b.clients { + delete(b.clients, c) + close(c) + } + if c.Err() != nil { + log.Warning("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) + return + } case s := <-b.newClients: // Register new client b.clients[s.Queue] = s.User @@ -82,9 +92,43 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { delete(b.clients, s) close(s) case msg := <-b.messages: + var lastModif sdk.LastModification + if err := json.Unmarshal([]byte(msg), &lastModif); err != nil { + log.Warning("lastUpdate.CacheSubscribe> Cannot unmarshal message: %s", msg) + continue + } + //Receive new message - for s := range b.clients { - s <- msg + for c, u := range b.clients { + if err := loadUserPermissions(db, u); err != nil { + log.Warning("lastUpdate.CacheSubscribe> Cannot load auser permission: %s", err) + continue + } + + hasPermission := false + groups: + for _, g := range u.Groups { + hasPermission = false + switch lastModif.Type { + case sdk.ApplicationLastModificationType: + for _, ag := range g.ApplicationGroups { + if ag.Application.Name == lastModif.Name && ag.Application.ProjectKey == lastModif.Key { + hasPermission = true + break groups + } + } + case sdk.PipelineLastModificationType: + for _, pg := range g.PipelineGroups { + if pg.Pipeline.Name == lastModif.Name && pg.Pipeline.ProjectKey == lastModif.Key { + hasPermission = true + break groups + } + } + } + } + if hasPermission { + c <- msg + } } } } diff --git a/engine/api/pipeline/pipeline.go b/engine/api/pipeline/pipeline.go index 48f91ab1f0..3f9a00f05d 100644 --- a/engine/api/pipeline/pipeline.go +++ b/engine/api/pipeline/pipeline.go @@ -2,6 +2,7 @@ package pipeline import ( "database/sql" + "encoding/json" "time" "github.com/go-gorp/gorp" @@ -35,6 +36,18 @@ func UpdatePipelineLastModified(db gorp.SqlExecutor, proj *sdk.Project, p *sdk.P }, 0) } + updates := sdk.LastModification{ + Key: proj.Key, + Name: p.Name, + LastModified: lastModified.Unix(), + Username: u.Username, + Type: sdk.PipelineLastModificationType, + } + b, errP := json.Marshal(updates) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } + return err } diff --git a/engine/api/project/dao.go b/engine/api/project/dao.go index 2b403a1580..9b86bd3fd2 100644 --- a/engine/api/project/dao.go +++ b/engine/api/project/dao.go @@ -2,6 +2,7 @@ package project import ( "database/sql" + "encoding/json" "fmt" "time" @@ -12,7 +13,6 @@ import ( "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" - "encoding/json" ) // LoadAll returns all projects @@ -151,14 +151,14 @@ func UpdateLastModified(db gorp.SqlExecutor, u *sdk.User, proj *sdk.Project) err _, err := db.Exec("update project set last_modified = $2 where projectkey = $1", proj.Key, t) proj.LastModified = t - projLastUpdate := sdk.ProjectLastUpdates{ - LastModification: sdk.LastModification{ - Name: proj.Key, - LastModified: t.Unix(), - Username: u.Username, - }, + updates := sdk.LastModification{ + Key: proj.Key, + Name: proj.Name, + LastModified: t.Unix(), + Username: u.Username, + Type: sdk.ProjectLastModiciationType, } - b, errP := json.Marshal(projLastUpdate) + b, errP := json.Marshal(updates) if errP == nil { cache.Publish("lastUpdates", string(b)) } diff --git a/engine/api/workflow/run_workflow_test.go b/engine/api/workflow/run_workflow_test.go index 83147f0190..df2559854c 100644 --- a/engine/api/workflow/run_workflow_test.go +++ b/engine/api/workflow/run_workflow_test.go @@ -31,7 +31,7 @@ func TestManualRun1(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true diff --git a/sdk/project.go b/sdk/project.go index 059a3b283b..88bf96ccec 100644 --- a/sdk/project.go +++ b/sdk/project.go @@ -43,12 +43,21 @@ type Metadata map[string]string //LastModification is stored in cache and used for ProjectLastUpdates computing type LastModification struct { + Key string `json:key,omitempty` Name string `json:"name"` Username string `json:"username"` LastModified int64 `json:"last_modified"` + Type string `json:"type,omitempty"` } +const ( + ApplicationLastModificationType = "application" + PipelineLastModificationType = "pipeline" + ProjectLastModiciationType = "project" +) + //ProjectLastUpdates update times of project, application and pipelines +// Deprecated type ProjectLastUpdates struct { LastModification Applications []LastModification `json:"applications"` From 2b8698efc6bd7f4d7664444c466b94e72a7923f3 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 11:15:05 +0200 Subject: [PATCH 04/15] fix (api): go fmt --- engine/api/admin.go | 2 +- engine/api/application_group.go | 2 +- engine/api/application_pipeline_notif_test.go | 2 +- engine/api/application_variable.go | 2 +- engine/api/businesscontext/context.go | 8 ++++---- engine/api/cache/cache.go | 1 - engine/api/cache/local.go | 3 +-- engine/api/cache/redis.go | 4 ++-- engine/api/group.go | 11 ++++------- engine/api/pipeline.go | 1 - engine/api/pipeline_parameter.go | 3 +-- engine/api/router.go | 2 +- engine/api/router_util.go | 2 +- engine/api/status.go | 2 +- sdk/keychain/keychain_darwin.go | 4 +--- sdk/project.go | 4 ++-- .../docker-credential-helpers/credentials/error.go | 5 +---- 17 files changed, 23 insertions(+), 35 deletions(-) diff --git a/engine/api/admin.go b/engine/api/admin.go index 1d87fd4e63..89ddf3418a 100644 --- a/engine/api/admin.go +++ b/engine/api/admin.go @@ -5,8 +5,8 @@ import ( "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/sdk/log" ) diff --git a/engine/api/application_group.go b/engine/api/application_group.go index 546c526370..097c05c737 100644 --- a/engine/api/application_group.go +++ b/engine/api/application_group.go @@ -7,8 +7,8 @@ import ( "github.com/gorilla/mux" "github.com/ovh/cds/engine/api/application" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/engine/api/permission" "github.com/ovh/cds/engine/api/project" diff --git a/engine/api/application_pipeline_notif_test.go b/engine/api/application_pipeline_notif_test.go index b53f3a2fbd..c7534a5e52 100644 --- a/engine/api/application_pipeline_notif_test.go +++ b/engine/api/application_pipeline_notif_test.go @@ -17,8 +17,8 @@ import ( "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/auth" "github.com/ovh/cds/engine/api/bootstrap" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/environment" "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/engine/api/notification" diff --git a/engine/api/application_variable.go b/engine/api/application_variable.go index 81db875d38..33c1dda4a9 100644 --- a/engine/api/application_variable.go +++ b/engine/api/application_variable.go @@ -8,8 +8,8 @@ import ( "github.com/gorilla/mux" "github.com/ovh/cds/engine/api/application" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/engine/api/sanity" "github.com/ovh/cds/engine/api/secret" diff --git a/engine/api/businesscontext/context.go b/engine/api/businesscontext/context.go index 3d079004ec..8b26bdb77a 100644 --- a/engine/api/businesscontext/context.go +++ b/engine/api/businesscontext/context.go @@ -6,8 +6,8 @@ import ( // Ctx gather information about http call origin type Ctx struct { - Agent string - User *sdk.User - Worker *sdk.Worker - Hatchery *sdk.Hatchery + Agent string + User *sdk.User + Worker *sdk.Worker + Hatchery *sdk.Hatchery } diff --git a/engine/api/cache/cache.go b/engine/api/cache/cache.go index 949d563932..2901d747f8 100644 --- a/engine/api/cache/cache.go +++ b/engine/api/cache/cache.go @@ -160,4 +160,3 @@ func GetMessageFromSubscription(pb PubSub, c context.Context) (string, error) { } return s.GetMessageFromSubscription(c, pb) } - diff --git a/engine/api/cache/local.go b/engine/api/cache/local.go index 13b604ba0c..dad3168fa6 100644 --- a/engine/api/cache/local.go +++ b/engine/api/cache/local.go @@ -4,12 +4,12 @@ import ( "container/list" "context" "encoding/json" + "fmt" "strings" "sync" "time" "github.com/ovh/cds/sdk/log" - "fmt" ) var s Store @@ -231,4 +231,3 @@ func (s *LocalStore) GetMessageFromSubscription(c context.Context, pb PubSub) (s DequeueWithContext(c, lps.queueName, &msg) return msg, nil } - diff --git a/engine/api/cache/redis.go b/engine/api/cache/redis.go index 5616ca5ff8..80a0e82ef2 100644 --- a/engine/api/cache/redis.go +++ b/engine/api/cache/redis.go @@ -222,8 +222,8 @@ func (s *RedisStore) Publish(channel string, value interface{}) { } // Subscribe to a channel -func (s *RedisStore) Subscribe(channel string) PubSub { - return s.Client.Subscribe(channel) +func (s *RedisStore) Subscribe(channel string) PubSub { + return s.Client.Subscribe(channel) } // GetMessageFromSubscription from a redis PubSub diff --git a/engine/api/group.go b/engine/api/group.go index 252b9c2c86..ec2dbecc18 100644 --- a/engine/api/group.go +++ b/engine/api/group.go @@ -6,15 +6,15 @@ import ( "github.com/go-gorp/gorp" "github.com/gorilla/mux" + "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/environment" "github.com/ovh/cds/engine/api/group" + "github.com/ovh/cds/engine/api/pipeline" + "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/engine/api/user" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" - "github.com/ovh/cds/engine/api/project" - "github.com/ovh/cds/engine/api/application" - "github.com/ovh/cds/engine/api/pipeline" - "github.com/ovh/cds/engine/api/environment" ) func getGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { @@ -82,7 +82,6 @@ func deleteGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, } } - for _, pg := range g.PipelineGroups { p := &sdk.Project{ Key: pg.Pipeline.ProjectKey, @@ -92,14 +91,12 @@ func deleteGroupHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, } } - for _, pg := range g.EnvironmentGroups { if err := environment.UpdateLastModified(tx, c.User, &pg.Environment); err != nil { return sdk.WrapError(err, "deleteGroupHandler> Cannot update environment last modified date") } } - if err := tx.Commit(); err != nil { return sdk.WrapError(err, "deleteGroupHandler> cannot commit transaction") } diff --git a/engine/api/pipeline.go b/engine/api/pipeline.go index c156874800..12d10edd1c 100644 --- a/engine/api/pipeline.go +++ b/engine/api/pipeline.go @@ -774,7 +774,6 @@ func deletePipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *b return sdk.WrapError(errP, "deletePipeline> Cannot load project") } - p, err := pipeline.LoadPipeline(db, projectKey, pipelineName, false) if err != nil { return sdk.WrapError(err, "deletePipeline> Cannot load pipeline %s", pipelineName) diff --git a/engine/api/pipeline_parameter.go b/engine/api/pipeline_parameter.go index 0e60da5c7c..e511f4819f 100644 --- a/engine/api/pipeline_parameter.go +++ b/engine/api/pipeline_parameter.go @@ -6,12 +6,12 @@ import ( "github.com/go-gorp/gorp" "github.com/gorilla/mux" + "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/businesscontext" "github.com/ovh/cds/engine/api/pipeline" "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" - "github.com/ovh/cds/engine/api/application" ) func getParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { @@ -87,7 +87,6 @@ func updateParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, d key := vars["key"] pipelineName := vars["permPipelineKey"] - proj, errP := project.Load(db, key, c.User) if errP != nil { return sdk.WrapError(errP, "updateParametersInPipelineHandler: Cannot load project") diff --git a/engine/api/router.go b/engine/api/router.go index dc715adb03..6c34000505 100644 --- a/engine/api/router.go +++ b/engine/api/router.go @@ -2,6 +2,7 @@ package main import ( "compress/gzip" + "context" "errors" "fmt" "net/http" @@ -20,7 +21,6 @@ import ( "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" - "context" ) var ( diff --git a/engine/api/router_util.go b/engine/api/router_util.go index c08e635fef..e964a52752 100644 --- a/engine/api/router_util.go +++ b/engine/api/router_util.go @@ -10,8 +10,8 @@ import ( "github.com/gorilla/mux" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" ) diff --git a/engine/api/status.go b/engine/api/status.go index 0dc97efd6f..d5c371da62 100644 --- a/engine/api/status.go +++ b/engine/api/status.go @@ -8,8 +8,8 @@ import ( "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/database" "github.com/ovh/cds/engine/api/event" "github.com/ovh/cds/engine/api/internal" diff --git a/sdk/keychain/keychain_darwin.go b/sdk/keychain/keychain_darwin.go index ac1bf3b8fb..9b8ad93aec 100644 --- a/sdk/keychain/keychain_darwin.go +++ b/sdk/keychain/keychain_darwin.go @@ -2,11 +2,9 @@ package keychain import ( "github.com/docker/docker-credential-helpers/credentials" -"github.com/docker/docker-credential-helpers/osxkeychain" + "github.com/docker/docker-credential-helpers/osxkeychain" ) - - //StoreSecret store a credential through libsecret func StoreSecret(url, username, secret string) error { var nativeStore = osxkeychain.Osxkeychain{} diff --git a/sdk/project.go b/sdk/project.go index 88bf96ccec..2c3b9d6a8b 100644 --- a/sdk/project.go +++ b/sdk/project.go @@ -52,8 +52,8 @@ type LastModification struct { const ( ApplicationLastModificationType = "application" - PipelineLastModificationType = "pipeline" - ProjectLastModiciationType = "project" + PipelineLastModificationType = "pipeline" + ProjectLastModiciationType = "project" ) //ProjectLastUpdates update times of project, application and pipelines diff --git a/vendor/github.com/docker/docker-credential-helpers/credentials/error.go b/vendor/github.com/docker/docker-credential-helpers/credentials/error.go index 588d4a83d7..fe6a5aef45 100644 --- a/vendor/github.com/docker/docker-credential-helpers/credentials/error.go +++ b/vendor/github.com/docker/docker-credential-helpers/credentials/error.go @@ -8,7 +8,7 @@ const ( // ErrCredentialsMissingServerURL and ErrCredentialsMissingUsername standardize // invalid credentials or credentials management operations errCredentialsMissingServerURLMessage = "no credentials server URL" - errCredentialsMissingUsernameMessage = "no credentials username" + errCredentialsMissingUsernameMessage = "no credentials username" ) // errCredentialsNotFound represents an error @@ -43,7 +43,6 @@ func IsErrCredentialsNotFoundMessage(err string) bool { return err == errCredentialsNotFoundMessage } - // errCredentialsMissingServerURL represents an error raised // when the credentials object has no server URL or when no // server URL is provided to a credentials operation requiring @@ -64,7 +63,6 @@ func (errCredentialsMissingUsername) Error() string { return errCredentialsMissingUsernameMessage } - // NewErrCredentialsMissingServerURL creates a new error for // errCredentialsMissingServerURL. func NewErrCredentialsMissingServerURL() error { @@ -77,7 +75,6 @@ func NewErrCredentialsMissingUsername() error { return errCredentialsMissingUsername{} } - // IsCredentialsMissingServerURL returns true if the error // was an errCredentialsMissingServerURL. func IsCredentialsMissingServerURL(err error) bool { From 06a899c54d23b41faf860c26e66d9db72af067ec Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 11:31:54 +0200 Subject: [PATCH 05/15] fix (api): Update unit test --- engine/api/application_pipeline_notif_test.go | 4 +- engine/api/application_pipeline_test.go | 4 +- engine/api/application_test.go | 6 +-- engine/api/build_test.go | 4 +- .../pipeline/test/pipeline_importer_test.go | 14 +++--- engine/api/pipeline/test/pipeline_test.go | 4 +- engine/api/pipeline_job_test.go | 6 +-- engine/api/pipeline_stage_test.go | 12 ++--- engine/api/pipeline_test.go | 8 ++-- engine/api/polling_test.go | 10 ++-- engine/api/project_lastupdates_test.go | 10 ++-- engine/api/project_notif_test.go | 2 +- engine/api/scheduler/dao_test.go | 12 ++--- engine/api/scheduler/executer_test.go | 2 +- engine/api/scheduler_test.go | 8 ++-- engine/api/trigger_test.go | 12 ++--- engine/api/user_test.go | 2 +- engine/api/workflow/dao_test.go | 48 +++++++++---------- engine/api/workflow/run_workflow_test.go | 10 ++-- engine/api/workflow_queue_test.go | 2 +- engine/api/workflow_run_test.go | 24 +++++----- engine/api/workflow_test.go | 6 +-- 22 files changed, 105 insertions(+), 105 deletions(-) diff --git a/engine/api/application_pipeline_notif_test.go b/engine/api/application_pipeline_notif_test.go index c7534a5e52..8db2d55fc0 100644 --- a/engine/api/application_pipeline_notif_test.go +++ b/engine/api/application_pipeline_notif_test.go @@ -102,7 +102,7 @@ func testApplicationPipelineNotifBoilerPlate(t *testing.T, f func(*testing.T, *g ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - err := pipeline.InsertPipeline(db, pip, u) + err := pipeline.InsertPipeline(db, proj, pip, u) test.NoError(t, err) //Insert Application @@ -601,7 +601,7 @@ func Test_addNotificationsHandler(t *testing.T) { Type: "build", ProjectID: p.ID, } - err = pipeline.InsertPipeline(db, pip, u) + err = pipeline.InsertPipeline(db, p, pip, u) test.NoError(t, err) _, err = application.AttachPipeline(db, app.ID, pip.ID) diff --git a/engine/api/application_pipeline_test.go b/engine/api/application_pipeline_test.go index d08da343d0..8750309276 100644 --- a/engine/api/application_pipeline_test.go +++ b/engine/api/application_pipeline_test.go @@ -39,7 +39,7 @@ func Test_attachPipelinesToApplicationHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, u); err != nil { t.Fatal(err) } @@ -51,7 +51,7 @@ func Test_attachPipelinesToApplicationHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip2, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip2, u); err != nil { t.Fatal(err) } diff --git a/engine/api/application_test.go b/engine/api/application_test.go index 4dd273e6e9..b50422c680 100644 --- a/engine/api/application_test.go +++ b/engine/api/application_test.go @@ -41,7 +41,7 @@ func TestGetApplicationWithTriggersHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip1, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip1, u); err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func TestGetApplicationWithTriggersHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip2, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip2, u); err != nil { t.Fatal(err) } @@ -65,7 +65,7 @@ func TestGetApplicationWithTriggersHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip3, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip3, u); err != nil { t.Fatal(err) } // 6. Create application diff --git a/engine/api/build_test.go b/engine/api/build_test.go index 2b5f6eb188..4457ce1469 100644 --- a/engine/api/build_test.go +++ b/engine/api/build_test.go @@ -46,7 +46,7 @@ func Test_updateStepStatusHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, u); err != nil { t.Fatal(err) } @@ -142,7 +142,7 @@ func Test_addSpawnInfosPipelineBuildJobHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, u); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, u); err != nil { t.Fatal(err) } diff --git a/engine/api/pipeline/test/pipeline_importer_test.go b/engine/api/pipeline/test/pipeline_importer_test.go index 32a57752ef..9f5eda3388 100644 --- a/engine/api/pipeline/test/pipeline_importer_test.go +++ b/engine/api/pipeline/test/pipeline_importer_test.go @@ -95,7 +95,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ sdk.Stage{ BuildOrder: 1, @@ -134,7 +134,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { @@ -190,7 +190,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { @@ -234,7 +234,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { @@ -277,7 +277,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { @@ -362,7 +362,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { @@ -442,7 +442,7 @@ func TestImportUpdate(t *testing.T) { args.pip.Name = proj.Key + "_PIP" args.pip.ProjectID = proj.ID args.pip.ProjectKey = proj.Key - test.NoError(t, pipeline.InsertPipeline(db, args.pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, args.pip, nil)) args.pip.Stages = []sdk.Stage{ { diff --git a/engine/api/pipeline/test/pipeline_test.go b/engine/api/pipeline/test/pipeline_test.go index 5bda4d9a35..e418dfbf26 100644 --- a/engine/api/pipeline/test/pipeline_test.go +++ b/engine/api/pipeline/test/pipeline_test.go @@ -58,7 +58,7 @@ func TestInsertPipeline(t *testing.T) { }, } for _, tt := range tests { - if err := pipeline.InsertPipeline(db, tt.p, nil); (err != nil) != tt.wantErr { + if err := pipeline.InsertPipeline(db, &p, tt.p, nil); (err != nil) != tt.wantErr { t.Errorf("%q. InsertPipeline() error = %v, wantErr %v", tt.name, err, tt.wantErr) } } @@ -94,7 +94,7 @@ func TestInsertPipelineWithParemeters(t *testing.T) { }, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) pip1, err := pipeline.LoadPipeline(db, p.Key, "Name", true) test.NoError(t, err) diff --git a/engine/api/pipeline_job_test.go b/engine/api/pipeline_job_test.go index d807464d52..3bcf4480bd 100644 --- a/engine/api/pipeline_job_test.go +++ b/engine/api/pipeline_job_test.go @@ -40,7 +40,7 @@ func TestAddJobHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //4. Add Stage stage := &sdk.Stage{ @@ -109,7 +109,7 @@ func TestUpdateJobHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //4. Add Stage stage := &sdk.Stage{ @@ -193,7 +193,7 @@ func TestDeleteJobHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //4. Add Stage stage := &sdk.Stage{ diff --git a/engine/api/pipeline_stage_test.go b/engine/api/pipeline_stage_test.go index fc80c62e58..ae98fa06bd 100644 --- a/engine/api/pipeline_stage_test.go +++ b/engine/api/pipeline_stage_test.go @@ -28,7 +28,7 @@ func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutPrerequisite(t *testing ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage := &sdk.Stage{ @@ -82,7 +82,7 @@ func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutPrerequisite(t *testing ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage := &sdk.Stage{ @@ -153,7 +153,7 @@ func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2Actio ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage0 := &sdk.Stage{ @@ -259,7 +259,7 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage := &sdk.Stage{ @@ -395,7 +395,7 @@ func TestDeleteStageByIDShouldDeleteStagePrerequisites(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage := &sdk.Stage{ @@ -453,7 +453,7 @@ func TestUpdateSTageShouldUpdateStagePrerequisites(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //Insert Stage stage := &sdk.Stage{ diff --git a/engine/api/pipeline_test.go b/engine/api/pipeline_test.go index 897b5c83b4..45454a53b6 100644 --- a/engine/api/pipeline_test.go +++ b/engine/api/pipeline_test.go @@ -36,7 +36,7 @@ func insertTestPipeline(db *gorp.DbMap, t *testing.T, name string) (*sdk.Project } test.NoError(t, application.Insert(db, projectFoo, app, nil)) - test.NoError(t, pipeline.InsertPipeline(db, p, nil)) + test.NoError(t, pipeline.InsertPipeline(db, projectFoo, p, nil)) return projectFoo, p, app } @@ -61,7 +61,7 @@ func Test_runPipelineHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -139,7 +139,7 @@ func Test_runPipelineWithLastParentHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -205,7 +205,7 @@ func Test_runPipelineWithLastParentHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err = pipeline.InsertPipeline(db, pip2, u) + err = pipeline.InsertPipeline(db, proj, pip2, u) test.NoError(t, err) //11. Insert another Application diff --git a/engine/api/polling_test.go b/engine/api/polling_test.go index b370ae2941..1795131c0c 100644 --- a/engine/api/polling_test.go +++ b/engine/api/polling_test.go @@ -80,7 +80,7 @@ func TestAddPollerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -152,7 +152,7 @@ func TestUpdatePollerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -245,7 +245,7 @@ func TestGetApplicationPollersHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -341,7 +341,7 @@ func TestGetPollersHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) @@ -433,7 +433,7 @@ func TestDeletePollerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) //4. Insert Application appName := sdk.RandomString(10) diff --git a/engine/api/project_lastupdates_test.go b/engine/api/project_lastupdates_test.go index 1a68a60ab1..c0ee7e0f18 100644 --- a/engine/api/project_lastupdates_test.go +++ b/engine/api/project_lastupdates_test.go @@ -81,7 +81,7 @@ func Test_getUserLastUpdatesShouldReturns1Project1App1Pipeline(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //Insert Application app := &sdk.Application{ @@ -181,7 +181,7 @@ func Test_getUserLastUpdatesShouldReturns1Project2Apps1Pipeline(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //Insert Application app := &sdk.Application{ @@ -296,7 +296,7 @@ func Test_getUserLastUpdatesShouldReturns2Project2Apps1Pipeline(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - test.NoError(t, pipeline.InsertPipeline(db, pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip, u)) //Insert Application app := &sdk.Application{ @@ -422,7 +422,7 @@ func Test_getUserLastUpdatesShouldReturns1Project1Apps1PipelineWithSinceHeader(t ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - err := pipeline.InsertPipeline(db, pip, u) + err := pipeline.InsertPipeline(db, proj, pip, u) test.NoError(t, err) //Insert Application @@ -544,7 +544,7 @@ func Test_getUserLastUpdatesShouldReturnsNothingWithSinceHeader(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - err = pipeline.InsertPipeline(db, pip, u) + err = pipeline.InsertPipeline(db, proj, pip, u) test.NoError(t, err) //Insert Application diff --git a/engine/api/project_notif_test.go b/engine/api/project_notif_test.go index 32ef23fd5f..a9d9ed71ce 100644 --- a/engine/api/project_notif_test.go +++ b/engine/api/project_notif_test.go @@ -45,7 +45,7 @@ func Test_getProjectNotificationsHandler(t *testing.T) { Type: "build", ProjectID: p.ID, } - err = pipeline.InsertPipeline(db, pip, nil) + err = pipeline.InsertPipeline(db, p, pip, nil) test.NoError(t, err) test.NoError(t, group.InsertGroupInPipeline(db, pip.ID, p.ProjectGroups[0].Group.ID, 7)) diff --git a/engine/api/scheduler/dao_test.go b/engine/api/scheduler/dao_test.go index 7291e504c9..f3aa215db1 100644 --- a/engine/api/scheduler/dao_test.go +++ b/engine/api/scheduler/dao_test.go @@ -39,7 +39,7 @@ func TestInsert(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -106,7 +106,7 @@ func TestUpdate(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -190,7 +190,7 @@ func TestGetByApplication(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ func TestGetByPipeline(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -316,7 +316,7 @@ func TestGetByApplicationPipeline(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -379,7 +379,7 @@ func TestGetByApplicationPipelineEnv(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } diff --git a/engine/api/scheduler/executer_test.go b/engine/api/scheduler/executer_test.go index 6848653e2f..bd9b60d705 100644 --- a/engine/api/scheduler/executer_test.go +++ b/engine/api/scheduler/executer_test.go @@ -29,7 +29,7 @@ func TestExecuterRun(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } diff --git a/engine/api/scheduler_test.go b/engine/api/scheduler_test.go index cb6c779122..d8ef45ff5e 100644 --- a/engine/api/scheduler_test.go +++ b/engine/api/scheduler_test.go @@ -43,7 +43,7 @@ func Test_getSchedulerApplicationPipelineHandler(t *testing.T) { ProjectID: proj.ID, } t.Logf("Insert Pipeline %s for Project %s", pip.Name, proj.Name) - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func Test_addSchedulerApplicationPipelineHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -195,7 +195,7 @@ func Test_updateSchedulerApplicationPipelineHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } @@ -275,7 +275,7 @@ func Test_deleteSchedulerApplicationPipelineHandler(t *testing.T) { ProjectID: proj.ID, } - if err := pipeline.InsertPipeline(db, pip, nil); err != nil { + if err := pipeline.InsertPipeline(db, proj, pip, nil); err != nil { t.Fatal(err) } diff --git a/engine/api/trigger_test.go b/engine/api/trigger_test.go index e8e27630f6..3cbc0ec765 100644 --- a/engine/api/trigger_test.go +++ b/engine/api/trigger_test.go @@ -41,7 +41,7 @@ func TestAddTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - test.NoError(t, pipeline.InsertPipeline(db, pip1, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, pip1, u)) //4. Create Pipeline 2 pipelineKey2 := sdk.RandomString(10) @@ -51,7 +51,7 @@ func TestAddTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err := pipeline.InsertPipeline(db, pip2, u) + err := pipeline.InsertPipeline(db, proj, pip2, u) test.NoError(t, err) //5. Create Application @@ -127,7 +127,7 @@ func TestUpdateTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err := pipeline.InsertPipeline(db, pip1, u) + err := pipeline.InsertPipeline(db, proj, pip1, u) test.NoError(t, err) //4. Create Pipeline 2 @@ -138,7 +138,7 @@ func TestUpdateTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err = pipeline.InsertPipeline(db, pip2, u) + err = pipeline.InsertPipeline(db, proj, pip2, u) test.NoError(t, err) //5. Create Application @@ -221,7 +221,7 @@ func TestRemoveTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err := pipeline.InsertPipeline(db, pip1, u) + err := pipeline.InsertPipeline(db, proj, pip1, u) //4. Create Pipeline 2 pipelineKey2 := sdk.RandomString(10) @@ -231,7 +231,7 @@ func TestRemoveTriggerHandler(t *testing.T) { ProjectKey: proj.Key, ProjectID: proj.ID, } - err = pipeline.InsertPipeline(db, pip2, u) + err = pipeline.InsertPipeline(db, proj, pip2, u) //5. Create Application applicationName := sdk.RandomString(10) diff --git a/engine/api/user_test.go b/engine/api/user_test.go index 007bf97bf6..d6bafe099b 100644 --- a/engine/api/user_test.go +++ b/engine/api/user_test.go @@ -222,7 +222,7 @@ func TestLoadUserWithGroup(t *testing.T) { Type: sdk.BuildPipeline, } - err = pipeline.InsertPipeline(db, pipelinePip1, nil) + err = pipeline.InsertPipeline(db, project1, pipelinePip1, nil) if err != nil { t.Fatalf("cannot insert pipeline: %s", err) } diff --git a/engine/api/workflow/dao_test.go b/engine/api/workflow/dao_test.go index 78a3272ed1..e2b52dcae7 100644 --- a/engine/api/workflow/dao_test.go +++ b/engine/api/workflow/dao_test.go @@ -43,7 +43,7 @@ func TestInsertSimpleWorkflow(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) w := sdk.Workflow{ Name: "test_1", @@ -85,7 +85,7 @@ func TestInsertSimpleWorkflowWithApplicationAndEnv(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) app := sdk.Application{ ProjectID: proj.ID, @@ -139,7 +139,7 @@ func TestInsertComplexeWorkflow(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip1, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip1, u)) pip2 := sdk.Pipeline{ ProjectID: proj.ID, @@ -148,7 +148,7 @@ func TestInsertComplexeWorkflow(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) pip3 := sdk.Pipeline{ ProjectID: proj.ID, @@ -157,7 +157,7 @@ func TestInsertComplexeWorkflow(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip3, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip3, u)) pip4 := sdk.Pipeline{ ProjectID: proj.ID, @@ -166,7 +166,7 @@ func TestInsertComplexeWorkflow(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip4, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip4, u)) w := sdk.Workflow{ Name: "test_1", @@ -272,7 +272,7 @@ func TestUpdateSimpleWorkflowWithApplicationEnvPipelineParametersAndPayload(t *t }, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) pip2 := sdk.Pipeline{ ProjectID: proj.ID, @@ -288,7 +288,7 @@ func TestUpdateSimpleWorkflowWithApplicationEnvPipelineParametersAndPayload(t *t }, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) pip3 := sdk.Pipeline{ ProjectID: proj.ID, @@ -297,7 +297,7 @@ func TestUpdateSimpleWorkflowWithApplicationEnvPipelineParametersAndPayload(t *t Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip3, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip3, u)) app := sdk.Application{ ProjectID: proj.ID, @@ -398,7 +398,7 @@ func TestInsertComplexeWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip1, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip1, u)) pip2 := sdk.Pipeline{ ProjectID: proj.ID, @@ -407,7 +407,7 @@ func TestInsertComplexeWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) pip3 := sdk.Pipeline{ ProjectID: proj.ID, @@ -416,7 +416,7 @@ func TestInsertComplexeWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip3, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip3, u)) pip4 := sdk.Pipeline{ ProjectID: proj.ID, @@ -425,7 +425,7 @@ func TestInsertComplexeWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip4, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip4, u)) pip5 := sdk.Pipeline{ ProjectID: proj.ID, @@ -434,7 +434,7 @@ func TestInsertComplexeWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip5, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip5, u)) w := sdk.Workflow{ Name: "test_1", @@ -577,7 +577,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip1, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip1, u)) pip2 := sdk.Pipeline{ ProjectID: proj.ID, @@ -586,7 +586,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) pip3 := sdk.Pipeline{ ProjectID: proj.ID, @@ -595,7 +595,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip3, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip3, u)) pip4 := sdk.Pipeline{ ProjectID: proj.ID, @@ -604,7 +604,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip4, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip4, u)) pip5 := sdk.Pipeline{ ProjectID: proj.ID, @@ -613,7 +613,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip5, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip5, u)) pip6 := sdk.Pipeline{ ProjectID: proj.ID, @@ -622,7 +622,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip6, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip6, u)) pip7 := sdk.Pipeline{ ProjectID: proj.ID, @@ -631,7 +631,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip7, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip7, u)) w := sdk.Workflow{ Name: "test_1", @@ -786,7 +786,7 @@ func TestUpdateWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) pip2 := sdk.Pipeline{ ProjectID: proj.ID, @@ -795,7 +795,7 @@ func TestUpdateWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) pip3 := sdk.Pipeline{ ProjectID: proj.ID, @@ -804,7 +804,7 @@ func TestUpdateWorkflowWithJoins(t *testing.T) { Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip3, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip3, u)) w := sdk.Workflow{ Name: "test_1", diff --git a/engine/api/workflow/run_workflow_test.go b/engine/api/workflow/run_workflow_test.go index df2559854c..d90a96656b 100644 --- a/engine/api/workflow/run_workflow_test.go +++ b/engine/api/workflow/run_workflow_test.go @@ -55,7 +55,7 @@ func TestManualRun1(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -196,7 +196,7 @@ func TestManualRun2(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -220,7 +220,7 @@ func TestManualRun2(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -293,7 +293,7 @@ func TestManualRun3(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -317,7 +317,7 @@ func TestManualRun3(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID diff --git a/engine/api/workflow_queue_test.go b/engine/api/workflow_queue_test.go index cee18e1f0f..6a517d0a2e 100644 --- a/engine/api/workflow_queue_test.go +++ b/engine/api/workflow_queue_test.go @@ -56,7 +56,7 @@ func test_runWorkflow(t *testing.T, db *gorp.DbMap, testName string) test_runWor Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true diff --git a/engine/api/workflow_run_test.go b/engine/api/workflow_run_test.go index 98009e336a..3176591485 100644 --- a/engine/api/workflow_run_test.go +++ b/engine/api/workflow_run_test.go @@ -34,7 +34,7 @@ func Test_getWorkflowRunsHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -58,7 +58,7 @@ func Test_getWorkflowRunsHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -161,7 +161,7 @@ func Test_getLatestWorkflowRunHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -185,7 +185,7 @@ func Test_getLatestWorkflowRunHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -261,7 +261,7 @@ func Test_getWorkflowRunHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -285,7 +285,7 @@ func Test_getWorkflowRunHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -362,7 +362,7 @@ func Test_getWorkflowNodeRunHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -386,7 +386,7 @@ func Test_getWorkflowNodeRunHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -465,7 +465,7 @@ func Test_postWorkflowRunHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -489,7 +489,7 @@ func Test_postWorkflowRunHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID @@ -560,7 +560,7 @@ func Test_getWorkflowNodeRunJobStepHandler(t *testing.T) { Name: "pip1", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, u)) s := sdk.NewStage("stage 1") s.Enabled = true @@ -584,7 +584,7 @@ func Test_getWorkflowNodeRunJobStepHandler(t *testing.T) { Name: "pip2", Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip2, u)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip2, u)) s = sdk.NewStage("stage 1") s.Enabled = true s.PipelineID = pip2.ID diff --git a/engine/api/workflow_test.go b/engine/api/workflow_test.go index 7242d033c1..6f050259b8 100644 --- a/engine/api/workflow_test.go +++ b/engine/api/workflow_test.go @@ -111,7 +111,7 @@ func Test_postWorkflowHandlerWithRootShouldSuccess(t *testing.T) { ProjectID: proj.ID, Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, nil)) //Prepare request vars := map[string]string{ @@ -155,7 +155,7 @@ func Test_putWorkflowHandler(t *testing.T) { ProjectID: proj.ID, Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, nil)) //Prepare request vars := map[string]string{ @@ -226,7 +226,7 @@ func Test_deleteWorkflowHandler(t *testing.T) { ProjectID: proj.ID, Type: sdk.BuildPipeline, } - test.NoError(t, pipeline.InsertPipeline(db, &pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, proj, &pip, nil)) //Prepare request vars := map[string]string{ From 0c94352f11dafd66ca23233a1d687f28129d7dfc Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 11:47:29 +0200 Subject: [PATCH 06/15] fix (api): godoc --- engine/api/cache/cache.go | 1 + engine/api/cache/local.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/engine/api/cache/cache.go b/engine/api/cache/cache.go index 2901d747f8..e6d8fb5aa3 100644 --- a/engine/api/cache/cache.go +++ b/engine/api/cache/cache.go @@ -13,6 +13,7 @@ import ( //Status : local ok redis var Status string +// PubSub represents a subscriber type PubSub interface { Unsubscribe(channels ...string) error } diff --git a/engine/api/cache/local.go b/engine/api/cache/local.go index dad3168fa6..9540e20149 100644 --- a/engine/api/cache/local.go +++ b/engine/api/cache/local.go @@ -188,10 +188,12 @@ func (s *LocalStore) DequeueWithContext(c context.Context, queueName string, val return } +// LocalPubSub local subscriber type LocalPubSub struct { queueName string } +// Unsubscribe a subscriber func (s *LocalPubSub) Unsubscribe(channels ...string) error { return nil } From 10c617c99b3fb3e95fe7c9812f5e237232e58273 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 19 Jul 2017 14:10:03 +0200 Subject: [PATCH 07/15] fix (api): Unit test fix --- engine/api/application/dao.go | 23 +++++++++++---------- engine/api/pipeline/pipeline.go | 22 ++++++++++---------- engine/api/pipeline/test/pipeline_test.go | 2 +- engine/api/project/dao.go | 25 +++++++++++++---------- engine/api/router.go | 2 -- 5 files changed, 38 insertions(+), 36 deletions(-) diff --git a/engine/api/application/dao.go b/engine/api/application/dao.go index b5cb5bd6c4..d19f5b3e4f 100644 --- a/engine/api/application/dao.go +++ b/engine/api/application/dao.go @@ -211,19 +211,20 @@ func UpdateLastModified(db gorp.SqlExecutor, app *sdk.Application, u *sdk.User) Username: u.Username, LastModified: lastModified.Unix(), }, 0) - } - updates := sdk.LastModification{ - Key: app.ProjectKey, - Name: app.Name, - LastModified: lastModified.Unix(), - Username: u.Username, - Type: sdk.ApplicationLastModificationType, - } - b, errP := json.Marshal(updates) - if errP == nil { - cache.Publish("lastUpdates", string(b)) + updates := sdk.LastModification{ + Key: app.ProjectKey, + Name: app.Name, + LastModified: lastModified.Unix(), + Username: u.Username, + Type: sdk.ApplicationLastModificationType, + } + b, errP := json.Marshal(updates) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } } + return sdk.WrapError(err, "application.UpdateLastModified %s(%d)", app.Name, app.ID) } diff --git a/engine/api/pipeline/pipeline.go b/engine/api/pipeline/pipeline.go index 3f9a00f05d..6fa410731b 100644 --- a/engine/api/pipeline/pipeline.go +++ b/engine/api/pipeline/pipeline.go @@ -34,18 +34,18 @@ func UpdatePipelineLastModified(db gorp.SqlExecutor, proj *sdk.Project, p *sdk.P Username: u.Username, LastModified: t.Unix(), }, 0) - } - updates := sdk.LastModification{ - Key: proj.Key, - Name: p.Name, - LastModified: lastModified.Unix(), - Username: u.Username, - Type: sdk.PipelineLastModificationType, - } - b, errP := json.Marshal(updates) - if errP == nil { - cache.Publish("lastUpdates", string(b)) + updates := sdk.LastModification{ + Key: proj.Key, + Name: p.Name, + LastModified: lastModified.Unix(), + Username: u.Username, + Type: sdk.PipelineLastModificationType, + } + b, errP := json.Marshal(updates) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } } return err diff --git a/engine/api/pipeline/test/pipeline_test.go b/engine/api/pipeline/test/pipeline_test.go index e418dfbf26..e15da9037e 100644 --- a/engine/api/pipeline/test/pipeline_test.go +++ b/engine/api/pipeline/test/pipeline_test.go @@ -94,7 +94,7 @@ func TestInsertPipelineWithParemeters(t *testing.T) { }, } - test.NoError(t, pipeline.InsertPipeline(db, proj, pip, nil)) + test.NoError(t, pipeline.InsertPipeline(db, &p, pip, nil)) pip1, err := pipeline.LoadPipeline(db, p.Key, "Name", true) test.NoError(t, err) diff --git a/engine/api/project/dao.go b/engine/api/project/dao.go index 9b86bd3fd2..b413d3384f 100644 --- a/engine/api/project/dao.go +++ b/engine/api/project/dao.go @@ -151,18 +151,21 @@ func UpdateLastModified(db gorp.SqlExecutor, u *sdk.User, proj *sdk.Project) err _, err := db.Exec("update project set last_modified = $2 where projectkey = $1", proj.Key, t) proj.LastModified = t - updates := sdk.LastModification{ - Key: proj.Key, - Name: proj.Name, - LastModified: t.Unix(), - Username: u.Username, - Type: sdk.ProjectLastModiciationType, - } - b, errP := json.Marshal(updates) - if errP == nil { - cache.Publish("lastUpdates", string(b)) + if u != nil { + updates := sdk.LastModification{ + Key: proj.Key, + Name: proj.Name, + LastModified: t.Unix(), + Username: u.Username, + Type: sdk.ProjectLastModiciationType, + } + b, errP := json.Marshal(updates) + if errP == nil { + cache.Publish("lastUpdates", string(b)) + } + return err } - return err + return nil } // DeleteByID removes given project from database (project and project_group table) diff --git a/engine/api/router.go b/engine/api/router.go index 1bf18c888a..18a226bd27 100644 --- a/engine/api/router.go +++ b/engine/api/router.go @@ -2,7 +2,6 @@ package main import ( "compress/gzip" - "context" "errors" "fmt" "net/http" @@ -157,7 +156,6 @@ func (r *Router) Handle(uri string, handlers ...RouterConfigParam) { } f := func(w http.ResponseWriter, req *http.Request) { - req = req.WithContext(context.Background()) // Close indicates to close the connection after replying to this request req.Close = true From 2033efb26bf202ff50738d597c70c448cb115f70 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Thu, 20 Jul 2017 18:01:05 +0200 Subject: [PATCH 08/15] feat (api): last update sse handler --- engine/api/lastupdate.go | 96 +++++++++++++++++++++------------------ engine/api/main.go | 5 +- engine/api/main_routes.go | 1 + sdk/project.go | 2 +- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/engine/api/lastupdate.go b/engine/api/lastupdate.go index aaa75eb4c0..e0e58cdfea 100644 --- a/engine/api/lastupdate.go +++ b/engine/api/lastupdate.go @@ -11,31 +11,31 @@ import ( "github.com/ovh/cds/engine/api/businesscontext" "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/engine/api/sessionstore" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" ) // LastUpdateBrokerSubscribe is the information need to to subscribe type LastUpdateBrokerSubscribe struct { + UIID string User *sdk.User Queue chan string } // LastUpdateBroker keeps connected client of the current route, type LastUpdateBroker struct { - clients map[chan string]*sdk.User - newClients chan LastUpdateBrokerSubscribe - exitClients chan chan string - messages chan string + clients map[string]*LastUpdateBrokerSubscribe + newClients chan *LastUpdateBrokerSubscribe + messages chan string } var lastUpdateBroker *LastUpdateBroker -func Initialize(c context.Context, DBFunc func() *gorp.DbMap) { +func InitLastUpdateBroker(c context.Context, DBFunc func() *gorp.DbMap) { lastUpdateBroker = &LastUpdateBroker{ - make(map[chan string]*sdk.User), - make(chan LastUpdateBrokerSubscribe), - make(chan (chan string)), + make(map[string]*LastUpdateBrokerSubscribe), + make(chan *LastUpdateBrokerSubscribe), make(chan string), } @@ -48,7 +48,7 @@ func Initialize(c context.Context, DBFunc func() *gorp.DbMap) { // CacheSubscribe subscribe to a channel and push received message in a channel func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { - pubSub := cache.Subscribe("lastupdates") + pubSub := cache.Subscribe("lastUpdates") tick := time.NewTicker(250 * time.Millisecond).C for { select { @@ -60,7 +60,7 @@ func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { case <-tick: msg, err := cache.GetMessageFromSubscription(pubSub, c) if err != nil { - log.Warning("lastUpdate.CacheSubscribe> Cannot get message") + log.Warning("lastUpdate.CacheSubscribe> Cannot get message %s: %s", msg, err) time.Sleep(5 * time.Second) continue } @@ -76,9 +76,9 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { select { case <-c.Done(): // Close all channels - for c := range b.clients { + for c, v := range b.clients { delete(b.clients, c) - close(c) + close(v.Queue) } if c.Err() != nil { log.Warning("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) @@ -86,11 +86,7 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { } case s := <-b.newClients: // Register new client - b.clients[s.Queue] = s.User - case s := <-b.exitClients: - // Deleting client - delete(b.clients, s) - close(s) + b.clients[s.UIID] = s case msg := <-b.messages: var lastModif sdk.LastModification if err := json.Unmarshal([]byte(msg), &lastModif); err != nil { @@ -99,35 +95,40 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { } //Receive new message - for c, u := range b.clients { - if err := loadUserPermissions(db, u); err != nil { + for _, i := range b.clients { + if err := loadUserPermissions(db, i.User); err != nil { log.Warning("lastUpdate.CacheSubscribe> Cannot load auser permission: %s", err) continue } hasPermission := false - groups: - for _, g := range u.Groups { - hasPermission = false - switch lastModif.Type { - case sdk.ApplicationLastModificationType: - for _, ag := range g.ApplicationGroups { - if ag.Application.Name == lastModif.Name && ag.Application.ProjectKey == lastModif.Key { - hasPermission = true - break groups + if i.User.Admin { + hasPermission = true + } else { + groups: + for _, g := range i.User.Groups { + hasPermission = false + switch lastModif.Type { + case sdk.ApplicationLastModificationType: + for _, ag := range g.ApplicationGroups { + if ag.Application.Name == lastModif.Name && ag.Application.ProjectKey == lastModif.Key { + hasPermission = true + break groups + } } - } - case sdk.PipelineLastModificationType: - for _, pg := range g.PipelineGroups { - if pg.Pipeline.Name == lastModif.Name && pg.Pipeline.ProjectKey == lastModif.Key { - hasPermission = true - break groups + case sdk.PipelineLastModificationType: + for _, pg := range g.PipelineGroups { + if pg.Pipeline.Name == lastModif.Name && pg.Pipeline.ProjectKey == lastModif.Key { + hasPermission = true + break groups + } } } } } + if hasPermission { - c <- msg + i.Queue <- msg } } } @@ -136,14 +137,19 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { } func (b *LastUpdateBroker) ServeHTTP(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { - // Make sure that the writer supports flushing. f, ok := w.(http.Flusher) if !ok { http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) return nil } - messageChan := LastUpdateBrokerSubscribe{ + + uuid, errS := sessionstore.NewSessionKey() + if errS != nil { + return sdk.WrapError(errS, "LastUpdateBroker.Serve> Cannot generate UUID") + } + messageChan := &LastUpdateBrokerSubscribe{ + UIID: string(uuid), User: c.User, Queue: make(chan string), } @@ -156,19 +162,19 @@ func (b *LastUpdateBroker) ServeHTTP(w http.ResponseWriter, r *http.Request, db w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") +leave: for { - select { - case <-r.Context().Done(): - // Close - b.exitClients <- messageChan.Queue + case <-w.(http.CloseNotifier).CloseNotify(): + delete(b.clients, messageChan.UIID) + close(messageChan.Queue) + break leave case msg, open := <-messageChan.Queue: if !open { - break + delete(b.clients, messageChan.UIID) + break leave } - - // Message must start with data: https://developer.mozilla.org/fr/docs/Server-sent_events/Using_server-sent_events - fmt.Fprintf(w, "data: %s", msg) + fmt.Fprintf(w, "data: %s\n\n", msg) f.Flush() } } diff --git a/engine/api/main.go b/engine/api/main.go index 49c862170c..f1ae07b99f 100644 --- a/engine/api/main.go +++ b/engine/api/main.go @@ -175,6 +175,9 @@ var mainCmd = &cobra.Command{ log.Error("Cannot setup databases: %s", err) } + cache.Initialize(viper.GetString(viperCacheMode), viper.GetString(viperCacheRedisHost), viper.GetString(viperCacheRedisPassword), viper.GetInt(viperCacheTTL)) + InitLastUpdateBroker(ctx, database.GetDBMap) + router = &Router{ mux: mux.NewRouter(), } @@ -236,8 +239,6 @@ var mainCmd = &cobra.Command{ log.Fatalf("Error: %v", errdriver) } - cache.Initialize(viper.GetString(viperCacheMode), viper.GetString(viperCacheRedisHost), viper.GetString(viperCacheRedisPassword), viper.GetInt(viperCacheTTL)) - kafkaOptions := event.KafkaConfig{ Enabled: viper.GetBool(viperEventsKafkaEnabled), BrokerAddresses: viper.GetString(viperEventsKafkaBroker), diff --git a/engine/api/main_routes.go b/engine/api/main_routes.go index cbb754e8de..2fef25eb9f 100644 --- a/engine/api/main_routes.go +++ b/engine/api/main_routes.go @@ -282,6 +282,7 @@ func (router *Router) init() { router.Handle("/worker/model/capability/type", GET(getWorkerModelCapaTypes)) router.Handle("/worker/model/{permModelID}/capability/{capa}", PUT(updateWorkerModelCapa), DELETE(deleteWorkerModelCapa)) + // SSE router.Handle("/mon/lastupdates/events", GET(lastUpdateBroker.ServeHTTP)) //Not Found handler diff --git a/sdk/project.go b/sdk/project.go index 2c3b9d6a8b..5906ea02bd 100644 --- a/sdk/project.go +++ b/sdk/project.go @@ -43,7 +43,7 @@ type Metadata map[string]string //LastModification is stored in cache and used for ProjectLastUpdates computing type LastModification struct { - Key string `json:key,omitempty` + Key string `json:"key,omitempty""` Name string `json:"name"` Username string `json:"username"` LastModified int64 `json:"last_modified"` From 8695bd1f121b01a180719f42f2c4753a4d5911d8 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 10:15:37 +0200 Subject: [PATCH 09/15] feat (ui): add lastUpdate on SSE handler --- ui/karma.conf.js | 2 +- ui/package.json | 1 + ui/src/app/app.component.spec.ts | 127 +++++++------- ui/src/app/app.component.ts | 28 +--- ui/src/app/app.service.ts | 157 +++++++++--------- ui/src/app/model/lastupdate.model.ts | 2 + ui/src/app/service/services.module.ts | 3 + ui/src/app/service/sse/lastupdate.sservice.ts | 36 ++++ ui/src/assets/worker/web/lastupdate.js | 25 --- ui/src/polyfills.ts | 1 - ui/src/typings.d.ts | 1 - 11 files changed, 195 insertions(+), 188 deletions(-) create mode 100644 ui/src/app/service/sse/lastupdate.sservice.ts delete mode 100644 ui/src/assets/worker/web/lastupdate.js diff --git a/ui/karma.conf.js b/ui/karma.conf.js index 45bb1a93a5..98a994ca4e 100644 --- a/ui/karma.conf.js +++ b/ui/karma.conf.js @@ -39,7 +39,7 @@ module.exports = function (config) { }, reporters: config.angularCli && config.angularCli.codeCoverage ? ['progress', 'coverage-istanbul', 'junit'] - : ['progress', 'kjhtml'], + : ['progress'], junitReporter: { outputDir: 'tests', // results will be saved as $outputDir/$browserName.xml outputFile: 'results.xml', // if included, results will be saved as $outputDir/$browserName/$outputFile diff --git a/ui/package.json b/ui/package.json index 8e9d8119c3..162fece2ed 100644 --- a/ui/package.json +++ b/ui/package.json @@ -37,6 +37,7 @@ "immutable": "^3.8.1", "lodash": "~4.17.2", "moment": "^2.17.1", + "ng-event-source": "^1.0.1", "ng-semantic": "https://github.com/sguiheux/ngSemantic/tarball/sgu-master", "ng2-codemirror-typescript": "^1.1.1", "ng2-completer": "^1.4.0", diff --git a/ui/src/app/app.component.spec.ts b/ui/src/app/app.component.spec.ts index e1f92c60d8..652d337e87 100644 --- a/ui/src/app/app.component.spec.ts +++ b/ui/src/app/app.component.spec.ts @@ -98,6 +98,10 @@ describe('App: CDS', () => { expect(fixture.componentInstance.isConnected).toBeFalsy('IsConnected flag must be false'); expect(compiled.querySelector('#navbar.connected')).toBeFalsy('Nav bar must not have the css class "connected"'); + fixture.componentInstance.startLastUpdateSSE = () => { + + }; + fixture.componentInstance.ngOnInit(); authStore.addUser(new User(), false); @@ -120,66 +124,57 @@ describe('App: CDS', () => { pipelineStore.getPipelines('key2', 'pip3').first().subscribe(() => {}).unsubscribe(); - let lastUpdateData = new Array(); - - // Proj to keep - let prj1 = new ProjectLastUpdates(); - prj1.name = 'key1'; - prj1.last_modified = 1497169222; - prj1.username = 'fooApp'; - prj1.applications = new Array(); - prj1.pipelines = new Array(); - - // App not modified - let app1 = new LastModification(); - app1.last_modified = 1494490822; - app1.name = 'app1'; - - // Modified to keep - let app2 = new LastModification(); - app2.last_modified = 1497169222; - app2.username = 'fooApp'; - app2.name = 'app2'; - - prj1.applications.push(app1); - prj1.applications.push(app2); - - // Pip not updated - let pip1 = new LastModification(); - pip1.last_modified = 1494490822; - pip1.name = 'pip1'; - - // Pip to keep - let pip2 = new LastModification(); - pip2.last_modified = 1497169222; - pip2.name = 'pip2'; - pip2.username = 'fooApp'; - - prj1.pipelines.push(pip1, pip2); - - // Proj to delete - let prj2 = new ProjectLastUpdates(); - prj2.name = 'key2'; - prj2.last_modified = 1497169222; - prj2.applications = new Array(); - prj2.pipelines = new Array(); - - // App to delete - let app3 = new LastModification(); - app3.name = 'app3'; - app3.last_modified = 1497169222; - app3.username = 'bar'; - - prj2.applications.push(app3); - - let pip3 = new LastModification(); - pip3.name = 'pip3'; - pip3.last_modified = 1497169222; - pip3.username = 'bar'; - - prj2.pipelines.push(pip3); - - lastUpdateData.push(prj1, prj2); + let prj1Update = new LastModification(); + prj1Update.key = 'key1'; + prj1Update.last_modified = 1497169222; + prj1Update.username = 'fooApp'; + prj1Update.type = 'project'; + + let app1Update = new LastModification(); + app1Update.key = 'key1'; + app1Update.last_modified = 1494490822; + app1Update.name = 'app1'; + app1Update.type = 'application'; + + + let app2Update = new LastModification(); + app2Update.key = 'key1'; + app2Update.last_modified = 1497169222; + app2Update.name = 'app2'; + app2Update.username = 'fooApp'; + app2Update.type = 'application'; + + let pip1Update = new LastModification(); + pip1Update.key = 'key1'; + pip1Update.last_modified = 1494490822; + pip1Update.name = 'pip1'; + pip1Update.type = 'pipeline'; + + let pip2Update = new LastModification(); + pip2Update.key = 'key1'; + pip2Update.last_modified = 1497169222; + pip2Update.name = 'pip2'; + pip2Update.username = 'fooApp'; + pip2Update.type = 'pipeline'; + + let prj2Update = new LastModification(); + prj2Update.key = 'key2'; + prj2Update.last_modified = 1497169222; + prj2Update.type = 'project'; + + let app3Update = new LastModification(); + app3Update.key = 'key2'; + app3Update.last_modified = 1497169222; + app3Update.name = 'app3'; + app3Update.username = 'bar'; + app3Update.type = 'application'; + + let pip3Update = new LastModification(); + pip3Update.key = 'key2'; + pip3Update.last_modified = 1497169222; + pip3Update.name = 'pip3'; + pip3Update.username = 'bar'; + pip3Update.type = 'pipeline'; let AuthStore = injector.get(AuthentificationStore); let user = new User(); @@ -188,10 +183,10 @@ describe('App: CDS', () => { let appService = injector.get(AppService); - appService.updateCache(lastUpdateData); + appService.updateCache(prj1Update); + appService.updateCache(prj2Update); // Check project result - let check = false; projectStore.getProjects().subscribe(projs => { check = true; @@ -201,6 +196,10 @@ describe('App: CDS', () => { }).unsubscribe(); expect(check).toBe(true); + appService.updateCache(app1Update); + appService.updateCache(app2Update); + appService.updateCache(app3Update); + // Check application result let checkApp = false; applicationStore.getApplications('key').subscribe(apps => { @@ -213,6 +212,10 @@ describe('App: CDS', () => { }).unsubscribe(); expect(checkApp).toBe(true); + appService.updateCache(pip1Update); + appService.updateCache(pip2Update); + appService.updateCache(pip3Update); + // Check pipeline result let checkPip = false; pipelineStore.getPipelines('key').subscribe(pips => { diff --git a/ui/src/app/app.component.ts b/ui/src/app/app.component.ts index 1f0d04df67..9af5fcf4e3 100644 --- a/ui/src/app/app.component.ts +++ b/ui/src/app/app.component.ts @@ -9,6 +9,8 @@ import {LanguageStore} from './service/language/language.store'; import {NotificationService} from './service/notification/notification.service'; import {AutoUnsubscribe} from './shared/decorator/autoUnsubscribe'; import {AppService} from './app.service'; +import {LastUpdateService} from './service/sse/lastupdate.sservice'; +import {LastModification} from './model/lastupdate.model'; @Component({ selector: 'app-root', @@ -22,7 +24,6 @@ export class AppComponent implements OnInit { isConnected = false; warningWorker: CDSWorker; versionWorker: CDSWorker; - lastUpdateWorker: CDSWorker; zone: NgZone; currentVersion = 0; @@ -31,11 +32,10 @@ export class AppComponent implements OnInit { warningWorkerSubscription: Subscription; languageSubscriber: Subscription; versionWorkerSubscription: Subscription; - lastUpdateWorkerSubscription: Subscription; constructor(private _translate: TranslateService, private _language: LanguageStore, private _authStore: AuthentificationStore, private _warnStore: WarningStore, - private _notification: NotificationService, private _appService: AppService) { + private _notification: NotificationService, private _appService: AppService, private _last: LastUpdateService) { this.zone = new NgZone({enableLongStackTrace: false}); _translate.addLangs(['en', 'fr']); _translate.setDefaultLang('en'); @@ -60,7 +60,7 @@ export class AppComponent implements OnInit { this.stopWorker(this.warningWorker, this.warningWorkerSubscription); } else { this.isConnected = true; - this.startLastUpdateWorker(); + this.startLastUpdateSSE(); this.startWarningWorker(); } this.startVersionWorker(); @@ -76,21 +76,11 @@ export class AppComponent implements OnInit { } } - startLastUpdateWorker(): void { - this.stopWorker(this.lastUpdateWorker, this.lastUpdateWorkerSubscription); - this.lastUpdateWorker = new CDSWorker('./assets/worker/web/lastupdate.js'); - this.lastUpdateWorker.start({ - 'user': this._authStore.getUser(), - 'session': this._authStore.getSessionToken(), - 'api': environment.apiURL - }); - this.lastUpdateWorker.response().subscribe(msg => { - if (msg) { - this.zone.run(() => { - let lastUpdates = JSON.parse(msg); - this._appService.updateCache(lastUpdates); - }); - } + startLastUpdateSSE(): void { + this._last.getLastUpdate().subscribe(msg => { + console.log(msg); + let lastUpdateEvent: LastModification = JSON.parse(msg); + this._appService.updateCache(lastUpdateEvent); }); } diff --git a/ui/src/app/app.service.ts b/ui/src/app/app.service.ts index b8faedf3d5..c924e96586 100644 --- a/ui/src/app/app.service.ts +++ b/ui/src/app/app.service.ts @@ -1,7 +1,7 @@ import {Injectable} from '@angular/core'; -import {ProjectLastUpdates} from './model/lastupdate.model'; +import {LastModification} from './model/lastupdate.model'; import {ProjectStore} from './service/project/project.store'; -import {ActivatedRoute, Router} from '@angular/router'; +import {ActivatedRoute} from '@angular/router'; import {ApplicationStore} from './service/application/application.store'; import {NotificationService} from './service/notification/notification.service'; import {AuthentificationStore} from './service/auth/authentification.store'; @@ -13,120 +13,119 @@ import {RouterService} from './service/router/router.service'; export class AppService { constructor(private _projStore: ProjectStore, private _routeActivated: ActivatedRoute, - private _appStore: ApplicationStore, private _notif: NotificationService, private _authStore: AuthentificationStore, - private _translate: TranslateService, private _pipStore: PipelineStore, private _routerService: RouterService) { + private _appStore: ApplicationStore, private _notif: NotificationService, private _authStore: AuthentificationStore, + private _translate: TranslateService, private _pipStore: PipelineStore, private _routerService: RouterService) { } - updateCache(lastUpdates: Array) { - if (!lastUpdates) { + updateCache(lastUpdate: LastModification) { + if (!lastUpdate) { return; } - // Get current route params - let params = this._routerService.getRouteParams({}, this._routeActivated); + if (lastUpdate.type === 'project') { + this.updateProjectCache(lastUpdate); + } else if (lastUpdate.type === 'application') { + this.updateApplicationCache(lastUpdate); + } else if (lastUpdate.type === 'pipeline') { + this.updatePipelineCache(lastUpdate); + } + } + + updateProjectCache(lastUpdate: LastModification): void { // Get all projects this._projStore.getProjects().first().subscribe(projects => { - // browse last updates - lastUpdates.forEach(plu => { - // Project not in cache - if (!projects.get(plu.name)) { - return; - } + // Project not in cache + if (!projects.get(lastUpdate.key)) { + return; + } - // Project - if ((new Date(projects.get(plu.name).last_modified)).getTime() < plu.last_modified * 1000) { - // If working on project on sub resources - if (params['key'] && params['key'] === plu.name) { - if (plu.username !== this._authStore.getUser().username) { - this._projStore.externalModification(plu.name); - this._notif.create(this._translate.instant('project_modification', { username: plu.username})); - } - - // If working on sub resources - resync project - if (params['pipName'] || params['appName'] || plu.username === this._authStore.getUser().username) { - this._projStore.resync(plu.name).first().subscribe(() => {}); - } - } else { - // remove from cache - this._projStore.removeFromStore(plu.name); - } - } + // Project + if ((new Date(projects.get(lastUpdate.key).last_modified)).getTime() < lastUpdate.last_modified * 1000) { + // Get current route params + let params = this._routerService.getRouteParams({}, this._routeActivated); - if (plu.applications && plu.applications.length > 0) { - // update application cache - this.updateApplicationCache(plu, params); - } + // If working on project on sub resources + if (params['key'] && params['key'] === lastUpdate.key) { + if (lastUpdate.username !== this._authStore.getUser().username) { + this._projStore.externalModification(lastUpdate.key); + this._notif.create(this._translate.instant('project_modification', {username: lastUpdate.username})); + } - if (plu.pipelines && plu.pipelines.length > 0) { - this.updatePipelineCache(plu, params); + // If working on sub resources - resync project + if (params['pipName'] || params['appName'] || lastUpdate.username === this._authStore.getUser().username) { + this._projStore.resync(lastUpdate.key).first().subscribe(() => {}); + } + } else { + // remove from cache + this._projStore.removeFromStore(lastUpdate.key); } - }); + } }); - } - updateApplicationCache(plu: ProjectLastUpdates, params: {}): void { - this._appStore.getApplications(plu.name).first().subscribe(apps => { + updateApplicationCache(lastUpdate: LastModification): void { + this._appStore.getApplications(lastUpdate.key).first().subscribe(apps => { if (!apps) { return; } - plu.applications.forEach( a => { - let appKey = plu.name + '-' + a.name; - if (!apps.get(appKey)) { - return; - } + let appKey = lastUpdate.key + '-' + lastUpdate.name; + if (!apps.get(appKey)) { + return; + } - if ((new Date(apps.get(appKey).last_modified)).getTime() < a.last_modified * 1000) { - if (params['key'] && params['key'] === plu.name && params['appName'] === a.name ) { + if ((new Date(apps.get(appKey).last_modified)).getTime() < lastUpdate.last_modified * 1000) { - if (a.username !== this._authStore.getUser().username) { - this._appStore.externalModification(appKey); - this._notif.create(this._translate.instant('application_modification', { username: plu.username})); - } + let params = this._routerService.getRouteParams({}, this._routeActivated); - if (params['pipName'] || a.username === this._authStore.getUser().username) { - this._appStore.resync(plu.name, a.name); - } - } else { - this._appStore.removeFromStore(appKey); + if (params['key'] && params['key'] === lastUpdate.key && params['appName'] === lastUpdate.name) { + + if (lastUpdate.username !== this._authStore.getUser().username) { + this._appStore.externalModification(appKey); + this._notif.create(this._translate.instant('application_modification', {username: lastUpdate.username})); } + + if (params['pipName'] || lastUpdate.username === this._authStore.getUser().username) { + this._appStore.resync(lastUpdate.key, lastUpdate.name); + } + } else { + this._appStore.removeFromStore(appKey); } - }); + } }); } - updatePipelineCache(plu: ProjectLastUpdates, params: {}): void { - this._pipStore.getPipelines(plu.name).first().subscribe(pips => { + updatePipelineCache(lastUpdate: LastModification): void { + this._pipStore.getPipelines(lastUpdate.name).first().subscribe(pips => { if (!pips) { return; } - plu.pipelines.forEach(p => { - let pipKey = plu.name + '-' + p.name; - if (!pips.get(pipKey)) { - return; - } - if (pips.get(pipKey).last_modified < p.last_modified) { - if (params['key'] && params['key'] === plu.name && params['pipName'] === p.name) { - this._pipStore.externalModification(pipKey); + let pipKey = lastUpdate.key + '-' + lastUpdate.name; + if (!pips.get(pipKey)) { + return; + } - if (p.username !== this._authStore.getUser().username) { - this._notif.create(this._translate.instant('pipeline_modification', {username: plu.username})); - } + if (pips.get(pipKey).last_modified < lastUpdate.last_modified) { + let params = this._routerService.getRouteParams({}, this._routeActivated); - if (params['buildNumber'] || p.username === this._authStore.getUser().username) { - this._pipStore.resync(plu.name, p.name); - } - } else { - this._pipStore.removeFromStore(pipKey); + if (params['key'] && params['key'] === lastUpdate.key && params['pipName'] === lastUpdate.name) { + this._pipStore.externalModification(pipKey); + + if (lastUpdate.username !== this._authStore.getUser().username) { + this._notif.create(this._translate.instant('pipeline_modification', {username: lastUpdate.username})); } + + if (params['buildNumber'] || lastUpdate.username === this._authStore.getUser().username) { + this._pipStore.resync(lastUpdate.key, lastUpdate.name); + } + } else { + this._pipStore.removeFromStore(pipKey); } - }); + } }); - } } diff --git a/ui/src/app/model/lastupdate.model.ts b/ui/src/app/model/lastupdate.model.ts index 3d35509e14..7d63459d9a 100644 --- a/ui/src/app/model/lastupdate.model.ts +++ b/ui/src/app/model/lastupdate.model.ts @@ -8,7 +8,9 @@ export class ProjectLastUpdates { } export class LastModification { + key: string; name: string; username: string; last_modified: number; + type: string; } diff --git a/ui/src/app/service/services.module.ts b/ui/src/app/service/services.module.ts index 658a151aff..5f99e4d97b 100644 --- a/ui/src/app/service/services.module.ts +++ b/ui/src/app/service/services.module.ts @@ -39,6 +39,7 @@ import {WorkflowStore} from './workflow/workflow.store'; import {WorkflowRunService} from './workflow/run/workflow.run.service'; import {RouterService} from './router/router.service'; import {WarningService} from './warning/warning.service'; +import {LastUpdateService} from './sse/lastupdate.sservice'; @NgModule({}) export class ServicesModule { @@ -63,6 +64,7 @@ export class ServicesModule { EnvironmentAuditService, GroupService, LanguageStore, + LastUpdateService, NotificationService, ParameterService, PipelineResolver, @@ -119,6 +121,7 @@ export { EnvironmentAuditService, GroupService, LanguageStore, + LastUpdateService, ParameterService, PipelineResolver, PipelineStore, diff --git a/ui/src/app/service/sse/lastupdate.sservice.ts b/ui/src/app/service/sse/lastupdate.sservice.ts new file mode 100644 index 0000000000..42ec8e968a --- /dev/null +++ b/ui/src/app/service/sse/lastupdate.sservice.ts @@ -0,0 +1,36 @@ +import {Injectable, NgZone} from '@angular/core'; +import {Observable} from 'rxjs/Observable'; +import {environment} from '../../../environments/environment'; +import {EventSourcePolyfill} from 'ng-event-source'; +import {AuthentificationStore} from '../auth/authentification.store'; + +@Injectable() +export class LastUpdateService { + + zone: NgZone; + + constructor(private _authStore: AuthentificationStore) { + this.zone = new NgZone({enableLongStackTrace: false}); + } + + getLastUpdate(): Observable { + let authHeader = {}; + // ADD user AUTH + let sessionToken = this._authStore.getSessionToken(); + if (sessionToken) { + authHeader[this._authStore.localStorageSessionKey] = sessionToken; + } else { + authHeader['Authorization'] = 'Basic ' + this._authStore.getUser().token; + } + + return Observable.create((observer) => { + let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader}); + eventSource.onmessage = (data => { + this.zone.run(() => { + observer.next(data.data); + }); + + }); + }); + } +} diff --git a/ui/src/assets/worker/web/lastupdate.js b/ui/src/assets/worker/web/lastupdate.js deleted file mode 100644 index 08d775197e..0000000000 --- a/ui/src/assets/worker/web/lastupdate.js +++ /dev/null @@ -1,25 +0,0 @@ -importScripts('../common.js'); - -onmessage = function (e) { - loadLastUpdates(e.data.user, e.data.session, e.data.api); -}; - -var lastUpdate; - -function loadLastUpdates (user, session, api) { - loop(1, function () { - var header = {}; - if (lastUpdate) { - header = {"If-Modified-Since": lastUpdate}; - } - var xhr = httpCall('/mon/lastupdates', api, user, session, header); - if (xhr.status >= 400) { - return true; - } - lastUpdate = xhr.getResponseHeader("ETag"); - if (xhr.status === 200) { - postMessage(xhr.responseText); - } - return false; - }); -} diff --git a/ui/src/polyfills.ts b/ui/src/polyfills.ts index 400600cfbe..9a4f2b071a 100644 --- a/ui/src/polyfills.ts +++ b/ui/src/polyfills.ts @@ -60,7 +60,6 @@ import 'zone.js/dist/zone'; // Included with Angular CLI. /*************************************************************************************************** * APPLICATION IMPORTS */ - /** * Date, currency, decimal and percent pipes. * Needed for: All but Chrome, Firefox, Edge, IE11 and Safari 10 diff --git a/ui/src/typings.d.ts b/ui/src/typings.d.ts index 38e02be2d1..5784cda712 100644 --- a/ui/src/typings.d.ts +++ b/ui/src/typings.d.ts @@ -2,4 +2,3 @@ declare var module: { id: string; }; -declare var Prism: any; From b019a8594328050c4f3285fd2a97a1efb939f5cb Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 10:44:13 +0200 Subject: [PATCH 10/15] fix (api): cr + bug fix --- engine/api/cache/cache.go | 2 +- engine/api/environment_group.go | 12 +++------ engine/api/lastupdate.go | 17 ++++++------ engine/api/pipeline.go | 14 +++------- engine/api/pipeline_parameter.go | 10 ++----- engine/api/project_group.go | 1 - engine/api/repositories_manager.go | 18 +++++-------- engine/api/router.go | 42 ++++++++++++++++-------------- 8 files changed, 47 insertions(+), 69 deletions(-) diff --git a/engine/api/cache/cache.go b/engine/api/cache/cache.go index e6d8fb5aa3..3306dca6f3 100644 --- a/engine/api/cache/cache.go +++ b/engine/api/cache/cache.go @@ -155,7 +155,7 @@ func Subscribe(queueName string) PubSub { } // GetMessageFromSubscription Get a message from a subscription -func GetMessageFromSubscription(pb PubSub, c context.Context) (string, error) { +func GetMessageFromSubscription(c context.Context, pb PubSub) (string, error) { if s == nil { return "", fmt.Errorf("Cache > Client store is nil") } diff --git a/engine/api/environment_group.go b/engine/api/environment_group.go index 37e7c26b3d..60795ead04 100644 --- a/engine/api/environment_group.go +++ b/engine/api/environment_group.go @@ -201,16 +201,10 @@ func addGroupInEnvironmentHandler(w http.ResponseWriter, r *http.Request, db *go func deleteGroupFromEnvironmentHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { // Get project name in URL vars := mux.Vars(r) - key := vars["key"] envName := vars["permEnvironmentName"] groupName := vars["group"] - proj, errP := project.Load(db, key, c.User) - if errP != nil { - return sdk.WrapError(errP, "deleteGroupFromEnvironmentHandler: Cannot load project") - } - - env, errE := environment.LoadEnvironmentByName(db, key, envName) + env, errE := environment.LoadEnvironmentByName(db, c.Project.Key, envName) if errE != nil { return sdk.WrapError(errE, "deleteGroupFromEnvironmentHandler: Cannot load environment") } @@ -221,11 +215,11 @@ func deleteGroupFromEnvironmentHandler(w http.ResponseWriter, r *http.Request, d } defer tx.Rollback() - if err := group.DeleteGroupFromEnvironment(tx, key, envName, groupName); err != nil { + if err := group.DeleteGroupFromEnvironment(tx, c.Project.Key, envName, groupName); err != nil { return sdk.WrapError(err, "deleteGroupFromEnvironmentHandler: Cannot delete group %s from pipeline %s", groupName, envName) } - if err := project.UpdateLastModified(tx, c.User, proj); err != nil { + if err := project.UpdateLastModified(tx, c.User, c.Project); err != nil { return sdk.WrapError(err, "deleteGroupFromEnvironmentHandler: Cannot update project last modified date") } diff --git a/engine/api/lastupdate.go b/engine/api/lastupdate.go index e0e58cdfea..cacbacc2ce 100644 --- a/engine/api/lastupdate.go +++ b/engine/api/lastupdate.go @@ -43,22 +43,23 @@ func InitLastUpdateBroker(c context.Context, DBFunc func() *gorp.DbMap) { go CacheSubscribe(c, lastUpdateBroker.messages) // Start processing events - go lastUpdateBroker.Start(c, DBFunc()) + go lastUpdateBroker.Start(c, DBFunc) } // CacheSubscribe subscribe to a channel and push received message in a channel func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { pubSub := cache.Subscribe("lastUpdates") - tick := time.NewTicker(250 * time.Millisecond).C + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() for { select { case <-c.Done(): if c.Err() != nil { - log.Warning("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) + log.Error("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) return } - case <-tick: - msg, err := cache.GetMessageFromSubscription(pubSub, c) + case <-tick.C: + msg, err := cache.GetMessageFromSubscription(c, pubSub) if err != nil { log.Warning("lastUpdate.CacheSubscribe> Cannot get message %s: %s", msg, err) time.Sleep(5 * time.Second) @@ -70,9 +71,9 @@ func CacheSubscribe(c context.Context, cacheMsgChan chan<- string) { } // Start the broker -func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { - +func (b *LastUpdateBroker) Start(c context.Context, DBFunc func() *gorp.DbMap) { for { + db := DBFunc() select { case <-c.Done(): // Close all channels @@ -81,7 +82,7 @@ func (b *LastUpdateBroker) Start(c context.Context, db gorp.SqlExecutor) { close(v.Queue) } if c.Err() != nil { - log.Warning("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) + log.Error("lastUpdate.CacheSubscribe> Exiting: %v", c.Err()) return } case s := <-b.newClients: diff --git a/engine/api/pipeline.go b/engine/api/pipeline.go index 12d10edd1c..83dc28992c 100644 --- a/engine/api/pipeline.go +++ b/engine/api/pipeline.go @@ -766,15 +766,9 @@ func getPipelineHistoryHandler(w http.ResponseWriter, r *http.Request, db *gorp. func deletePipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { // Get pipeline and action name in URL vars := mux.Vars(r) - projectKey := vars["key"] pipelineName := vars["permPipelineKey"] - proj, errP := project.Load(db, projectKey, c.User) - if errP != nil { - return sdk.WrapError(errP, "deletePipeline> Cannot load project") - } - - p, err := pipeline.LoadPipeline(db, projectKey, pipelineName, false) + p, err := pipeline.LoadPipeline(db, c.Project.Key, pipelineName, false) if err != nil { return sdk.WrapError(err, "deletePipeline> Cannot load pipeline %s", pipelineName) } @@ -799,7 +793,7 @@ func deletePipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *b return err } - if err := project.UpdateLastModified(db, c.User, proj); err != nil { + if err := project.UpdateLastModified(db, c.User, c.Project); err != nil { return sdk.WrapError(err, "deletePipeline> Cannot update project last modified date") } @@ -808,8 +802,8 @@ func deletePipeline(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *b return err } - cache.DeleteAll(cache.Key("application", projectKey, "*")) - cache.Delete(cache.Key("pipeline", projectKey, pipelineName)) + cache.DeleteAll(cache.Key("application", c.Project.Key, "*")) + cache.Delete(cache.Key("pipeline", c.Project.Key, pipelineName)) return nil } diff --git a/engine/api/pipeline_parameter.go b/engine/api/pipeline_parameter.go index e511f4819f..9a218b9021 100644 --- a/engine/api/pipeline_parameter.go +++ b/engine/api/pipeline_parameter.go @@ -84,20 +84,14 @@ func deleteParameterFromPipelineHandler(w http.ResponseWriter, r *http.Request, // Deprecated func updateParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { vars := mux.Vars(r) - key := vars["key"] pipelineName := vars["permPipelineKey"] - proj, errP := project.Load(db, key, c.User) - if errP != nil { - return sdk.WrapError(errP, "updateParametersInPipelineHandler: Cannot load project") - } - var pipParams []sdk.Parameter if err := UnmarshalBody(r, &pipParams); err != nil { return err } - pip, err := pipeline.LoadPipeline(db, key, pipelineName, false) + pip, err := pipeline.LoadPipeline(db, c.Project.Key, pipelineName, false) if err != nil { log.Warning("updateParametersInPipelineHandler: Cannot load %s: %s\n", pipelineName, err) return err @@ -169,7 +163,7 @@ func updateParametersInPipelineHandler(w http.ResponseWriter, r *http.Request, d } } - if err := pipeline.UpdatePipelineLastModified(tx, proj, pip, c.User); err != nil { + if err := pipeline.UpdatePipelineLastModified(tx, c.Project, pip, c.User); err != nil { log.Warning("UpdatePipelineParameters> Cannot update pipeline last_modified date: %s", err) return err diff --git a/engine/api/project_group.go b/engine/api/project_group.go index 2d02e37b01..e3ced7ca9c 100644 --- a/engine/api/project_group.go +++ b/engine/api/project_group.go @@ -134,7 +134,6 @@ func updateGroupsInProject(w http.ResponseWriter, r *http.Request, db *gorp.DbMa } defer tx.Rollback() - if err := group.DeleteGroupProjectByProject(tx, c.Project.ID); err != nil { return sdk.WrapError(err, "updateGroupsInProject: Cannot delete groups") } diff --git a/engine/api/repositories_manager.go b/engine/api/repositories_manager.go index 3693612343..b719c08057 100644 --- a/engine/api/repositories_manager.go +++ b/engine/api/repositories_manager.go @@ -94,16 +94,10 @@ func getRepositoriesManagerForProjectHandler(w http.ResponseWriter, r *http.Requ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { // Get project name in URL vars := mux.Vars(r) - projectKey := vars["permProjectKey"] rmName := vars["name"] - proj, errP := project.Load(db, projectKey, c.User) - if errP != nil { - return sdk.WrapError(errP, "repositoriesManagerAuthorize> Cannot load project") - } - //Load the repositories manager from the DB - rm, err := repositoriesmanager.LoadForProject(db, projectKey, rmName) + rm, err := repositoriesmanager.LoadForProject(db, c.Project.Key, rmName) var lastModified time.Time //If we don't find any repositories manager for the project, let's insert it @@ -122,12 +116,12 @@ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *go } defer tx.Rollback() - if errI := repositoriesmanager.InsertForProject(tx, rm, projectKey); errI != nil { - log.Warning("repositoriesManagerAuthorize> error while inserting repositories manager for project %s: %s\n", projectKey, errI) + if errI := repositoriesmanager.InsertForProject(tx, rm, c.Project.Key); errI != nil { + log.Warning("repositoriesManagerAuthorize> error while inserting repositories manager for project %s: %s\n", c.Project.Key, errI) return errI } - if err := project.UpdateLastModified(tx, c.User, proj); err != nil { + if err := project.UpdateLastModified(tx, c.User, c.Project); err != nil { return sdk.WrapError(err, "repositoriesManagerAuthorize> Cannot update project last modified") } @@ -146,10 +140,10 @@ func repositoriesManagerAuthorize(w http.ResponseWriter, r *http.Request, db *go return sdk.ErrNoReposManagerAuth } - log.Info("repositoriesManagerAuthorize> [%s] RequestToken=%s; URL=%s\n", projectKey, token, url) + log.Info("repositoriesManagerAuthorize> [%s] RequestToken=%s; URL=%s\n", c.Project.Key, token, url) data := map[string]string{ - "project_key": projectKey, + "project_key": c.Project.Key, "last_modified": strconv.FormatInt(lastModified.Unix(), 10), "repositories_manager": rmName, "url": url, diff --git a/engine/api/router.go b/engine/api/router.go index e5abfc9c3c..0f015de14b 100644 --- a/engine/api/router.go +++ b/engine/api/router.go @@ -529,31 +529,33 @@ func businessContext(db gorp.SqlExecutor, c *businesscontext.Ctx, req *http.Requ } } - vars := mux.Vars(req) - key := vars["key"] - if key == "" { - key = vars["permProjectKey"] - } + if c.User != nil { + vars := mux.Vars(req) + key := vars["key"] + if key == "" { + key = vars["permProjectKey"] + } - if key != "" { - proj, errproj := project.Load(db, key, c.User, project.LoadOptions.Default) - if errproj != nil { - return errproj + if key != "" { + proj, errproj := project.Load(db, key, c.User, project.LoadOptions.Default) + if errproj != nil { + return errproj + } + c.Project = proj } - c.Project = proj - } - app := vars["permApplicationName"] - if app == "" { - app = vars["app"] - } + app := vars["permApplicationName"] + if app == "" { + app = vars["app"] + } - if app != "" { - app, errapp := application.LoadByName(db, key, app, c.User, application.LoadOptions.Default) - if errapp != nil { - return errapp + if app != "" { + app, errapp := application.LoadByName(db, key, app, c.User, application.LoadOptions.Default) + if errapp != nil { + return errapp + } + c.Application = app } - c.Application = app } return nil From bc9728226b95df9b2c8f9633fa8fc52fd92a9870 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 10:51:00 +0200 Subject: [PATCH 11/15] fix import --- engine/api/application_group.go | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/api/application_group.go b/engine/api/application_group.go index 35d771a3a5..1137e11f4b 100644 --- a/engine/api/application_group.go +++ b/engine/api/application_group.go @@ -8,7 +8,6 @@ import ( "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/businesscontext" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/engine/api/permission" "github.com/ovh/cds/engine/api/project" From d9d308d0348186876ed20909dba0b1c99089983b Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 15:37:21 +0200 Subject: [PATCH 12/15] feat: not print error on reconnecting --- ui/src/app/service/sse/lastupdate.sservice.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/src/app/service/sse/lastupdate.sservice.ts b/ui/src/app/service/sse/lastupdate.sservice.ts index 42ec8e968a..2e76233edc 100644 --- a/ui/src/app/service/sse/lastupdate.sservice.ts +++ b/ui/src/app/service/sse/lastupdate.sservice.ts @@ -24,12 +24,11 @@ export class LastUpdateService { } return Observable.create((observer) => { - let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader}); + let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader, errorOnTimeout: true}); eventSource.onmessage = (data => { this.zone.run(() => { observer.next(data.data); }); - }); }); } From 64e9942645b4bb256ab7a74e647501dd038f4b7b Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 15:38:46 +0200 Subject: [PATCH 13/15] feat: not print error on reconnecting --- ui/src/app/service/sse/lastupdate.sservice.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/app/service/sse/lastupdate.sservice.ts b/ui/src/app/service/sse/lastupdate.sservice.ts index 2e76233edc..be19df9a5b 100644 --- a/ui/src/app/service/sse/lastupdate.sservice.ts +++ b/ui/src/app/service/sse/lastupdate.sservice.ts @@ -24,7 +24,7 @@ export class LastUpdateService { } return Observable.create((observer) => { - let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader, errorOnTimeout: true}); + let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader, errorOnTimeout: false}); eventSource.onmessage = (data => { this.zone.run(() => { observer.next(data.data); From 1c2e7faa8b2af402435c5dad99803c6eb7d45df5 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 21 Jul 2017 15:56:42 +0200 Subject: [PATCH 14/15] fix; lint --- ui/src/app/service/sse/lastupdate.sservice.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/src/app/service/sse/lastupdate.sservice.ts b/ui/src/app/service/sse/lastupdate.sservice.ts index be19df9a5b..e6360a34e0 100644 --- a/ui/src/app/service/sse/lastupdate.sservice.ts +++ b/ui/src/app/service/sse/lastupdate.sservice.ts @@ -24,7 +24,8 @@ export class LastUpdateService { } return Observable.create((observer) => { - let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', {headers: authHeader, errorOnTimeout: false}); + let eventSource = new EventSourcePolyfill(environment.apiURL + '/mon/lastupdates/events', + {headers: authHeader, errorOnTimeout: false}); eventSource.onmessage = (data => { this.zone.run(() => { observer.next(data.data); From 32ae1be628848cff8e3d201d0c32cfd50ce95c70 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Mon, 24 Jul 2017 14:04:23 +0200 Subject: [PATCH 15/15] fix(ui): fixdeps: https://github.com/angular/angular-cli/issues/7113 --- ui/.angular-cli.json | 1 - ui/package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/.angular-cli.json b/ui/.angular-cli.json index 5eca800ab6..070ff2529a 100644 --- a/ui/.angular-cli.json +++ b/ui/.angular-cli.json @@ -25,7 +25,6 @@ "../node_modules/semantic-ui/dist/semantic.css", "../node_modules/angular2-toaster/toaster.css", "../node_modules/font-awesome/css/font-awesome.css", - "../node_modules/hover.css/css/hover.css", "styles.scss" ], "scripts": [ diff --git a/ui/package.json b/ui/package.json index 162fece2ed..ac95f7f09c 100644 --- a/ui/package.json +++ b/ui/package.json @@ -32,8 +32,8 @@ "dagre-d3": "https://github.com/sguiheux/dagre-d3/tarball/reworkDependencies", "dragula": "^3.7.2", "duration-js": "^3.9.2", + "enhanced-resolve": "3.3.0", "font-awesome": "^4.6.3", - "hover.css": "^2.0.2", "immutable": "^3.8.1", "lodash": "~4.17.2", "moment": "^2.17.1",