diff --git a/cmd/env.go b/cmd/env.go index efc37b548f..420a0c7da3 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -28,6 +28,8 @@ var envList = []string{ localdata.EnvNameSSHPassPrompt, localdata.EnvNameSSHPath, localdata.EnvNameSCPPath, + localdata.EnvNameKeepSourceTarget, + localdata.EnvNameMirrorSyncScript, localdata.EnvNameLogPath, } diff --git a/cmd/mirror.go b/cmd/mirror.go index 15721ac3e1..bb01b960e1 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "runtime" "sort" @@ -27,7 +28,9 @@ import ( "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/repository" - "github.com/pingcap/tiup/pkg/repository/remote" + "github.com/pingcap/tiup/pkg/repository/model" + ru "github.com/pingcap/tiup/pkg/repository/utils" + "github.com/pingcap/tiup/pkg/repository/v0manifest" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/utils" @@ -35,10 +38,6 @@ import ( "github.com/spf13/pflag" ) -var ( - repoPath string -) - func newMirrorCmd() *cobra.Command { cmd := &cobra.Command{ Use: "mirror ", @@ -48,16 +47,6 @@ it to create a private repository, or to add new component to an existing reposi The repository can be used either online or offline. It also provides some useful utilities to help managing keys, users and versions of components or the repository itself.`, - Args: func(cmd *cobra.Command, args []string) error { - if repoPath == "" { - var err error - repoPath, err = os.Getwd() - if err != nil { - return err - } - } - return nil - }, RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return cmd.Help() @@ -66,21 +55,16 @@ of components or the repository itself.`, }, } - cmd.PersistentFlags().StringVar(&repoPath, "repo", "", "Path to the repository") - cmd.AddCommand( newMirrorInitCmd(), newMirrorSignCmd(), - newMirrorOwnerCmd(), - newMirrorCompCmd(), - newMirrorAddCompCmd(), - newMirrorYankCompCmd(), - newMirrorDelCompCmd(), newMirrorGenkeyCmd(), newMirrorCloneCmd(), + newMirrorMergeCmd(), newMirrorPublishCmd(), newMirrorSetCmd(), newMirrorModifyCmd(), + newMirrorGrantCmd(), ) return cmd @@ -108,93 +92,6 @@ func newMirrorSignCmd() *cobra.Command { return cmd } -// the `mirror add` sub command -func newMirrorAddCompCmd() *cobra.Command { - var nightly bool // if this is a nightly version - cmd := &cobra.Command{ - Use: "add ", - Short: "Add a file to a component", - Long: `Add a file to a component, and set its metadata of platform ID and version.`, - Hidden: true, // WIP, remove when it becomes working and stable - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 4 { - return cmd.Help() - } - - return addCompFile(repoPath, args[0], args[1], args[2], args[3], nightly) - }, - } - - // If adding legacy nightly build (e.g., add a version from yesterday), just - // omit the flag to treat it as normal versions - cmd.Flags().BoolVar(&nightly, "nightly", false, "Mark this version as the latest nightly build") - - return cmd -} - -func addCompFile(repo, id, platform, version, file string, nightly bool) error { - // TODO - return nil -} - -// the `mirror component` sub command -func newMirrorCompCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "component ", - Short: "Create a new component in the repository", - Long: `Create a new component in the repository, and sign with the local owner key.`, - Hidden: true, // WIP, remove when it becomes working and stable - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 2 { - return cmd.Help() - } - - return createComp(repoPath, args[0], args[1]) - }, - } - - return cmd -} - -func createComp(repo, id, name string) error { - // TODO - return nil -} - -// the `mirror del` sub command -func newMirrorDelCompCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "del [version]", - Short: "Delete a component from the repository", - Long: `Delete a component from the repository. If version is not specified, all versions -of the given component will be deleted. -Manifests and files of a deleted component will be removed from the repository, -clients can no longer fetch the component, but files already download by clients -may still be available for them.`, - Hidden: true, // WIP, remove when it becomes working and stable - RunE: func(cmd *cobra.Command, args []string) error { - compVer := "" - switch len(args) { - case 2: - compVer = args[1] - default: - return cmd.Help() - } - - return delComp(repoPath, args[0], compVer) - }, - } - - return cmd -} - -func delComp(repo, id, version string) error { - // TODO: implement the func - - // TODO: check if version is the latest nightly, refuse if it is - return nil -} - // the `mirror set` sub command func newMirrorSetCmd() *cobra.Command { root := "" @@ -221,69 +118,119 @@ func newMirrorSetCmd() *cobra.Command { return cmd } +// the `mirror grant` sub command +func newMirrorGrantCmd() *cobra.Command { + name := "" + privPath := "" + + cmd := &cobra.Command{ + Use: "grant ", + Short: "grant a new owner", + Long: "grant a new owner to current mirror", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return cmd.Help() + } + + id := args[0] + if name == "" { + fmt.Printf("The --name is not specified, using %s as default\n", id) + name = id + } + + // the privPath can point to a public key becase the Public method of KeyInfo works on both priv and pub key + privKey, err := loadPrivKey(privPath) + if err != nil { + return err + } + pubKey, err := privKey.Public() + if err != nil { + return err + } + + env := environment.GlobalEnv() + return env.V1Repository().Mirror().Grant(id, name, pubKey) + }, + } + + cmd.Flags().StringVarP(&name, "name", "n", "", "Specify the name of the owner, default: id of the owner") + cmd.Flags().StringVarP(&privPath, "key", "k", "", "Specify the private key path of the owner") + + return cmd +} + // the `mirror modify` sub command func newMirrorModifyCmd() *cobra.Command { var privPath string - endpoint := "" desc := "" standalone := false hidden := false yanked := false cmd := &cobra.Command{ - Use: "modify [:version] [flags]", - Long: "modify component attributes (hidden, standalone, yanked)", + Use: "modify [:version] [flags]", + Short: "Modify published component", + Long: "Modify component attributes (hidden, standalone, yanked)", RunE: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { return cmd.Help() } - env := environment.GlobalEnv() - if privPath == "" { - privPath = env.Profile().Path(localdata.KeyInfoParentDir, "private.json") - } - // Get the private key - f, err := os.Open(privPath) - if err != nil { - return err - } - defer f.Close() + component := args[0] - ki := v1manifest.KeyInfo{} - if err := json.NewDecoder(f).Decode(&ki); err != nil { - return err - } + env := environment.GlobalEnv() - comp, ver := environment.ParseCompVersion(args[0]) + comp, ver := environment.ParseCompVersion(component) m, err := env.V1Repository().FetchComponentManifest(comp, true) if err != nil { return err } - if endpoint == "" { - endpoint = environment.Mirror() + v1manifest.RenewManifest(m, time.Now()) + if desc != "" { + m.Description = desc } - e := remote.NewEditor(endpoint, comp).WithDesc(desc).WithVersion(ver.String()) + flagSet := set.NewStringSet() cmd.Flags().Visit(func(f *pflag.Flag) { flagSet.Insert(f.Name) }) - if flagSet.Exist("standalone") { - e.Standalone(standalone) - } - if flagSet.Exist("hide") { - e.Hide(hidden) + + publishInfo := &model.PublishInfo{} + if ver == "" { + if flagSet.Exist("standalone") { + publishInfo.Stand = &standalone + } + if flagSet.Exist("hide") { + publishInfo.Hide = &hidden + } + if flagSet.Exist("yank") { + publishInfo.Yank = &yanked + } + } else if flagSet.Exist("yank") { + if v0manifest.Version(ver).IsNightly() { + return errors.New("nightly version can't be yanked") + } + for p := range m.Platforms { + vi, ok := m.Platforms[p][ver.String()] + if !ok { + continue + } + vi.Yanked = yanked + m.Platforms[p][ver.String()] = vi + } } - if flagSet.Exist("yank") { - e.Yank(yanked) + + manifest, err := sign(privPath, m) + if err != nil { + return err } - return e.Sign(&ki, m) + return env.V1Repository().Mirror().Publish(manifest, publishInfo) }, } cmd.Flags().StringVarP(&privPath, "key", "k", "", "private key path") - cmd.Flags().StringVarP(&endpoint, "endpoint", "", endpoint, "endpoint of the server") cmd.Flags().StringVarP(&desc, "desc", "", desc, "description of the component") cmd.Flags().BoolVarP(&standalone, "standalone", "", standalone, "can this component run directly") cmd.Flags().BoolVarP(&hidden, "hide", "", hidden, "is this component visible in list") @@ -291,10 +238,72 @@ func newMirrorModifyCmd() *cobra.Command { return cmd } +func loadPrivKey(privPath string) (*v1manifest.KeyInfo, error) { + env := environment.GlobalEnv() + if privPath == "" { + privPath = env.Profile().Path(localdata.KeyInfoParentDir, "private.json") + } + + // Get the private key + f, err := os.Open(privPath) + if err != nil { + return nil, err + } + defer f.Close() + + ki := v1manifest.KeyInfo{} + if err := json.NewDecoder(f).Decode(&ki); err != nil { + return nil, errors.Annotate(err, "decode key") + } + + return &ki, nil +} + +func loadPrivKeys(keysDir string) (map[string]*v1manifest.KeyInfo, error) { + keys := map[string]*v1manifest.KeyInfo{} + + err := filepath.Walk(keysDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + ki, err := loadPrivKey(path) + if err != nil { + return err + } + + id, err := ki.ID() + if err != nil { + return err + } + + keys[id] = ki + return nil + }) + + if err != nil { + return nil, err + } + + return keys, nil +} + +func sign(privPath string, signed v1manifest.ValidManifest) (*v1manifest.Manifest, error) { + ki, err := loadPrivKey(privPath) + if err != nil { + return nil, err + } + + return v1manifest.SignManifest(signed, ki) +} + // the `mirror publish` sub command func newMirrorPublishCmd() *cobra.Command { var privPath string - endpoint := environment.Mirror() goos := runtime.GOOS goarch := runtime.GOARCH desc := "" @@ -310,64 +319,53 @@ func newMirrorPublishCmd() *cobra.Command { return cmd.Help() } + component, version, tarpath, entry := args[0], args[1], args[2], args[3] + if err := validatePlatform(goos, goarch); err != nil { return err } - env := environment.GlobalEnv() - if privPath == "" { - privPath = env.Profile().Path(localdata.KeyInfoParentDir, "private.json") + hashes, length, err := ru.HashFile(tarpath) + if err != nil { + return err } - // Get the private key - f, err := os.Open(privPath) + tarfile, err := os.Open(tarpath) if err != nil { - return err + return errors.Annotatef(err, "open tarball: %s", tarpath) } - defer f.Close() + defer tarfile.Close() - ki := v1manifest.KeyInfo{} - if err := json.NewDecoder(f).Decode(&ki); err != nil { - return err + publishInfo := &model.PublishInfo{ + ComponentData: &model.TarInfo{Reader: tarfile, Name: fmt.Sprintf("%s-%s-%s-%s.tar.gz", component, version, goos, goarch)}, } flagSet := set.NewStringSet() cmd.Flags().Visit(func(f *pflag.Flag) { flagSet.Insert(f.Name) }) - m, err := env.V1Repository().FetchComponentManifest(args[0], true) + env := environment.GlobalEnv() + m, err := env.V1Repository().FetchComponentManifest(component, true) if err != nil { fmt.Printf("Fetch local manifest: %s\n", err.Error()) fmt.Printf("Failed to load component manifest, create a new one\n") + publishInfo.Stand = &standalone + publishInfo.Hide = &hidden } else if flagSet.Exist("standalone") || flagSet.Exist("hide") { fmt.Println("This is not a new component, --standalone and --hide flag will be omited") } - t := remote.NewTransporter(endpoint, args[0], args[1], args[3]).WithDesc(desc).WithOS(goos).WithArch(goarch) - if m == nil && standalone { - t = t.Standalone() - } - if m == nil && hidden { - t = t.Hide() - } - - if err := t.Open(args[2]); err != nil { - return err - } - defer t.Close() - - if err := t.Upload(); err != nil { - fmt.Printf("Failed to upload component: %s\n", err.Error()) - return err - } + m = repository.UpdateManifestForPublish(m, component, version, entry, goos, goarch, desc, v1manifest.FileHash{ + Hashes: hashes, + Length: uint(length), + }) - if err := t.Sign(&ki, m); err != nil { - fmt.Printf("Sign component manifest: %s\n", err.Error()) + manifest, err := sign(privPath, m) + if err != nil { return err } - fmt.Printf("Upload %s(%s) for platform %s/%s success\n", args[0], args[1], goos, goarch) - return nil + return env.V1Repository().Mirror().Publish(manifest, publishInfo) }, } @@ -375,7 +373,6 @@ func newMirrorPublishCmd() *cobra.Command { cmd.Flags().StringVarP(&goos, "os", "", goos, "the target operation system") cmd.Flags().StringVarP(&goarch, "arch", "", goarch, "the target system architecture") cmd.Flags().StringVarP(&desc, "desc", "", desc, "description of the component") - cmd.Flags().StringVarP(&endpoint, "endpoint", "", endpoint, "endpoint of the server") cmd.Flags().BoolVarP(&standalone, "standalone", "", standalone, "can this component run directly") cmd.Flags().BoolVarP(&hidden, "hide", "", hidden, "is this component invisible on listing") return cmd @@ -400,34 +397,30 @@ func newMirrorGenkeyCmd() *cobra.Command { var ( showPublic bool saveKey bool - privPath string + name string ) cmd := &cobra.Command{ Use: "genkey", Short: "Generate a new key pair", Long: `Generate a new key pair that can be used to sign components.`, - PreRunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { env := environment.GlobalEnv() - privPath = env.Profile().Path(localdata.KeyInfoParentDir, "private.json") + privPath := env.Profile().Path(localdata.KeyInfoParentDir, name+".json") keyDir := filepath.Dir(privPath) if utils.IsNotExist(keyDir) { - return os.Mkdir(keyDir, 0755) + if err := os.Mkdir(keyDir, 0755); err != nil { + return errors.Annotate(err, "create private key dir") + } } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { + + var ki *v1manifest.KeyInfo + var err error if showPublic { - f, err := os.Open(privPath) + ki, err = loadPrivKey(privPath) if err != nil { return err } - defer f.Close() - - ki := v1manifest.KeyInfo{} - if err := json.NewDecoder(f).Decode(&ki); err != nil { - return err - } pki, err := ki.Public() if err != nil { return err @@ -442,55 +435,41 @@ func newMirrorGenkeyCmd() *cobra.Command { } fmt.Printf("KeyID: %s\nKeyContent: \n%s\n", id, string(content)) - - // TODO: suggest key type from input, there will also be owner keys - if saveKey { - pubKey, err := ki.Public() - if err != nil { - return err - } - if err = v1manifest.SaveKeyInfo(pubKey, "root", ""); err != nil { - return err - } - fmt.Printf("public key have been write to current working dir\n") + } else { + if utils.IsExist(privPath) { + fmt.Println("Key already exists, skipped") + return nil } - return nil - } - if utils.IsExist(privPath) { - fmt.Println("Key already exists, skipped") - return nil - } + ki, err = v1manifest.GenKeyInfo() + if err != nil { + return err + } - key, err := v1manifest.GenKeyInfo() - if err != nil { - return err - } + f, err := os.Create(privPath) + if err != nil { + return err + } + defer f.Close() - f, err := os.Create(privPath) - if err != nil { - return err - } - defer f.Close() + // set private key permission + if err = f.Chmod(0600); err != nil { + return err + } - // set private key permission - if err = f.Chmod(0600); err != nil { - return err - } + if err := json.NewEncoder(f).Encode(ki); err != nil { + return err + } - if err := json.NewEncoder(f).Encode(key); err != nil { - return err + fmt.Printf("private key have been write to %s\n", privPath) } - fmt.Printf("private key have been write to %s\n", privPath) - - // TODO: suggest key type from input, there will also be owner keys if saveKey { - pubKey, err := key.Public() + pubKey, err := ki.Public() if err != nil { return err } - if err = v1manifest.SaveKeyInfo(pubKey, "root", ""); err != nil { + if err = v1manifest.SaveKeyInfo(pubKey, "public", ""); err != nil { return err } fmt.Printf("public key have been write to current working dir\n") @@ -498,8 +477,10 @@ func newMirrorGenkeyCmd() *cobra.Command { return nil }, } - cmd.Flags().BoolVarP(&showPublic, "public", "p", showPublic, fmt.Sprintf("show public content of %s", privPath)) + + cmd.Flags().BoolVarP(&showPublic, "public", "p", showPublic, "show public content") cmd.Flags().BoolVar(&saveKey, "save", false, "Save public key to a file at current working dir") + cmd.Flags().StringVarP(&name, "name", "n", "private", "the file name of the key") return cmd } @@ -510,14 +491,15 @@ func newMirrorInitCmd() *cobra.Command { keyDir string // Directory to write genreated key files ) cmd := &cobra.Command{ - Use: "init [path]", + Use: "init ", Short: "Initialize an empty repository", Long: `Initialize an empty TiUP repository at given path. If path is not specified, the current working directory (".") will be used.`, RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 1 { - repoPath = args[0] + if len(args) != 1 { + return cmd.Help() } + repoPath := args[0] // create the target path if not exist if utils.IsNotExist(repoPath) { @@ -535,6 +517,9 @@ current working directory (".") will be used.`, return errors.Errorf("the target path '%s' is not an empty directory", repoPath) } + if keyDir == "" { + keyDir = path.Join(repoPath, "keys") + } return initRepo(repoPath, keyDir) }, } @@ -548,63 +533,46 @@ func initRepo(path, keyDir string) error { return v1manifest.Init(path, keyDir, time.Now().UTC()) } -// the `mirror owner` sub command -func newMirrorOwnerCmd() *cobra.Command { +// the `mirror merge` sub command +func newMirrorMergeCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "owner ", - Short: "Create a new owner for the repository", - Long: `Create a new owner role for the repository, the owner can then perform management -actions on authorized resources.`, - Hidden: true, // WIP, remove when it becomes working and stable + Use: "merge [mirror-dir-N]", + Example: ` tiup mirror merge tidb-community-v4.0.1 # merge v4.0.1 into current mirror + tiup mirror merge tidb-community-v4.0.1 tidb-community-v4.0.2 # merge v4.0.1 and v4.0.2 into current mirror`, + Short: "Merge two or more offline mirror", RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 2 { + if len(args) < 1 { return cmd.Help() } - return createOwner(repoPath, args[0], args[1]) - }, - } + sources := args - return cmd -} + env := environment.GlobalEnv() + baseMirror := env.V1Repository().Mirror() -func createOwner(repo, id, name string) error { - // TODO - return nil -} + sourceMirrors := []repository.Mirror{} + for _, source := range sources { + sourceMirror := repository.NewMirror(source, repository.MirrorOptions{}) + if err := sourceMirror.Open(); err != nil { + return err + } + defer sourceMirror.Close() -// the `mirror yank` sub command -func newMirrorYankCompCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "yank [version]", - Short: "Yank a component in the repository", - Long: `Yank a component in the repository. If version is not specified, all versions -of the given component will be yanked. -A yanked component is still in the repository, but not visible to client, and is -no longer considered stable to use. A yanked component is expected to be removed -from the repository in the future.`, - Hidden: true, // WIP, remove when it becomes working and stable - RunE: func(cmd *cobra.Command, args []string) error { - compVer := "" - switch len(args) { - case 2: - compVer = args[1] - default: - return cmd.Help() + sourceMirrors = append(sourceMirrors, sourceMirror) } - return yankComp(repoPath, args[0], compVer) + keys, err := loadPrivKeys(env.Profile().Path(localdata.KeyInfoParentDir)) + if err != nil { + return err + } + + return repository.MergeMirror(keys, baseMirror, sourceMirrors...) }, } return cmd } -func yankComp(repo, id, version string) error { - // TODO - return nil -} - // the `mirror clone` sub command func newMirrorCloneCmd() *cobra.Command { var ( diff --git a/go.mod b/go.mod index 7ac8dd8635..bf8e12c5d5 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/gibson042/canonicaljson-go v1.0.3 github.com/gizak/termui/v3 v3.1.0 github.com/go-sql-driver/mysql v1.5.0 + github.com/gofrs/flock v0.8.0 github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.3.4 github.com/google/uuid v1.1.1 diff --git a/go.sum b/go.sum index 71f33b3a6f..ae7b459cbd 100644 --- a/go.sum +++ b/go.sum @@ -307,6 +307,8 @@ github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQF github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gocql/gocql v0.0.0-20200103014340-68f928edb90a/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/godror/godror v0.10.3/go.mod h1:9MVLtu25FBJBMHkPs0m3Ngf/VmwGcLpM2HS8PlNGw9U= +github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= +github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= diff --git a/pkg/cluster/audit/audit.go b/pkg/cluster/audit/audit.go index 004eafea4f..d5a2406b44 100644 --- a/pkg/cluster/audit/audit.go +++ b/pkg/cluster/audit/audit.go @@ -17,6 +17,7 @@ import ( "bufio" "fmt" "io/ioutil" + "math/rand" "os" "path/filepath" "sort" @@ -81,7 +82,7 @@ func ShowAuditList(dir string) error { // OutputAuditLog outputs audit log. func OutputAuditLog(dir string, data []byte) error { - fname := filepath.Join(dir, base52.Encode(time.Now().UnixNano())) + fname := filepath.Join(dir, base52.Encode(time.Now().UnixNano()+rand.Int63n(1000))) return ioutil.WriteFile(fname, data, 0644) } diff --git a/pkg/file/file.go b/pkg/file/file.go index 3d91f1d1ca..db3212d1dc 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -20,8 +20,6 @@ var ( // e.g., backup meta.yaml as meta-2006-01-02T15:04:05Z07:00.yaml // backup the files in the same dir of path if backupDir is empty. func SaveFileWithBackup(path string, data []byte, backupDir string) error { - timestr := time.Now().Format(time.RFC3339Nano) - info, err := os.Stat(path) if err != nil && !os.IsNotExist(err) { return errors.AddStack(err) @@ -46,6 +44,7 @@ func SaveFileWithBackup(path string, data []byte, backupDir string) error { dir := filepath.Dir(path) var backupName string + timestr := time.Now().Format(time.RFC3339Nano) p := strings.Split(base, ".") if len(p) == 1 { backupName = base + "-" + timestr diff --git a/pkg/localdata/constant.go b/pkg/localdata/constant.go index c61ac3bbbf..8e15fa8146 100644 --- a/pkg/localdata/constant.go +++ b/pkg/localdata/constant.go @@ -81,6 +81,9 @@ const ( // EnvNameKeepSourceTarget is the variable name by which user can keep the source target or not EnvNameKeepSourceTarget = "TIUP_KEEP_SOURCE_TARGET" + // EnvNameMirrorSyncScript make it possible for user to sync mirror commit to other place (eg. CDN) + EnvNameMirrorSyncScript = "TIUP_MIRROR_SYNC_SCRIPT" + // EnvNameLogPath is the variable name by which user can write the log files into EnvNameLogPath = "TIUP_LOG_PATH" diff --git a/pkg/repository/merge_mirror.go b/pkg/repository/merge_mirror.go new file mode 100644 index 0000000000..17d8198266 --- /dev/null +++ b/pkg/repository/merge_mirror.go @@ -0,0 +1,292 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "fmt" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/repository/model" + "github.com/pingcap/tiup/pkg/repository/v0manifest" + "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/pingcap/tiup/pkg/version" +) + +type diffItem struct { + name string + componentItem v1manifest.ComponentItem + versionItem v1manifest.VersionItem + version string + os string + arch string + desc string +} + +// returns component that exists in addition but not in base +func diffMirror(base, addition Mirror) ([]diffItem, error) { + baseIndex, err := fetchIndexManifestFromMirror(base) + if err != nil { + return nil, err + } + additionIndex, err := fetchIndexManifestFromMirror(addition) + if err != nil { + return nil, err + } + + items := []diffItem{} + + baseComponents := baseIndex.ComponentListWithYanked() + additionComponents := additionIndex.ComponentList() + + for name, comp := range additionComponents { + if baseComponents[name].Yanked { + continue + } + + baseComponent, err := fetchComponentManifestFromMirror(base, name) + if err != nil { + return nil, err + } + additionComponent, err := fetchComponentManifestFromMirror(addition, name) + if err != nil { + return nil, err + } + + items = append(items, component2Diff(name, baseComponents[name], baseComponent, comp, additionComponent)...) + } + + return items, nil +} + +func component2Diff(name string, baseItem v1manifest.ComponentItem, baseManifest *v1manifest.Component, + additionItem v1manifest.ComponentItem, additionManifest *v1manifest.Component) []diffItem { + items := []diffItem{} + + for plat := range additionManifest.Platforms { + versions := additionManifest.VersionList(plat) + for ver, verinfo := range versions { + // Don't merge nightly this time + if v0manifest.Version(ver).IsNightly() { + continue + } + + // this version not exits in base + if baseManifest.VersionListWithYanked(plat)[ver].URL == "" { + osArch := strings.Split(plat, "/") + if len(osArch) != 2 { + continue + } + + item := diffItem{ + name: name, + componentItem: baseItem, + versionItem: verinfo, + version: ver, + os: osArch[0], + arch: osArch[1], + desc: additionManifest.Description, + } + + if baseItem.URL == "" { + item.componentItem = additionItem + } + items = append(items, item) + } + } + } + + return items +} + +// MergeMirror merges two or more mirrors +func MergeMirror(keys map[string]*v1manifest.KeyInfo, base Mirror, additions ...Mirror) error { + ownerKeys, err := mapOwnerKeys(base, keys) + if err != nil { + return err + } + + for _, addition := range additions { + diffs, err := diffMirror(base, addition) + if err != nil { + return err + } + + for _, diff := range diffs { + if len(ownerKeys[diff.componentItem.Owner]) == 0 { + return errors.Errorf("missing owner keys for owner %s on component %s", diff.componentItem.Owner, diff.name) + } + + comp, err := fetchComponentManifestFromMirror(base, diff.name) + if err != nil { + return err + } + + comp = UpdateManifestForPublish(comp, diff.name, diff.version, diff.versionItem.Entry, diff.os, diff.arch, diff.desc, diff.versionItem.FileHash) + manifest, err := v1manifest.SignManifest(comp, ownerKeys[diff.componentItem.Owner]...) + if err != nil { + return err + } + + resource := strings.TrimPrefix(diff.versionItem.URL, "/") + tarfile, err := addition.Fetch(resource, 0) + if err != nil { + return err + } + defer tarfile.Close() + + publishInfo := &model.PublishInfo{ + ComponentData: &model.TarInfo{Reader: tarfile, Name: resource}, + Stand: &diff.componentItem.Standalone, + Hide: &diff.componentItem.Hidden, + } + + if err := base.Publish(manifest, publishInfo); err != nil { + return err + } + } + } + return nil +} + +func fetchComponentManifestFromMirror(mirror Mirror, component string) (*v1manifest.Component, error) { + r, err := mirror.Fetch(v1manifest.ManifestFilenameSnapshot, 0) + if err != nil { + return nil, err + } + defer r.Close() + + snap := v1manifest.Snapshot{} + if _, err := v1manifest.ReadNoVerify(r, &snap); err != nil { + return nil, err + } + + v := snap.Meta[fmt.Sprintf("/%s.json", component)].Version + if v == 0 { + // nil means that the component manifest not found + return nil, nil + } + + r, err = mirror.Fetch(fmt.Sprintf("%d.%s.json", v, component), 0) + if err != nil { + return nil, err + } + defer r.Close() + + role := v1manifest.Component{} + // TODO: this time we just assume the addition mirror is trusted + if _, err := v1manifest.ReadNoVerify(r, &role); err != nil { + return nil, err + } + + return &role, nil +} + +func fetchIndexManifestFromMirror(mirror Mirror) (*v1manifest.Index, error) { + r, err := mirror.Fetch(v1manifest.ManifestFilenameSnapshot, 0) + if err != nil { + return nil, err + } + defer r.Close() + + snap := v1manifest.Snapshot{} + if _, err := v1manifest.ReadNoVerify(r, &snap); err != nil { + return nil, err + } + + indexVersion := snap.Meta[v1manifest.ManifestURLIndex].Version + if indexVersion == 0 { + return nil, errors.Errorf("missing index manifest in base mirror") + } + + r, err = mirror.Fetch(fmt.Sprintf("%d.%s", indexVersion, v1manifest.ManifestFilenameIndex), 0) + if err != nil { + return nil, err + } + defer r.Close() + + index := v1manifest.Index{} + if _, err := v1manifest.ReadNoVerify(r, &index); err != nil { + return nil, err + } + + return &index, nil +} + +// the keys in param is keyID -> KeyInfo, we should map it to ownerID -> KeyInfoList +func mapOwnerKeys(base Mirror, keys map[string]*v1manifest.KeyInfo) (map[string][]*v1manifest.KeyInfo, error) { + index, err := fetchIndexManifestFromMirror(base) + if err != nil { + return nil, err + } + + keyList := map[string][]*v1manifest.KeyInfo{} + for ownerID, owner := range index.Owners { + for keyID := range owner.Keys { + if key := keys[keyID]; key != nil { + keyList[ownerID] = append(keyList[ownerID], key) + } + } + if len(keyList[ownerID]) < owner.Threshold { + // We set keys of this owner to empty becase we can't clone components belong to this owner + keyList[ownerID] = nil + } + } + return keyList, nil +} + +// UpdateManifestForPublish set corresponding field for component manifest +func UpdateManifestForPublish(m *v1manifest.Component, + name, ver, entry, os, arch, desc string, + filehash v1manifest.FileHash) *v1manifest.Component { + initTime := time.Now() + + // update manifest + if m == nil { + m = v1manifest.NewComponent(name, desc, initTime) + } else { + v1manifest.RenewManifest(m, initTime) + if desc != "" { + m.Description = desc + } + } + + if strings.Contains(ver, version.NightlyVersion) { + m.Nightly = ver + } + + // Remove history nightly + for plat := range m.Platforms { + for v := range m.Platforms[plat] { + if strings.Contains(v, version.NightlyVersion) && v != m.Nightly { + delete(m.Platforms[plat], v) + } + } + } + + platformStr := fmt.Sprintf("%s/%s", os, arch) + if m.Platforms[platformStr] == nil { + m.Platforms[platformStr] = map[string]v1manifest.VersionItem{} + } + + m.Platforms[platformStr][ver] = v1manifest.VersionItem{ + Entry: entry, + Released: initTime.Format(time.RFC3339), + URL: fmt.Sprintf("/%s-%s-%s-%s.tar.gz", name, ver, os, arch), + FileHash: filehash, + } + + return m +} diff --git a/pkg/repository/merge_mirror_test.go b/pkg/repository/merge_mirror_test.go new file mode 100644 index 0000000000..28186f12f3 --- /dev/null +++ b/pkg/repository/merge_mirror_test.go @@ -0,0 +1,232 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "encoding/json" + "io/ioutil" + "testing" + + "github.com/pingcap/tiup/pkg/repository/model" + "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/pingcap/tiup/pkg/utils/mock" + "github.com/stretchr/testify/assert" +) + +func manifest2str(m *v1manifest.Manifest) string { + b, err := json.Marshal(m) + if err != nil { + panic(err) + } + return string(b) +} + +func baseMirror4test(ownerKeys map[string]*v1manifest.KeyInfo) Mirror { + return &MockMirror{ + Resources: map[string]string{ + "snapshot.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Snapshot{ + Meta: map[string]v1manifest.FileVersion{ + "/index.json": { + Version: 1, + }, + "/test.json": { + Version: 1, + }, + }, + }, + }), + "1.index.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Index{ + Owners: map[string]v1manifest.Owner{ + "pingcap": { + Name: "PingCAP", + Keys: ownerKeys, + Threshold: 1, + }, + }, + Components: map[string]v1manifest.ComponentItem{ + "test": { + Owner: "pingcap", + URL: "/test.json", + }, + }, + }, + }), + "1.test.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Component{ + Platforms: map[string]map[string]v1manifest.VersionItem{ + "linux/amd64": { + "v1.0.0": { + URL: "/test-v1.0.0-linux-amd64.tar.gz", + Entry: "test", + }, + }, + }, + }, + }), + }, + } +} + +func sourceMirror4test() Mirror { + return &MockMirror{ + Resources: map[string]string{ + "snapshot.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Snapshot{ + Meta: map[string]v1manifest.FileVersion{ + "/index.json": { + Version: 2, + }, + "/hello.json": { + Version: 1, + }, + "/test.json": { + Version: 1, + }, + }, + }, + }), + "2.index.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Index{ + Components: map[string]v1manifest.ComponentItem{ + "test": { + Owner: "pingcap", + URL: "/test.json", + }, + "hello": { + Owner: "pingcap", + URL: "/hello.json", + }, + }, + }, + }), + "1.test.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Component{ + Platforms: map[string]map[string]v1manifest.VersionItem{ + "linux/amd64": { + "v1.0.0": { + URL: "/test-v1.0.0-linux-amd64.tar.gz", + Entry: "test", + }, + "v1.0.1": { + URL: "/test-v1.0.1-linux-amd64.tar.gz", + Entry: "test", + }, + }, + "linux/arm64": { + "v1.0.0": { + URL: "/test-v1.0.0-linux-arm64.tar.gz", + Entry: "test", + }, + }, + }, + }, + }), + "1.hello.json": manifest2str(&v1manifest.Manifest{ + Signed: &v1manifest.Component{ + Platforms: map[string]map[string]v1manifest.VersionItem{ + "linux/amd64": { + "v1.0.0": { + URL: "/hello-v1.0.0-linux-amd64.tar.gz", + Entry: "hello", + }, + }, + }, + }, + }), + "hello-v1.0.0-linux-amd64.tar.gz": "hello-v1.0.0-linux-amd64.tar.gz", + "test-v1.0.1-linux-amd64.tar.gz": "test-v1.0.1-linux-amd64.tar.gz", + "test-v1.0.0-linux-arm64.tar.gz": "test-v1.0.0-linux-arm64.tar.gz", + }, + } +} + +func TestDiffMirror(t *testing.T) { + base := baseMirror4test(nil) + source := sourceMirror4test() + + items, err := diffMirror(base, source) + assert.Nil(t, err) + assert.Equal(t, 3, len(items)) + for _, it := range items { + assert.Contains(t, []string{ + "/hello-v1.0.0-linux-amd64.tar.gz", + "/test-v1.0.0-linux-arm64.tar.gz", + "/test-v1.0.1-linux-amd64.tar.gz", + }, it.versionItem.URL) + } +} + +func TestMergeMirror(t *testing.T) { + ki, err := v1manifest.GenKeyInfo() + if err != nil { + panic(err) + } + id, err := ki.ID() + if err != nil { + panic(err) + } + + keys := map[string]*v1manifest.KeyInfo{ + id: ki, + } + base := baseMirror4test(keys) + source := sourceMirror4test() + + //manifestList := []*v1manifest.Manifest{} + //componentInfoList := []model.ComponentInfo{} + defer mock.With("Publish", func(manifest *v1manifest.Manifest, info model.ComponentInfo) { + assert.Contains(t, []string{ + "hello-v1.0.0-linux-amd64.tar.gz", + "test-v1.0.0-linux-arm64.tar.gz", + "test-v1.0.1-linux-amd64.tar.gz", + }, info.Filename()) + + b, err := ioutil.ReadAll(info) + assert.Nil(t, err) + assert.Contains(t, []string{ + "hello-v1.0.0-linux-amd64.tar.gz", + "test-v1.0.0-linux-arm64.tar.gz", + "test-v1.0.1-linux-amd64.tar.gz", + }, string(b)) + })() + + err = MergeMirror(keys, base, source) + assert.Nil(t, err) +} + +func TestFetchIndex(t *testing.T) { + source := sourceMirror4test() + index, err := fetchIndexManifestFromMirror(source) + assert.Nil(t, err) + assert.NotEmpty(t, index.Components["hello"].URL) + assert.NotEmpty(t, index.Components["test"].URL) + + base := baseMirror4test(nil) + index, err = fetchIndexManifestFromMirror(base) + assert.Nil(t, err) + assert.NotEmpty(t, index.Owners["pingcap"].Name) +} + +func TestFetchComponent(t *testing.T) { + source := sourceMirror4test() + comp, err := fetchComponentManifestFromMirror(source, "test") + assert.Nil(t, err) + assert.NotEmpty(t, comp.Platforms["linux/amd64"]) + assert.NotEmpty(t, comp.Platforms["linux/arm64"]) + assert.NotEmpty(t, comp.Platforms["linux/amd64"]["v1.0.0"].URL) + assert.NotEmpty(t, comp.Platforms["linux/amd64"]["v1.0.1"].URL) + assert.NotEmpty(t, comp.Platforms["linux/arm64"]["v1.0.0"].URL) +} diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index 901810f9c6..78585e4f7d 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -14,12 +14,15 @@ package repository import ( + "bytes" + "encoding/json" stderrors "errors" "fmt" "io" "io/ioutil" "math/rand" "net/http" + "net/url" "os" "path/filepath" "strconv" @@ -27,12 +30,26 @@ import ( "time" "github.com/cavaliercoder/grab" + "github.com/google/uuid" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/repository/model" + "github.com/pingcap/tiup/pkg/repository/store" + "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/utils" + "github.com/pingcap/tiup/pkg/utils/mock" "github.com/pingcap/tiup/pkg/verbose" ) +const ( + // OptionYanked is the key that represents a component is yanked or not + OptionYanked = "yanked" + // OptionStandalone is the key that represents a component is standalone or not + OptionStandalone = "standalone" + // OptionHidden is the key that represents a component is hidden or not + OptionHidden = "hidden" +) + // ErrNotFound represents the resource not exists. var ErrNotFound = stderrors.New("not found") @@ -47,11 +64,13 @@ type ( // MirrorOptions is used to customize the mirror download options MirrorOptions struct { Progress DownloadProgress + Upstream string } // Mirror represents a repository mirror, which can be remote HTTP // server or a local file system directory Mirror interface { + model.Backend // Source returns the address of the mirror Source() string // Open initialize the mirror. @@ -79,11 +98,13 @@ func NewMirror(mirror string, options MirrorOptions) Mirror { options: options, } } - return &localFilesystem{rootPath: mirror} + return &localFilesystem{rootPath: mirror, upstream: options.Upstream} } type localFilesystem struct { rootPath string + upstream string + keys map[string]*v1manifest.KeyInfo } // Source implements the Mirror interface @@ -100,6 +121,72 @@ func (l *localFilesystem) Open() error { if !fi.IsDir() { return errors.Errorf("local system mirror `%s` should be a directory", l.rootPath) } + + if utils.IsNotExist(filepath.Join(l.rootPath, "keys")) { + return nil + } + + return l.loadKeys() +} + +// load mirror keys +func (l *localFilesystem) loadKeys() error { + l.keys = make(map[string]*v1manifest.KeyInfo) + return filepath.Walk(filepath.Join(l.rootPath, "keys"), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return errors.Annotate(err, "open file while loadKeys") + } + defer f.Close() + + ki := v1manifest.KeyInfo{} + if err := json.NewDecoder(f).Decode(&ki); err != nil { + return errors.Annotate(err, "decode key") + } + + id, err := ki.ID() + if err != nil { + return err + } + + l.keys[id] = &ki + return nil + }) +} + +// Publish implements the model.Backend interface +func (l *localFilesystem) Publish(manifest *v1manifest.Manifest, info model.ComponentInfo) error { + txn, err := store.New(l.rootPath, l.upstream).Begin() + if err != nil { + return err + } + + if err := model.New(txn, l.keys).Publish(manifest, info); err != nil { + _ = txn.Rollback() + return err + } + + return nil +} + +// Grant implements the model.Backend interface +func (l *localFilesystem) Grant(id, name string, key *v1manifest.KeyInfo) error { + txn, err := store.New(l.rootPath, l.upstream).Begin() + if err != nil { + return err + } + + if err := model.New(txn, l.keys).Grant(id, name, key); err != nil { + _ = txn.Rollback() + return err + } + return nil } @@ -249,6 +336,72 @@ func (l *httpMirror) prepareURL(resource string) string { return url } +// Grant implements the model.Backend interface +func (l *httpMirror) Grant(id, name string, key *v1manifest.KeyInfo) error { + return errors.Errorf("introduce a fresher via the internet is not allowd, please set you mirror to a local one") +} + +// Publish implements the model.Backend interface +func (l *httpMirror) Publish(manifest *v1manifest.Manifest, info model.ComponentInfo) error { + sid := uuid.New().String() + + if info.Filename() != "" { + tarAddr := fmt.Sprintf("%s/api/v1/tarball/%s", l.Source(), sid) + resp, err := utils.PostFile(info, tarAddr, "file", info.Filename()) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + return errors.Errorf("error on uplaod tarbal, server returns %d", resp.StatusCode) + } + } + + payload, err := json.Marshal(manifest) + if err != nil { + return err + } + bodyBuf := bytes.NewBuffer(payload) + q := url.Values{} + if info.Yanked() != nil { + q.Set(OptionYanked, fmt.Sprintf("%t", *info.Yanked())) + } + if info.Standalone() != nil { + q.Set(OptionStandalone, fmt.Sprintf("%t", *info.Standalone())) + } + if info.Hidden() != nil { + q.Set(OptionHidden, fmt.Sprintf("%t", *info.Hidden())) + } + qstr := "" + if len(q) > 0 { + qstr = "?" + q.Encode() + } + manifestAddr := fmt.Sprintf("%s/api/v1/component/%s/%s%s", l.Source(), sid, manifest.Signed.(*v1manifest.Component).ID, qstr) + + client := http.Client{Timeout: time.Minute} + resp, err := client.Post(manifestAddr, "text/json", bodyBuf) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 300 { + return nil + } else if resp.StatusCode == http.StatusConflict { + return errors.Errorf("Local component manifest is not new enough, update it first") + } else if resp.StatusCode == http.StatusForbidden { + return errors.Errorf("The server refused, make sure you have access to this component") + } + + buf := new(strings.Builder) + if _, err := io.Copy(buf, resp.Body); err != nil { + return err + } + + return fmt.Errorf("Unknow error from server, response body: %s", buf.String()) +} + func (l *httpMirror) isRetryable(err error) bool { retryableList := []string{ "unexpected EOF", @@ -346,6 +499,20 @@ func (l *MockMirror) Download(resource, targetDir string) error { return err } +// Grant implements the model.Backend interface +func (l *MockMirror) Grant(id, name string, key *v1manifest.KeyInfo) error { + return nil +} + +// Publish implements the Mirror interface +func (l *MockMirror) Publish(manifest *v1manifest.Manifest, info model.ComponentInfo) error { + // Mock point for unit test + if fn := mock.On("Publish"); fn != nil { + fn.(func(*v1manifest.Manifest, model.ComponentInfo))(manifest, info) + } + return nil +} + // Fetch implements Mirror. func (l *MockMirror) Fetch(resource string, maxSize int64) (io.ReadCloser, error) { content, ok := l.Resources[resource] diff --git a/server/model/error.go b/pkg/repository/model/error.go similarity index 76% rename from server/model/error.go rename to pkg/repository/model/error.go index 69f1aaed9a..07fafc4131 100644 --- a/server/model/error.go +++ b/pkg/repository/model/error.go @@ -22,4 +22,8 @@ var ( ErrorConflict = errors.New("manifest conflict") // ErrorMissingKey indicates that the private key is missing ErrorMissingKey = errors.New("the private key is missing") + // ErrorMissingOwner indicates that the owner is not found + ErrorMissingOwner = errors.New("owner not found") + // ErrorWrongSignature indicates that the signature is not correct + ErrorWrongSignature = errors.New("the signature is not correct") ) diff --git a/pkg/repository/model/model.go b/pkg/repository/model/model.go new file mode 100644 index 0000000000..e594cfdf8a --- /dev/null +++ b/pkg/repository/model/model.go @@ -0,0 +1,403 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "fmt" + "time" + + cjson "github.com/gibson042/canonicaljson-go" + "github.com/juju/errors" + "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/repository/store" + "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/pingcap/tiup/pkg/utils" +) + +// Backend defines operations on the manifests +type Backend interface { + // Publish push a new component to mirror or modify an exists component + Publish(manifest *v1manifest.Manifest, info ComponentInfo) error + // Introduce add a new owner to mirror + Grant(id, name string, key *v1manifest.KeyInfo) error +} + +type model struct { + txn store.FsTxn + keys map[string]*v1manifest.KeyInfo +} + +// New returns a object implemented Backend +func New(txn store.FsTxn, keys map[string]*v1manifest.KeyInfo) Backend { + return &model{txn, keys} +} + +// Grant implements Backend +func (m *model) Grant(id, name string, key *v1manifest.KeyInfo) error { + initTime := time.Now() + + keyID, err := key.ID() + if err != nil { + return err + } + + return utils.RetryUntil(func() error { + var indexFileVersion *v1manifest.FileVersion + if err := m.updateIndexManifest(initTime, func(im *v1manifest.Manifest) (*v1manifest.Manifest, error) { + signed := im.Signed.(*v1manifest.Index) + + for oid, owner := range signed.Owners { + if oid == id { + return nil, errors.Errorf("owner %s exists", id) + } + + for kid := range owner.Keys { + if kid == keyID { + return nil, errors.Errorf("key %s exists", keyID) + } + } + } + + signed.Owners[id] = v1manifest.Owner{ + Name: name, + Keys: map[string]*v1manifest.KeyInfo{ + keyID: key, + }, + // TODO: support configable threshold + Threshold: 1, + } + + indexFileVersion = &v1manifest.FileVersion{Version: signed.Version + 1} + return im, nil + }); err != nil { + return err + } + + if indexFi, err := m.txn.Stat(fmt.Sprintf("%d.index.json", indexFileVersion.Version)); err == nil { + indexFileVersion.Length = uint(indexFi.Size()) + } else { + return err + } + + if err := m.updateSnapshotManifest(initTime, func(om *v1manifest.Manifest) *v1manifest.Manifest { + signed := om.Signed.(*v1manifest.Snapshot) + if indexFileVersion != nil { + signed.Meta["/index.json"] = *indexFileVersion + } + return om + }); err != nil { + return err + } + + // Update timestamp.json and signature + if err := m.updateTimestampManifest(initTime); err != nil { + return err + } + + return m.txn.Commit() + }, func(err error) bool { + return err == store.ErrorFsCommitConflict && m.txn.ResetManifest() == nil + }) +} + +// Publish implements Backend +func (m *model) Publish(manifest *v1manifest.Manifest, info ComponentInfo) error { + signed := manifest.Signed.(*v1manifest.Component) + initTime := time.Now() + return utils.RetryUntil(func() error { + // Write the component manifest (component.json) + if err := m.updateComponentManifest(manifest); err != nil { + return err + } + + // Update snapshot.json and signature + fi, err := m.txn.Stat(fmt.Sprintf("%d.%s.json", signed.Version, signed.ID)) + if err != nil { + return err + } + + var indexFileVersion *v1manifest.FileVersion + var owner *v1manifest.Owner + if err := m.updateIndexManifest(initTime, func(im *v1manifest.Manifest) (*v1manifest.Manifest, error) { + // We only update index.json when it's a new component + // or the yanked, standalone, hidden fileds changed + var ( + compItem v1manifest.ComponentItem + compExist bool + ) + + componentName := signed.ID + signed := im.Signed.(*v1manifest.Index) + if compItem, compExist = signed.Components[componentName]; compExist { + // Find the owner of target component + o := signed.Owners[compItem.Owner] + owner = &o + if info.Yanked() == nil && info.Hidden() == nil && info.Standalone() == nil { + // No changes on index.json + return nil, nil + } + } else { + var ownerID string + // The component is a new component, so the owner is whoever first create it. + for _, sk := range manifest.Signatures { + if ownerID, owner = findKeyOwnerFromIndex(signed, sk.KeyID); owner != nil { + break + } + } + compItem = v1manifest.ComponentItem{ + Owner: ownerID, + URL: fmt.Sprintf("/%s.json", componentName), + } + } + if info.Yanked() != nil { + compItem.Yanked = *info.Yanked() + } + if info.Hidden() != nil { + compItem.Hidden = *info.Hidden() + } + if info.Standalone() != nil { + compItem.Standalone = *info.Standalone() + } + + signed.Components[componentName] = compItem + indexFileVersion = &v1manifest.FileVersion{Version: signed.Version + 1} + return im, nil + }); err != nil { + return err + } + + if err := verifyComponentManifest(owner, manifest); err != nil { + return err + } + + if indexFileVersion != nil { + if indexFi, err := m.txn.Stat(fmt.Sprintf("%d.index.json", indexFileVersion.Version)); err == nil { + indexFileVersion.Length = uint(indexFi.Size()) + } else { + return err + } + } + + if err := m.updateSnapshotManifest(initTime, func(om *v1manifest.Manifest) *v1manifest.Manifest { + componentName := signed.ID + manifestVersion := signed.Version + signed := om.Signed.(*v1manifest.Snapshot) + if indexFileVersion != nil { + signed.Meta["/index.json"] = *indexFileVersion + } + signed.Meta[fmt.Sprintf("/%s.json", componentName)] = v1manifest.FileVersion{ + Version: manifestVersion, + Length: uint(fi.Size()), + } + return om + }); err != nil { + return err + } + + // Update timestamp.json and signature + if err := m.updateTimestampManifest(initTime); err != nil { + return err + } + + if info.Filename() != "" { + if err := m.txn.Write(info.Filename(), info); err != nil { + return err + } + } + return m.txn.Commit() + }, func(err error) bool { + return err == store.ErrorFsCommitConflict && m.txn.ResetManifest() == nil + }) +} + +func findKeyOwnerFromIndex(signed *v1manifest.Index, keyID string) (string, *v1manifest.Owner) { + for on := range signed.Owners { + for k := range signed.Owners[on].Keys { + if k == keyID { + o := signed.Owners[on] + return on, &o + } + } + } + return "", nil +} + +func (m *model) updateComponentManifest(manifest *v1manifest.Manifest) error { + signed := manifest.Signed.(*v1manifest.Component) + snap, err := m.readSnapshotManifest() + if err != nil { + return err + } + snapSigned := snap.Signed.(*v1manifest.Snapshot) + lastVersion := snapSigned.Meta["/"+signed.Filename()].Version + if signed.Version != lastVersion+1 { + log.Debugf("Component version not expected, expect %d, got %d", lastVersion+1, signed.Version) + return ErrorConflict + } + return m.txn.WriteManifest(fmt.Sprintf("%d.%s.json", signed.Version, signed.ID), manifest) +} + +func (m *model) updateIndexManifest(initTime time.Time, f func(*v1manifest.Manifest) (*v1manifest.Manifest, error)) error { + snap, err := m.readSnapshotManifest() + if err != nil { + return err + } + snapSigned := snap.Signed.(*v1manifest.Snapshot) + lastVersion := snapSigned.Meta[v1manifest.ManifestURLIndex].Version + + last, err := m.txn.ReadManifest(fmt.Sprintf("%d.index.json", lastVersion), &v1manifest.Index{}) + if err != nil { + return err + } + manifest, err := f(last) + if err != nil { + return err + } + if manifest == nil { + return nil + } + signed := manifest.Signed.(*v1manifest.Index) + v1manifest.RenewManifest(signed, initTime) + manifest.Signatures, err = m.sign(manifest.Signed) + if err != nil { + return err + } + + return m.txn.WriteManifest(fmt.Sprintf("%d.index.json", signed.Version), manifest) +} + +func (m *model) updateSnapshotManifest(initTime time.Time, f func(*v1manifest.Manifest) *v1manifest.Manifest) error { + last, err := m.txn.ReadManifest(v1manifest.ManifestFilenameSnapshot, &v1manifest.Snapshot{}) + if err != nil { + return err + } + manifest := f(last) + if manifest == nil { + return nil + } + v1manifest.RenewManifest(manifest.Signed, initTime) + manifest.Signatures, err = m.sign(manifest.Signed) + if err != nil { + return err + } + + return m.txn.WriteManifest(v1manifest.ManifestFilenameSnapshot, manifest) +} + +// readSnapshotManifest returns snapshot.json +func (m *model) readSnapshotManifest() (*v1manifest.Manifest, error) { + return m.txn.ReadManifest(v1manifest.ManifestFilenameSnapshot, &v1manifest.Snapshot{}) +} + +// readRootManifest returns root.json +func (m *model) readRootManifest() (*v1manifest.Manifest, error) { + return m.txn.ReadManifest(v1manifest.ManifestFilenameRoot, &v1manifest.Root{}) +} + +func (m *model) updateTimestampManifest(initTime time.Time) error { + fi, err := m.txn.Stat(v1manifest.ManifestFilenameSnapshot) + if err != nil { + return err + } + reader, err := m.txn.Read(v1manifest.ManifestFilenameSnapshot) + if err != nil { + return err + } + sha256, err := utils.SHA256(reader) + if err != nil { + reader.Close() + return err + } + reader.Close() + + manifest, err := m.txn.ReadManifest(v1manifest.ManifestFilenameTimestamp, &v1manifest.Timestamp{}) + if err != nil { + return err + } + signed := manifest.Signed.(*v1manifest.Timestamp) + signed.Meta[v1manifest.ManifestURLSnapshot] = v1manifest.FileHash{ + Hashes: map[string]string{ + v1manifest.SHA256: sha256, + }, + Length: uint(fi.Size()), + } + v1manifest.RenewManifest(manifest.Signed, initTime) + manifest.Signatures, err = m.sign(manifest.Signed) + if err != nil { + return err + } + + return m.txn.WriteManifest(v1manifest.ManifestFilenameTimestamp, manifest) +} + +func (m *model) sign(signed v1manifest.ValidManifest) ([]v1manifest.Signature, error) { + payload, err := cjson.Marshal(signed) + if err != nil { + return nil, err + } + + rm, err := m.readRootManifest() + if err != nil { + return nil, err + } + root := rm.Signed.(*v1manifest.Root) + + signs := []v1manifest.Signature{} + for _, pubKey := range root.Roles[signed.Base().Ty].Keys { + id, err := pubKey.ID() + if err != nil { + return nil, err + } + + privKey := m.keys[id] + if privKey == nil { + return nil, ErrorMissingKey + } + + sign, err := privKey.Signature(payload) + if err != nil { + return nil, errors.Trace(err) + } + signs = append(signs, v1manifest.Signature{ + KeyID: id, + Sig: sign, + }) + } + + return signs, nil +} + +func verifyComponentManifest(owner *v1manifest.Owner, m *v1manifest.Manifest) error { + if owner == nil { + return ErrorMissingOwner + } + + payload, err := cjson.Marshal(m.Signed) + if err != nil { + return err + } + + for _, s := range m.Signatures { + k := owner.Keys[s.KeyID] + if k == nil { + continue + } + + if err := k.Verify(payload, s.Sig); err == nil { + return nil + } + } + + return ErrorWrongSignature +} diff --git a/pkg/repository/model/publish.go b/pkg/repository/model/publish.go new file mode 100644 index 0000000000..fa174299ae --- /dev/null +++ b/pkg/repository/model/publish.go @@ -0,0 +1,75 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import "io" + +// ComponentData is used to represent the tarbal +type ComponentData interface { + io.Reader + + // Filename is the name of tarbal + Filename() string +} + +// ComponentInfo is used to update component +type ComponentInfo interface { + ComponentData + + Standalone() *bool + Yanked() *bool + Hidden() *bool +} + +// PublishInfo implements ComponentInfo +type PublishInfo struct { + ComponentData + Stand *bool + Yank *bool + Hide *bool +} + +// TarInfo implements ComponentData +type TarInfo struct { + io.Reader + Name string +} + +// Filename implements ComponentData +func (ti *TarInfo) Filename() string { + return ti.Name +} + +// Filename implements ComponentData +func (i *PublishInfo) Filename() string { + if i.ComponentData == nil { + return "" + } + return i.ComponentData.Filename() +} + +// Standalone implements ComponentInfo +func (i *PublishInfo) Standalone() *bool { + return i.Stand +} + +// Yanked implements ComponentInfo +func (i *PublishInfo) Yanked() *bool { + return i.Yank +} + +// Hidden implements ComponentInfo +func (i *PublishInfo) Hidden() *bool { + return i.Hide +} diff --git a/pkg/repository/remote/common.go b/pkg/repository/remote/common.go deleted file mode 100644 index 2b9820c654..0000000000 --- a/pkg/repository/remote/common.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package remote - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - cjson "github.com/gibson042/canonicaljson-go" - "github.com/pingcap/tiup/pkg/repository/v1manifest" -) - -func signAndSend(url string, m *v1manifest.Component, key *v1manifest.KeyInfo, options map[string]bool) error { - id, err := key.ID() - if err != nil { - return err - } - payload, err := cjson.Marshal(m) - if err != nil { - return err - } - - sig, err := key.Signature(payload) - if err != nil { - return err - } - manifest := v1manifest.Manifest{ - Signatures: []v1manifest.Signature{{ - KeyID: id, - Sig: sig, - }}, - Signed: m, - } - - payload, err = json.Marshal(manifest) - if err != nil { - return err - } - bodyBuf := bytes.NewBuffer(payload) - - qpairs := []string{} - for k, v := range options { - qpairs = append(qpairs, fmt.Sprintf("%s=%t", k, v)) - } - qstr := "" - if len(qpairs) > 0 { - qstr = "?" + strings.Join(qpairs, "&") - } - addr := fmt.Sprintf("%s%s", url, qstr) - - resp, err := http.Post(addr, "text/json", bodyBuf) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode < 300 { - return nil - } else if resp.StatusCode == http.StatusConflict { - return fmt.Errorf("Local component manifest is not new enough, update it first") - } else if resp.StatusCode == http.StatusForbidden { - return fmt.Errorf("The server refused, make sure you have access to this component") - } - - buf := new(strings.Builder) - if _, err := io.Copy(buf, resp.Body); err != nil { - return err - } - - return fmt.Errorf("Unknow error from server, response body: %s", buf.String()) -} diff --git a/pkg/repository/remote/modify.go b/pkg/repository/remote/modify.go deleted file mode 100644 index ff3fbdad5c..0000000000 --- a/pkg/repository/remote/modify.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package remote - -import ( - "fmt" - "time" - - "github.com/google/uuid" - "github.com/juju/errors" - "github.com/pingcap/tiup/pkg/repository/v0manifest" - "github.com/pingcap/tiup/pkg/repository/v1manifest" -) - -// Editor defines the methods to modify a component attrs -type Editor interface { - WithVersion(version string) Editor - WithDesc(desc string) Editor - Standalone(bool) Editor - Hide(bool) Editor - Yank(bool) Editor - Sign(key *v1manifest.KeyInfo, m *v1manifest.Component) error -} - -type editor struct { - endpoint string - component string - version string - description string - options map[string]bool -} - -// NewEditor returns a Editor interface -func NewEditor(endpoint, component string) Editor { - return &editor{ - endpoint: endpoint, - component: component, - options: make(map[string]bool), - } -} - -// WithVersion set version field -func (e *editor) WithVersion(version string) Editor { - e.version = version - return e -} - -// WithDesc set description field -func (e *editor) WithDesc(desc string) Editor { - e.description = desc - return e -} - -// Hide set hidden flag -func (e *editor) Hide(hidden bool) Editor { - e.options["hidden"] = hidden - return e -} - -// Standalone set standalone flag -func (e *editor) Standalone(standalone bool) Editor { - e.options["standalone"] = standalone - return e -} - -// Yank set yanked flag -func (e *editor) Yank(yanked bool) Editor { - e.options["yanked"] = yanked - return e -} - -func (e *editor) Sign(key *v1manifest.KeyInfo, m *v1manifest.Component) error { - initTime := time.Now() - v1manifest.RenewManifest(m, initTime) - m.Version++ - if e.description != "" { - m.Description = e.description - } - - sid := uuid.New().String() - url := fmt.Sprintf("%s/api/v1/component/%s/%s", e.endpoint, sid, e.component) - - if e.version != "" { - // Only support modify yanked field for specified versiion - for p := range m.Platforms { - if v0manifest.Version(e.version).IsNightly() { - return errors.New("nightly version can't be yanked") - } - vi, ok := m.Platforms[p][e.version] - if !ok { - continue - } - vi.Yanked = e.options["yanked"] - m.Platforms[p][e.version] = vi - } - return signAndSend(url, m, key, nil) - } - return signAndSend(url, m, key, e.options) -} diff --git a/pkg/repository/remote/upload.go b/pkg/repository/remote/upload.go deleted file mode 100644 index a98653e915..0000000000 --- a/pkg/repository/remote/upload.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package remote - -import ( - "fmt" - "os" - "runtime" - "strings" - "time" - - "github.com/juju/errors" - ru "github.com/pingcap/tiup/pkg/repository/utils" - "github.com/pingcap/tiup/pkg/repository/v1manifest" - "github.com/pingcap/tiup/pkg/utils" - "github.com/pingcap/tiup/pkg/version" -) - -// Transporter defines methods to upload components -type Transporter interface { - WithOS(os string) Transporter - WithArch(arch string) Transporter - WithDesc(desc string) Transporter - Standalone() Transporter - Hide() Transporter - Open(tarball string) error - Close() error - Upload() error - Sign(key *v1manifest.KeyInfo, m *v1manifest.Component) error -} - -type transporter struct { - tarFile *os.File - os string - arch string - entry string - component string - version string - description string - endpoint string - options map[string]bool - filehash v1manifest.FileHash -} - -// NewTransporter returns a Transporter -func NewTransporter(endpoint, component, version, entry string) Transporter { - return &transporter{ - endpoint: strings.TrimSuffix(endpoint, "/"), - component: component, - entry: entry, - os: runtime.GOOS, - arch: runtime.GOARCH, - version: version, - options: make(map[string]bool), - } -} - -// WithOS set os field of transporter -func (t *transporter) WithOS(os string) Transporter { - t.os = os - return t -} - -// WithDesc set description field of transporter -func (t *transporter) WithDesc(desc string) Transporter { - t.description = desc - return t -} - -// WithArch set arch field of transporter -func (t *transporter) WithArch(arch string) Transporter { - t.arch = arch - return t -} - -// Standalone set standalone field to true -func (t *transporter) Standalone() Transporter { - t.options["standalone"] = true - return t -} - -// Hide set hidden field to true -func (t *transporter) Hide() Transporter { - t.options["hidden"] = true - return t -} - -// Open read the tarball -func (t *transporter) Open(tarball string) error { - hashes, length, err := ru.HashFile(tarball) - if err != nil { - return errors.Trace(err) - } - - t.filehash = v1manifest.FileHash{ - Hashes: hashes, - Length: uint(length), - } - - file, err := os.Open(tarball) - if err != nil { - return err - } - t.tarFile = file - - return nil -} - -func (t *transporter) Close() error { - return t.tarFile.Close() -} - -func (t *transporter) Upload() error { - sha256 := t.filehash.Hashes[v1manifest.SHA256] - if sha256 == "" { - return errors.New("sha256 not found for tarball") - } - postAddr := fmt.Sprintf("%s/api/v1/tarball/%s", t.endpoint, sha256) - tarballName := fmt.Sprintf("%s-%s-%s-%s.tar.gz", t.component, t.version, t.os, t.arch) - resp, err := utils.PostFile(t.tarFile, postAddr, "file", tarballName) - if err != nil { - return err - } - return resp.Body.Close() -} - -func (t *transporter) Sign(key *v1manifest.KeyInfo, m *v1manifest.Component) error { - sha256 := t.filehash.Hashes[v1manifest.SHA256] - if sha256 == "" { - return errors.New("sha256 not found for tarball") - } - - initTime := time.Now() - if m == nil { - m = t.defaultComponent(initTime) - } else { - v1manifest.RenewManifest(m, initTime) - m.Version++ - if t.description != "" { - m.Description = t.description - } - } - - if strings.Contains(t.version, version.NightlyVersion) { - m.Nightly = t.version - } - // Remove history nightly - for plat := range m.Platforms { - for ver := range m.Platforms[plat] { - if strings.Contains(ver, version.NightlyVersion) && ver != m.Nightly { - delete(m.Platforms[plat], ver) - } - } - } - - platformStr := fmt.Sprintf("%s/%s", t.os, t.arch) - if m.Platforms[platformStr] == nil { - m.Platforms[platformStr] = map[string]v1manifest.VersionItem{} - } - m.Platforms[platformStr][t.version] = v1manifest.VersionItem{ - Entry: t.entry, - Released: initTime.Format(time.RFC3339), - URL: fmt.Sprintf("/%s-%s-%s-%s.tar.gz", t.component, t.version, t.os, t.arch), - FileHash: t.filehash, - } - - url := fmt.Sprintf("%s/api/v1/component/%s/%s", t.endpoint, sha256, t.component) - return signAndSend(url, m, key, t.options) -} - -func (t *transporter) defaultComponent(initTime time.Time) *v1manifest.Component { - return v1manifest.NewComponent(t.component, t.description, initTime) -} diff --git a/pkg/repository/store/local.go b/pkg/repository/store/local.go new file mode 100644 index 0000000000..acbe6179cb --- /dev/null +++ b/pkg/repository/store/local.go @@ -0,0 +1,72 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "os" + "path" + "time" + + "github.com/gofrs/flock" + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/utils" +) + +type localStore struct { + root string + upstream string + flock *flock.Flock +} + +func newLocalStore(root, upstream string) *localStore { + return &localStore{ + root: root, + upstream: upstream, + flock: flock.New(path.Join(root, "lock")), + } +} + +// Begin implements the Store +func (s *localStore) Begin() (FsTxn, error) { + return newLocalTxn(s) +} + +// Returns the last modify time +func (s *localStore) last(filename string) (*time.Time, error) { + fp := path.Join(s.root, filename) + if utils.IsNotExist(fp) { + return nil, nil + } + fi, err := os.Stat(fp) + if err != nil { + return nil, errors.Annotate(err, "Stat file") + } + mt := fi.ModTime() + return &mt, nil +} + +func (s *localStore) path(filename string) string { + return path.Join(s.root, filename) +} + +func (s *localStore) lock() error { + return s.flock.Lock() +} + +func (s *localStore) unlock() { + // The unlock operation must success, otherwise the later operation will stuck + if err := s.flock.Unlock(); err != nil { + panic(errors.Annotate(err, "unlock filesystem failed")) + } +} diff --git a/server/store/store.go b/pkg/repository/store/store.go similarity index 69% rename from server/store/store.go rename to pkg/repository/store/store.go index fd3cbe7ebc..057a1c6376 100644 --- a/server/store/store.go +++ b/pkg/repository/store/store.go @@ -16,6 +16,8 @@ package store import ( "io" "os" + + "github.com/pingcap/tiup/pkg/repository/v1manifest" ) // Store represents the storage level @@ -27,16 +29,16 @@ type Store interface { type FsTxn interface { Write(filename string, reader io.Reader) error Read(filename string) (io.ReadCloser, error) - WriteManifest(filename string, manifest interface{}) error - ReadManifest(filename string, manifest interface{}) error + WriteManifest(filename string, manifest *v1manifest.Manifest) error + ReadManifest(filename string, role v1manifest.ValidManifest) (*v1manifest.Manifest, error) Stat(filename string) (os.FileInfo, error) - // Restart should reset the manifest state + // ResetManifest should reset the manifest state ResetManifest() error Commit() error Rollback() error } -// NewStore returns a Store, curretly only qcloud supported -func NewStore(root string, upstream string) Store { - return newQCloudStore(root, upstream) +// New returns a Store, curretly only qcloud supported +func New(root string, upstream string) Store { + return newLocalStore(root, upstream) } diff --git a/pkg/repository/store/store_test.go b/pkg/repository/store/store_test.go new file mode 100644 index 0000000000..9483689ec8 --- /dev/null +++ b/pkg/repository/store/store_test.go @@ -0,0 +1,115 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/stretchr/testify/assert" +) + +func TestEmptyCommit(t *testing.T) { + root, err := ioutil.TempDir("", "") + assert.Nil(t, err) + defer os.RemoveAll(root) + + store := New(root, "") + txn, err := store.Begin() + assert.Nil(t, err) + err = txn.Commit() + assert.Nil(t, err) +} + +func TestSingleWrite(t *testing.T) { + root, err := ioutil.TempDir("", "") + assert.Nil(t, err) + defer os.RemoveAll(root) + + store := New(root, "") + txn, err := store.Begin() + assert.Nil(t, err) + err = txn.WriteManifest("test.json", &v1manifest.Manifest{ + Signed: &v1manifest.Timestamp{ + Meta: map[string]v1manifest.FileHash{ + "test": { + Length: 9527, + }, + }, + }, + }) + assert.Nil(t, err) + m, err := txn.ReadManifest("test.json", &v1manifest.Timestamp{}) + assert.Nil(t, err) + assert.Equal(t, uint(9527), m.Signed.(*v1manifest.Timestamp).Meta["test"].Length) +} + +func TestConflict(t *testing.T) { + root, err := ioutil.TempDir("", "") + assert.Nil(t, err) + defer os.RemoveAll(root) + + store := New(root, "") + + txn1, err := store.Begin() + assert.Nil(t, err) + txn2, err := store.Begin() + assert.Nil(t, err) + + test := &v1manifest.Manifest{ + Signed: &v1manifest.Timestamp{ + Meta: map[string]v1manifest.FileHash{ + "test": { + Length: 9527, + }, + }, + }, + } + err = txn1.WriteManifest("test.json", test) + assert.Nil(t, err) + m, err := txn1.ReadManifest("test.json", &v1manifest.Timestamp{}) + assert.Nil(t, err) + assert.Equal(t, uint(9527), m.Signed.(*v1manifest.Timestamp).Meta["test"].Length) + + err = txn2.WriteManifest("test.json", test) + assert.Nil(t, err) + m, err = txn2.ReadManifest("test.json", &v1manifest.Timestamp{}) + assert.Nil(t, err) + assert.Equal(t, uint(9527), m.Signed.(*v1manifest.Timestamp).Meta["test"].Length) + + err = txn1.Commit() + assert.Nil(t, err) + + err = txn2.Commit() + assert.NotNil(t, err) +} + +func TestUpstream(t *testing.T) { + root, err := ioutil.TempDir("", "") + assert.Nil(t, err) + defer os.RemoveAll(root) + + txn, err := New(root, "").Begin() + assert.Nil(t, err) + _, err = txn.ReadManifest("timestamp.json", &v1manifest.Timestamp{}) + assert.NotNil(t, err) + + txn, err = New(root, "https://tiup-mirrors.pingcap.com").Begin() + assert.Nil(t, err) + m, err := txn.ReadManifest("timestamp.json", &v1manifest.Timestamp{}) + assert.Nil(t, err) + assert.NotEmpty(t, m.Signed.(*v1manifest.Timestamp).Meta["/snapshot.json"].Hashes) +} diff --git a/server/store/sync.go b/pkg/repository/store/sync.go similarity index 90% rename from server/store/sync.go rename to pkg/repository/store/sync.go index 34fdef80f8..ef85d5ec3d 100644 --- a/server/store/sync.go +++ b/pkg/repository/store/sync.go @@ -76,15 +76,15 @@ func (s *fsSyncer) Sync(srcDir string) error { return nil } -type qcloudSyncer struct { +type externalSyncer struct { script string } -func newQcloudSyncer(scriptPath string) Syncer { - return &qcloudSyncer{scriptPath} +func newExternalSyncer(scriptPath string) Syncer { + return &externalSyncer{scriptPath} } -func (s *qcloudSyncer) Sync(srcDir string) error { +func (s *externalSyncer) Sync(srcDir string) error { cmd := exec.Command(s.script, srcDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/pkg/repository/store/txn.go b/pkg/repository/store/txn.go new file mode 100644 index 0000000000..ea4a260896 --- /dev/null +++ b/pkg/repository/store/txn.go @@ -0,0 +1,305 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "time" + + cjson "github.com/gibson042/canonicaljson-go" + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/localdata" + "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/pingcap/tiup/pkg/utils" +) + +var ( + // ErrorFsCommitConflict indicates concurrent writing file + ErrorFsCommitConflict = errors.New("conflict on fs commit") +) + +// The localTxn is used to implment a filesystem transaction, the basic principle is to +// record the timestamp before access any manifest file, and when writing them back, check +// if the origin files' timestamp is newer than recorded one, if so, a conflict is detected. +// +// To get current timestamp: +// 1. check if there is timestamp.json in root directory, if so, return it's modify time +// 2. check if there is any file in root directory, if so, return the newest one's modify time +// 3. return the root directory's modify time +// If the timestamp.json is in root directory, we always make sure it's newer than other files +// So current timestamp is the modify time of the newest file in root directory +// +// To read a manifest file: +// 1. get current timestamp and record it +// 2. read the file from root directory +// +// To write a manifest file: +// 1. get current timestamp and record it +// 2. write the manifest file to temporary directory +// +// To commit a localTxn: +// 1. for every accessed file, get the recorded timestamp +// 2. for every accessed file, get the current modify time of their origin file in root directory +// 3. check if the origin file is newer thant recorded timestamp, if so, there must be conflict +// 4. copy every file in temporary directory to root directory, if there is a timestamp.json in +// temporary directory, it should be the last one to copy +type localTxn struct { + syncer Syncer + store *localStore + root string + accessed map[string]*time.Time +} + +func newLocalTxn(store *localStore) (*localTxn, error) { + syncer := newFsSyncer(path.Join(store.root, "commits")) + if script := os.Getenv(localdata.EnvNameMirrorSyncScript); script != "" { + syncer = combine(syncer, newExternalSyncer(script)) + } + root, err := ioutil.TempDir(os.Getenv(localdata.EnvNameComponentDataDir), "tiup-commit-*") + if err != nil { + return nil, err + } + txn := &localTxn{ + syncer: syncer, + store: store, + root: root, + accessed: make(map[string]*time.Time), + } + + return txn, nil +} + +// Write implements FsTxn +func (t *localTxn) Write(filename string, reader io.Reader) error { + filepath := path.Join(t.root, filename) + file, err := os.Create(filepath) + if err != nil { + return errors.Annotate(err, "create file") + } + defer file.Close() + + _, err = io.Copy(file, reader) + return err +} + +// Read implements FsTxn +func (t *localTxn) Read(filename string) (io.ReadCloser, error) { + filepath := t.store.path(filename) + if utils.IsExist(path.Join(t.root, filename)) { + filepath = path.Join(t.root, filename) + } + + return os.Open(filepath) +} + +func (t *localTxn) WriteManifest(filename string, manifest *v1manifest.Manifest) error { + if err := t.access(filename); err != nil { + return err + } + filepath := path.Join(t.root, filename) + file, err := os.Create(filepath) + if err != nil { + return errors.Annotate(err, "create file") + } + defer file.Close() + + bytes, err := cjson.Marshal(manifest) + if err != nil { + return errors.Annotate(err, "marshal manifest") + } + + if _, err = file.Write(bytes); err != nil { + return errors.Annotate(err, "write file") + } + + if err = file.Close(); err != nil { + return errors.Annotate(err, "flush file content") + } + + fi, err := os.Stat(filepath) + if err != nil { + return errors.Annotate(err, "stat file") + } + // The modify time must increase + if !t.first(filename).Before(fi.ModTime()) { + mt := time.Unix(0, t.first(filename).UnixNano()+1) + return os.Chtimes(filepath, mt, mt) + } + + return nil +} + +func (t *localTxn) ReadManifest(filename string, role v1manifest.ValidManifest) (*v1manifest.Manifest, error) { + if err := t.access(filename); err != nil { + return nil, err + } + filepath := t.store.path(filename) + if utils.IsExist(path.Join(t.root, filename)) { + filepath = path.Join(t.root, filename) + } + var wc io.ReadCloser + if file, err := os.Open(filepath); err == nil { + wc = file + } else if os.IsNotExist(err) && t.store.upstream != "" { + url := fmt.Sprintf("%s/%s", t.store.upstream, filename) + client := http.Client{Timeout: time.Minute} + if resp, err := client.Get(url); err == nil { + wc = resp.Body + } else { + return nil, errors.Annotatef(err, "fetch %s", url) + } + } else { + log.Errorf("Error on read manifest: %s, upstream: %s", err.Error(), t.store.upstream) + return nil, errors.Annotate(err, "open file") + } + defer wc.Close() + + return v1manifest.ReadNoVerify(wc, role) +} + +func (t *localTxn) ResetManifest() error { + for file := range t.accessed { + fp := path.Join(t.root, file) + if utils.IsExist(fp) { + if err := os.Remove(fp); err != nil { + return err + } + } + } + t.accessed = make(map[string]*time.Time) + return nil +} + +func (t *localTxn) Stat(filename string) (os.FileInfo, error) { + if err := t.access(filename); err != nil { + return nil, err + } + filepath := t.store.path(filename) + if utils.IsExist(path.Join(t.root, filename)) { + filepath = path.Join(t.root, filename) + } + return os.Stat(filepath) +} + +func (t *localTxn) Commit() error { + if err := t.store.lock(); err != nil { + return err + } + defer t.store.unlock() + + if err := t.checkConflict(); err != nil { + return err + } + + files, err := ioutil.ReadDir(t.root) + if err != nil { + return err + } + + hasTimestamp := false + for _, f := range files { + // Make sure modify time of the timestamp.json is the newest + if f.Name() == v1manifest.ManifestFilenameTimestamp { + hasTimestamp = true + continue + } + if err := utils.Copy(path.Join(t.root, f.Name()), t.store.path(f.Name())); err != nil { + return err + } + } + if hasTimestamp { + if err := utils.Copy(path.Join(t.root, v1manifest.ManifestFilenameTimestamp), t.store.path(v1manifest.ManifestFilenameTimestamp)); err != nil { + return err + } + } + + if err := t.syncer.Sync(t.root); err != nil { + return err + } + + return t.release() +} + +func (t *localTxn) Rollback() error { + return t.release() +} + +func (t *localTxn) checkConflict() error { + for file := range t.accessed { + mt, err := t.store.last(file) + if err != nil { + return err + } + if mt != nil && mt.After(*t.first(file)) { + return ErrorFsCommitConflict + } + } + return nil +} + +func (t *localTxn) access(filename string) error { + // Use the earliest time + if t.accessed[filename] != nil { + return nil + } + + // Use the modify time of timestamp.json + timestamp := t.store.path(v1manifest.ManifestFilenameTimestamp) + fi, err := os.Stat(timestamp) + if err == nil { + mt := fi.ModTime() + t.accessed[filename] = &mt + } else if !os.IsNotExist(err) { + return errors.Annotatef(err, "read %s: %s", v1manifest.ManifestFilenameTimestamp, timestamp) + } + + // Use the newest file in t.store.root + files, err := ioutil.ReadDir(t.store.root) + if err != nil { + return errors.Annotatef(err, "read store root: %s", t.store.root) + } + for _, fi := range files { + if t.accessed[filename] == nil || t.accessed[filename].Before(fi.ModTime()) { + mt := fi.ModTime() + t.accessed[filename] = &mt + } + } + if t.accessed[filename] != nil { + return nil + } + + // Use the mod time of t.store.root + fi, err = os.Stat(t.store.root) + if err != nil { + return errors.Annotatef(err, "read store root: %s", t.store.root) + } + mt := fi.ModTime() + t.accessed[filename] = &mt + return nil +} + +// Returns the first access time +func (t *localTxn) first(filename string) *time.Time { + return t.accessed[filename] +} + +func (t *localTxn) release() error { + return os.RemoveAll(t.root) +} diff --git a/pkg/repository/v1manifest/key_store.go b/pkg/repository/v1manifest/key_store.go index 5bfd69d551..29a9054d8e 100644 --- a/pkg/repository/v1manifest/key_store.go +++ b/pkg/repository/v1manifest/key_store.go @@ -85,6 +85,10 @@ func (s *SignatureError) Error() string { // transitionRoot checks that signed is verified by signatures using newThreshold, and if so, updates the keys for the root // role in the key store. func (s *KeyStore) transitionRoot(signed []byte, newThreshold uint, expiry string, signatures []Signature, newKeys map[string]*KeyInfo) error { + if s == nil { + return nil + } + oldKeys, hasOldKeys := s.Load(ManifestTypeRoot) err := s.AddKeys(ManifestTypeRoot, newThreshold, expiry, newKeys) diff --git a/pkg/repository/v1manifest/keys.go b/pkg/repository/v1manifest/keys.go index f793d86d7a..8bdac08c29 100644 --- a/pkg/repository/v1manifest/keys.go +++ b/pkg/repository/v1manifest/keys.go @@ -96,6 +96,15 @@ func (ki *KeyInfo) Signature(payload []byte) (string, error) { return pk.Signature(payload) } +// SignManifest wrap Signature with the param manifest +func (ki *KeyInfo) SignManifest(m ValidManifest) (string, error) { + payload, err := cjson.Marshal(m) + if err != nil { + return "", errors.Annotate(err, "marshal for signature") + } + return ki.Signature(payload) +} + // Verify check the signature is right func (ki *KeyInfo) Verify(payload []byte, sig string) error { pk, err := ki.publicKey() diff --git a/pkg/repository/v1manifest/local_manifests.go b/pkg/repository/v1manifest/local_manifests.go index 6bd913e82e..674d82acf0 100644 --- a/pkg/repository/v1manifest/local_manifests.go +++ b/pkg/repository/v1manifest/local_manifests.go @@ -78,7 +78,7 @@ func NewManifests(profile *localdata.Profile) (*FsManifests, error) { // We must load without validation because we have no keys yet. var root Root - err = ReadNoVerify(strings.NewReader(manifest), &root) + _, err = ReadNoVerify(strings.NewReader(manifest), &root) if err != nil { return nil, errors.AddStack(err) } diff --git a/pkg/repository/v1manifest/manifest.go b/pkg/repository/v1manifest/manifest.go index 0f02d1cf53..97e89ff12b 100644 --- a/pkg/repository/v1manifest/manifest.go +++ b/pkg/repository/v1manifest/manifest.go @@ -360,11 +360,11 @@ func ReadComponentManifest(input io.Reader, com *Component, item *ComponentItem, // ReadNoVerify will read role from input and will not do any validation or verification. It is very dangerous to use // this function and it should only be used to read trusted data from local storage. -func ReadNoVerify(input io.Reader, role ValidManifest) error { +func ReadNoVerify(input io.Reader, role ValidManifest) (*Manifest, error) { decoder := json.NewDecoder(input) var m Manifest m.Signed = role - return decoder.Decode(&m) + return &m, decoder.Decode(&m) } // ReadManifest reads a manifest from input and validates it, the result is stored in role, which must be a pointer type. @@ -414,6 +414,10 @@ func ReadManifest(input io.Reader, role ValidManifest, keys *KeyStore) (*Manifes // RenewManifest resets and extends the expire time of manifest func RenewManifest(m ValidManifest, startTime time.Time) { + // manifest with 0 version means it's unversioned + if m.Base().Version > 0 { + m.Base().Version++ + } m.Base().Expires = startTime.Add( ManifestsConfig[m.Base().Ty].Expire, ).Format(time.RFC3339) diff --git a/pkg/repository/v1manifest/repo.go b/pkg/repository/v1manifest/repo.go index 380c02a17c..cc461f8d2f 100644 --- a/pkg/repository/v1manifest/repo.go +++ b/pkg/repository/v1manifest/repo.go @@ -25,13 +25,13 @@ import ( "os" "path" "path/filepath" - "strings" "time" cjson "github.com/gibson042/canonicaljson-go" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/crypto" "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/utils" ) // ErrorInsufficientKeys indicates that the key number is less than threshold @@ -56,33 +56,12 @@ func Init(dst, keyDir string, initTime time.Time) (err error) { // init index manifests[ManifestTypeIndex] = NewIndex(initTime) - signedManifests[ManifestTypeIndex], err = SignManifest(manifests[ManifestTypeIndex], keys[ManifestTypeIndex]...) - if err != nil { - return err - } - // snapshot and timestamp are the last two manifests to be initialized // init snapshot - manifests[ManifestTypeSnapshot], err = NewSnapshot(initTime).SetVersions(signedManifests) - if err != nil { - return err - } - signedManifests[ManifestTypeSnapshot], err = SignManifest(manifests[ManifestTypeSnapshot], keys[ManifestTypeSnapshot]...) - if err != nil { - return err - } + manifests[ManifestTypeSnapshot] = NewSnapshot(initTime) // init timestamp - timestamp, err := NewTimestamp(initTime).SetSnapshot(signedManifests[ManifestTypeSnapshot]) manifests[ManifestTypeTimestamp] = NewTimestamp(initTime) - if err != nil { - return err - } - manifests[ManifestTypeTimestamp] = timestamp - signedManifests[ManifestTypeTimestamp], err = SignManifest(manifests[ManifestTypeTimestamp], keys[ManifestTypeTimestamp]...) - if err != nil { - return err - } // root and snapshot has meta of each other inside themselves, but it's ok here // as we are still during the init process, not version bump needed @@ -100,8 +79,28 @@ func Init(dst, keyDir string, initTime time.Time) (err error) { // FIXME: log a warning about manifest not found instead of returning error return fmt.Errorf("manifest '%s' not initialized porperly", ty) } - signedManifests[ManifestTypeRoot], err = SignManifest(manifests[ManifestTypeRoot], keys[ManifestTypeRoot]...) - if err != nil { + + if signedManifests[ManifestTypeRoot], err = SignManifest(manifests[ManifestTypeRoot], keys[ManifestTypeRoot]...); err != nil { + return err + } + + if signedManifests[ManifestTypeIndex], err = SignManifest(manifests[ManifestTypeIndex], keys[ManifestTypeIndex]...); err != nil { + return err + } + + if _, err = manifests[ManifestTypeSnapshot].(*Snapshot).SetVersions(signedManifests); err != nil { + return err + } + + if signedManifests[ManifestTypeSnapshot], err = SignManifest(manifests[ManifestTypeSnapshot], keys[ManifestTypeSnapshot]...); err != nil { + return err + } + + if _, err = manifests[ManifestTypeTimestamp].(*Timestamp).SetSnapshot(signedManifests[ManifestTypeSnapshot]); err != nil { + return err + } + + if signedManifests[ManifestTypeTimestamp], err = SignManifest(manifests[ManifestTypeTimestamp], keys[ManifestTypeTimestamp]...); err != nil { return err } @@ -115,6 +114,18 @@ func SaveKeyInfo(key *KeyInfo, ty, dir string) error { return err } + if dir == "" { + dir, err = os.Getwd() + if err != nil { + return err + } + } + if utils.IsNotExist(dir) { + if err := os.MkdirAll(dir, 0755); err != nil { + return errors.Annotate(err, "create key directory") + } + } + f, err := os.Create(path.Join(dir, fmt.Sprintf("%s-%s.json", id[:ShortKeyIDLength], ty))) if err != nil { return err @@ -219,85 +230,6 @@ NextKey: return ioutil.WriteFile(mfile, content, 0664) } -// AddComponent adds a new component to an existing repository -func AddComponent(id, desc, owner, repo string, isDefault bool, pub, priv string) error { - // read key files - privBytes, err := ioutil.ReadFile(priv) - if err != nil { - return err - } - privKey := &KeyInfo{} - if err = json.Unmarshal(privBytes, &privKey); err != nil { - return err - } - - // read manifest index from disk - manifests, err := ReadManifestDir(repo, ManifestTypeIndex, ManifestTypeSnapshot) - if err != nil { - return err - } - signedManifests := make(map[string]*Manifest) - - // check id conflicts - if _, found := ManifestsConfig[strings.ToLower(id)]; found { - // reserved keywords - return fmt.Errorf("component id '%s' is not allowed, please use another one", id) - } - if _, found := manifests[ManifestTypeIndex].(*Index).Components[id]; found { - return fmt.Errorf("component id '%s' already exist, please use another one", id) - } - - // create new component manifest - currTime := time.Now().UTC() - comp := NewComponent(id, desc, currTime) - manifests[id] = comp - signedManifests[id], err = SignManifest(comp, privKey) - if err != nil { - return err - } - - // update repository - compInfo := ComponentItem{ - Owner: owner, - URL: fmt.Sprintf("/%s", comp.Filename()), - } - index := manifests[ManifestTypeIndex].(*Index) - index.Components[id] = compInfo - if isDefault { - index.DefaultComponents = append(index.DefaultComponents, id) - } - index.Version++ // bump index version - signedManifests[ManifestTypeIndex], err = SignManifest(index, privKey) - if err != nil { - return err - } - - // update snapshot - snapshot, err := manifests[ManifestTypeSnapshot].(*Snapshot).SetVersions(signedManifests) - if err != nil { - return err - } - snapshot.Expires = currTime.Add(ManifestsConfig[ManifestTypeSnapshot].Expire).Format(time.RFC3339) - snapshotSigned, err := SignManifest(snapshot, privKey) - if err != nil { - return err - } - - // update timestamp - timestamp, err := NewTimestamp(currTime).SetSnapshot(snapshotSigned) - if err != nil { - return err - } - timestamp.Version = manifests[ManifestTypeTimestamp].(*Timestamp).Version + 1 - manifests[ManifestTypeTimestamp] = timestamp - signedManifests[ManifestTypeTimestamp], err = SignManifest(timestamp, privKey) - if err != nil { - return err - } - - return BatchSaveManifests(repo, signedManifests) -} - // NewRoot creates a Root object func NewRoot(initTime time.Time) *Root { return &Root{ @@ -578,8 +510,12 @@ func SignAndWrite(out io.Writer, role ValidManifest, keys ...*KeyInfo) error { // Manifest in the manifestList map should already be signed, they are not checked // for signature again. func BatchSaveManifests(dst string, manifestList map[string]*Manifest) error { - for _, m := range manifestList { - writer, err := os.OpenFile(filepath.Join(dst, m.Signed.Filename()), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + for ty, m := range manifestList { + filename := m.Signed.Filename() + if ty == ManifestTypeIndex { + filename = fmt.Sprintf("%d.%s", m.Signed.Base().Version, m.Signed.Filename()) + } + writer, err := os.OpenFile(filepath.Join(dst, filename), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) if err != nil { return err } diff --git a/pkg/repository/v1manifest/types.go b/pkg/repository/v1manifest/types.go index 183101f861..c7aae1ddf2 100644 --- a/pkg/repository/v1manifest/types.go +++ b/pkg/repository/v1manifest/types.go @@ -256,6 +256,9 @@ func (manifest *Component) VersionList(platform string) map[string]VersionItem { // VersionListWithYanked return all versions include yanked versions func (manifest *Component) VersionListWithYanked(platform string) map[string]VersionItem { + if manifest == nil { + return nil + } vs, ok := manifest.Platforms[platform] if !ok { vs, ok = manifest.Platforms[AnyPlatform] diff --git a/pkg/utils/ioutil.go b/pkg/utils/ioutil.go index 8a5327d78f..d8a28d32a6 100644 --- a/pkg/utils/ioutil.go +++ b/pkg/utils/ioutil.go @@ -154,7 +154,21 @@ func Copy(src, dst string) error { return err } - return os.Chmod(dst, fi.Mode()) + err = os.Chmod(dst, fi.Mode()) + if err != nil { + return err + } + + // Make sure the created dst's modify time is newer (at least equal) than src + // this is used to workaround github action virtual filesystem + ofi, err := os.Stat(dst) + if err != nil { + return err + } + if fi.ModTime().After(ofi.ModTime()) { + return os.Chtimes(dst, fi.ModTime(), fi.ModTime()) + } + return nil } // Move moves a file from src to dst, this is done by copying the file and then diff --git a/server/handler/component.go b/server/handler/component.go index e3b6b0adcc..894b5c6001 100644 --- a/server/handler/component.go +++ b/server/handler/component.go @@ -14,48 +14,59 @@ package handler import ( - "fmt" + "encoding/json" "net/http" - "time" - cjson "github.com/gibson042/canonicaljson-go" "github.com/gorilla/mux" "github.com/pingcap/fn" "github.com/pingcap/tiup/pkg/logger/log" + "github.com/pingcap/tiup/pkg/repository" + "github.com/pingcap/tiup/pkg/repository/model" "github.com/pingcap/tiup/pkg/repository/v1manifest" - "github.com/pingcap/tiup/pkg/utils" - "github.com/pingcap/tiup/server/model" "github.com/pingcap/tiup/server/session" - "github.com/pingcap/tiup/server/store" ) // SignComponent handles requests to re-sign component manifest -func SignComponent(sm session.Manager, keys map[string]*v1manifest.KeyInfo) http.Handler { - return &componentSigner{sm, keys} +func SignComponent(sm session.Manager, mirror repository.Mirror) http.Handler { + return &componentSigner{sm, mirror} } type componentSigner struct { - sm session.Manager - keys map[string]*v1manifest.KeyInfo + sm session.Manager + mirror repository.Mirror } func (h *componentSigner) ServeHTTP(w http.ResponseWriter, r *http.Request) { fn.Wrap(h.sign).ServeHTTP(w, r) } -func (h *componentSigner) sign(r *http.Request, m *model.ComponentManifest) (sr *simpleResponse, err statusError) { - sid := mux.Vars(r)["sid"] - name := mux.Vars(r)["name"] - options := make(map[string]bool) +func buildInfo(r *http.Request, sid string) *model.PublishInfo { + info := &model.PublishInfo{} - for _, opt := range []string{"yanked", "standalone", "hidden"} { - if query(r, opt) == "true" { - options[opt] = true - } else if query(r, opt) == "false" { - options[opt] = false + m := map[string]**bool{ + repository.OptionYanked: &info.Yank, + repository.OptionStandalone: &info.Stand, + repository.OptionHidden: &info.Hide, + } + + for k, v := range m { + f := false + if query(r, k) == "true" { + f = true + *v = &f + } else if query(r, k) == "false" { + *v = &f } } + return info +} + +func (h *componentSigner) sign(r *http.Request, m *v1manifest.RawManifest) (sr *simpleResponse, err statusError) { + sid := mux.Vars(r)["sid"] + name := mux.Vars(r)["name"] + info := buildInfo(r, sid) + blackList := []string{"root", "index", "snapshot", "timestamp"} for _, b := range blackList { if name == b { @@ -64,159 +75,36 @@ func (h *componentSigner) sign(r *http.Request, m *model.ComponentManifest) (sr } log.Infof("Sign component manifest for %s, sid: %s", name, sid) - txn := h.sm.Load(sid) - if txn == nil { - if e := h.sm.Begin(sid); e != nil { - log.Errorf("Begin session %s", e.Error()) - return nil, ErrorInternalError - } - if txn = h.sm.Load(sid); txn == nil { - return nil, ErrorSessionMissing - } - } - - initTime := time.Now() - - md := model.New(txn, h.keys) - // retry util there is no conflict with other txns - if err := utils.RetryUntil(func() error { - // Write the component manifest (component.json) - if err := md.UpdateComponentManifest(name, m); err != nil { - if err == model.ErrorConflict { - return ErrorManifestConflict - } - return err - } - - // Update snapshot.json and signature - fi, err := txn.Stat(fmt.Sprintf("%d.%s.json", m.Signed.Version, name)) - if err != nil { - return err - } - - var indexFileVersion *v1manifest.FileVersion - var owner *v1manifest.Owner - if err := md.UpdateIndexManifest(initTime, func(om *model.IndexManifest) *model.IndexManifest { - // We only update index.json when it's a new component - // or the yanked, standalone, hidden fileds changed - var ( - compItem v1manifest.ComponentItem - compExist bool - ) - - if compItem, compExist = om.Signed.Components[name]; compExist { - // Find the owner of target component - o := om.Signed.Owners[compItem.Owner] - owner = &o - if len(options) == 0 { - // No changes on index.json - return nil - } - if opt, ok := options["yanked"]; ok { - compItem.Yanked = opt - } - if opt, ok := options["hidden"]; ok { - compItem.Hidden = opt - } - if opt, ok := options["standalone"]; ok { - compItem.Standalone = opt - } - } else { - var ownerID string - // The component is a new component, so the owner is whoever first create it. - for _, sk := range m.Signatures { - if ownerID, owner = om.KeyOwner(sk.KeyID); owner != nil { - break - } - } - compItem = v1manifest.ComponentItem{ - Owner: ownerID, - URL: fmt.Sprintf("/%s.json", name), - Yanked: options["yanked"], - Standalone: options["standalone"], - Hidden: options["hidden"], - } - } - - om.Signed.Components[name] = compItem - indexFileVersion = &v1manifest.FileVersion{Version: om.Signed.Version + 1} - return om - }); err != nil { - return err - } - - if err := validate(owner, m); err != nil { - return err - } - - if indexFileVersion != nil { - if indexFi, err := txn.Stat(fmt.Sprintf("%d.index.json", indexFileVersion.Version)); err == nil { - indexFileVersion.Length = uint(indexFi.Size()) - } else { - return err - } - } - - if err := md.UpdateSnapshotManifest(initTime, func(om *model.SnapshotManifest) *model.SnapshotManifest { - if indexFileVersion != nil { - om.Signed.Meta["/index.json"] = *indexFileVersion - } - om.Signed.Meta[fmt.Sprintf("/%s.json", name)] = v1manifest.FileVersion{ - Version: m.Signed.Version, - Length: uint(fi.Size()), - } - return om - }); err != nil { - return err - } - - // Update timestamp.json and signature - if err := md.UpdateTimestampManifest(initTime); err != nil { - return err - } - return txn.Commit() - }, func(err error) bool { - log.Infof("Sign error: %s", err.Error()) - return err == store.ErrorFsCommitConflict && txn.ResetManifest() == nil - }); err != nil { - log.Errorf("Sign component failed: %s", err.Error()) - if err, ok := err.(statusError); ok { - return nil, err + if fileName, readCloser, err := h.sm.Read(sid); err == nil { + info.ComponentData = &model.TarInfo{ + Reader: readCloser, + Name: fileName, } + } else { + log.Errorf("Read tar info for component %s, sid: %s", name, sid) return nil, ErrorInternalError } - h.sm.Delete(sid) - return nil, nil -} - -// ModifyComponent handles requests to modify index.json (yank or hide components) -func ModifyComponent(sm session.Manager, keys map[string]*v1manifest.KeyInfo) http.Handler { - return &componentSigner{sm, keys} -} - -func validate(owner *v1manifest.Owner, m *model.ComponentManifest) error { - if owner == nil { - return ErrorForbiden + comp := v1manifest.Component{} + if err := json.Unmarshal(m.Signed, &comp); err != nil { + log.Errorf("Unmarshal manifest %s", err.Error()) + return nil, ErrorInvalidManifest } - payload, err := cjson.Marshal(m.Signed) - if err != nil { - return err + manifest := &v1manifest.Manifest{ + Signatures: m.Signatures, + Signed: &comp, } - for _, s := range m.Signatures { - k := owner.Keys[s.KeyID] - if k == nil { - continue - } - - if err := k.Verify(payload, s.Sig); err == nil { - return nil - } + switch err := h.mirror.Publish(manifest, info); err { + case model.ErrorConflict: + return nil, ErrorManifestConflict + case nil: + return nil, nil + default: + h.sm.Delete(sid) + return nil, ErrorInternalError } - - return ErrorForbiden } func query(r *http.Request, q string) string { diff --git a/server/handler/error.go b/server/handler/error.go index b7d9a53170..404406043d 100644 --- a/server/handler/error.go +++ b/server/handler/error.go @@ -54,6 +54,8 @@ var ( ErrorManifestMissing = newHandlerError(http.StatusNotFound, "MANIFEST NOT FOUND", "that component doesn't have manifest yet") // ErrorInvalidTarball indicates that the tarball is not valid (eg. too large) ErrorInvalidTarball = newHandlerError(http.StatusBadRequest, "INVALID TARBALL", "the tarball content is not valid") + // ErrorInvalidManifest indicates that the manfiest is not valid + ErrorInvalidManifest = newHandlerError(http.StatusBadRequest, "INVALID MANIFEST", "the manifest content is not valid") // ErrorInternalError indicates that an internal error happened ErrorInternalError = newHandlerError(http.StatusInternalServerError, "INTERNAL ERROR", "an internal error happened") // ErrorManifestConflict indicates that the uploaded manifest is not new enough diff --git a/server/handler/tarball.go b/server/handler/tarball.go index 05e1d063d2..bc9fa20e82 100644 --- a/server/handler/tarball.go +++ b/server/handler/tarball.go @@ -42,23 +42,6 @@ func (h *tarballUploader) upload(r *http.Request) (*simpleResponse, statusError) sid := mux.Vars(r)["sid"] log.Infof("Uploading tarball, sid: %s", sid) - if err := h.sm.Begin(sid); err != nil { - if err == session.ErrorSessionConflict { - log.Warnf("Session already exists, this is a retransmission, try to restart session") - // Reset manifest to avoid conflict - if err := h.sm.Load(sid).ResetManifest(); err != nil { - log.Errorf("Failed to restart session: %s", err.Error()) - return nil, ErrorInternalError - } - log.Infof("Restart session success") - } else { - log.Errorf("Failed to start session: %s", err.Error()) - return nil, ErrorInternalError - } - } - - txn := h.sm.Load(sid) - if err := r.ParseMultipartForm(MaxMemory); err != nil { // TODO: log error here return nil, ErrorInvalidTarball @@ -71,7 +54,7 @@ func (h *tarballUploader) upload(r *http.Request) (*simpleResponse, statusError) } defer file.Close() - if err := txn.Write(handler.Filename, file); err != nil { + if err := h.sm.Write(sid, handler.Filename, file); err != nil { log.Errorf("Error to write tarball: %s", err.Error()) return nil, ErrorInternalError } diff --git a/server/main.go b/server/main.go index 97c0fdd744..6b5a8c7dc4 100644 --- a/server/main.go +++ b/server/main.go @@ -24,9 +24,6 @@ import ( func main() { addr := "0.0.0.0:8989" upstream := "https://tiup-mirrors.pingcap.com" - indexKey := "" - snapshotKey := "" - timestampKey := "" cmd := &cobra.Command{ Use: fmt.Sprintf("%s ", os.Args[0]), @@ -36,7 +33,7 @@ func main() { return cmd.Help() } - s, err := newServer(args[0], upstream, indexKey, snapshotKey, timestampKey) + s, err := newServer(args[0], upstream) if err != nil { return err } @@ -45,9 +42,6 @@ func main() { }, } cmd.Flags().StringVarP(&addr, "addr", "", addr, "addr to listen") - cmd.Flags().StringVarP(&indexKey, "index", "", "", "specific the private key for index") - cmd.Flags().StringVarP(&snapshotKey, "snapshot", "", "", "specific the private key for snapshot") - cmd.Flags().StringVarP(×tampKey, "timestamp", "", "", "specific the private key for timestamp") cmd.Flags().StringVarP(&upstream, "upstream", "", upstream, "specific the upstream mirror") if err := cmd.Execute(); err != nil { diff --git a/server/model/model.go b/server/model/model.go deleted file mode 100644 index 3dd5abaa3f..0000000000 --- a/server/model/model.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "fmt" - "time" - - cjson "github.com/gibson042/canonicaljson-go" - "github.com/juju/errors" - "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/repository/v1manifest" - "github.com/pingcap/tiup/pkg/utils" - "github.com/pingcap/tiup/server/store" -) - -// Model defines operations on the manifests -type Model interface { - UpdateComponentManifest(component string, manifest *ComponentManifest) error - UpdateRootManifest(manifest *RootManifest) error - UpdateIndexManifest(time.Time, func(*IndexManifest) *IndexManifest) error - UpdateSnapshotManifest(time.Time, func(*SnapshotManifest) *SnapshotManifest) error - UpdateTimestampManifest(time.Time) error -} - -type model struct { - txn store.FsTxn - keys map[string]*v1manifest.KeyInfo -} - -// New returns a object implemented Model -func New(txn store.FsTxn, keys map[string]*v1manifest.KeyInfo) Model { - return &model{txn, keys} -} - -func (m *model) UpdateComponentManifest(component string, manifest *ComponentManifest) error { - snap, err := m.ReadSnapshotManifest() - if err != nil { - return err - } - lastVersion := snap.Signed.Meta["/"+manifest.Signed.Filename()].Version - if manifest.Signed.Version != lastVersion+1 { - log.Debugf("Component version not expected, expect %d, got %d", lastVersion+1, manifest.Signed.Version) - return ErrorConflict - } - return m.txn.WriteManifest(fmt.Sprintf("%d.%s.json", manifest.Signed.Version, component), manifest) -} - -func (m *model) UpdateRootManifest(manifest *RootManifest) error { - var last RootManifest - if err := m.txn.ReadManifest(v1manifest.ManifestFilenameRoot, &last); err != nil { - return err - } - if manifest.Signed.Version != last.Signed.Version+1 { - return ErrorConflict - } - if err := m.txn.WriteManifest(v1manifest.ManifestFilenameRoot, manifest); err != nil { - return err - } - - return m.txn.WriteManifest(fmt.Sprintf("%d.root.json", manifest.Signed.Version), manifest) -} - -func (m *model) UpdateIndexManifest(initTime time.Time, f func(*IndexManifest) *IndexManifest) error { - snap, err := m.ReadSnapshotManifest() - if err != nil { - return err - } - lastVersion := snap.Signed.Meta[v1manifest.ManifestURLIndex].Version - - var last IndexManifest - if err := m.txn.ReadManifest(fmt.Sprintf("%d.index.json", lastVersion), &last); err != nil { - return err - } - manifest := f(&last) - if manifest == nil { - return nil - } - manifest.Signed.Version = last.Signed.Version + 1 - v1manifest.RenewManifest(&manifest.Signed, initTime) - manifest.Signatures, err = sign(manifest.Signed, m.keys[v1manifest.ManifestTypeIndex]) - if err != nil { - return err - } - - return m.txn.WriteManifest(fmt.Sprintf("%d.index.json", manifest.Signed.Version), manifest) -} - -func (m *model) UpdateSnapshotManifest(initTime time.Time, f func(*SnapshotManifest) *SnapshotManifest) error { - var last SnapshotManifest - err := m.txn.ReadManifest(v1manifest.ManifestFilenameSnapshot, &last) - if err != nil { - return err - } - manifest := f(&last) - if manifest == nil { - return nil - } - v1manifest.RenewManifest(&manifest.Signed, initTime) - manifest.Signatures, err = sign(manifest.Signed, m.keys[v1manifest.ManifestTypeSnapshot]) - if err != nil { - return err - } - - return m.txn.WriteManifest(v1manifest.ManifestFilenameSnapshot, manifest) -} - -// ReadSnapshotManifest returns snapshot.json -func (m *model) ReadSnapshotManifest() (*SnapshotManifest, error) { - var snap SnapshotManifest - if err := m.txn.ReadManifest(v1manifest.ManifestFilenameSnapshot, &snap); err != nil { - return nil, err - } - return &snap, nil -} - -// ReadRootManifest returns root.json -func (m *model) ReadRootManifest() (*RootManifest, error) { - var root RootManifest - if err := m.txn.ReadManifest(v1manifest.ManifestFilenameRoot, &root); err != nil { - return nil, err - } - return &root, nil -} - -func (m *model) UpdateTimestampManifest(initTime time.Time) error { - fi, err := m.txn.Stat(v1manifest.ManifestFilenameSnapshot) - if err != nil { - return err - } - reader, err := m.txn.Read(v1manifest.ManifestFilenameSnapshot) - if err != nil { - return err - } - sha256, err := utils.SHA256(reader) - if err != nil { - reader.Close() - return err - } - reader.Close() - - var manifest TimestampManifest - err = m.txn.ReadManifest(v1manifest.ManifestFilenameTimestamp, &manifest) - if err != nil { - return err - } - manifest.Signed.Version++ - manifest.Signed.Meta[v1manifest.ManifestURLSnapshot] = v1manifest.FileHash{ - Hashes: map[string]string{ - v1manifest.SHA256: sha256, - }, - Length: uint(fi.Size()), - } - v1manifest.RenewManifest(&manifest.Signed, initTime) - manifest.Signatures, err = sign(manifest.Signed, m.keys[v1manifest.ManifestTypeTimestamp]) - if err != nil { - return err - } - - return m.txn.WriteManifest(v1manifest.ManifestFilenameTimestamp, &manifest) -} - -func sign(signed interface{}, keys ...*v1manifest.KeyInfo) ([]v1manifest.Signature, error) { - payload, err := cjson.Marshal(signed) - if err != nil { - return nil, err - } - - signs := []v1manifest.Signature{} - for _, k := range keys { - if k == nil { - return nil, ErrorMissingKey - } - id, err := k.ID() - if err != nil { - return nil, errors.Trace(err) - } - sign, err := k.Signature(payload) - if err != nil { - return nil, errors.Trace(err) - } - signs = append(signs, v1manifest.Signature{ - KeyID: id, - Sig: sign, - }) - } - - return signs, nil -} diff --git a/server/model/types.go b/server/model/types.go deleted file mode 100644 index 7ae815cdb3..0000000000 --- a/server/model/types.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "github.com/pingcap/tiup/pkg/repository/v1manifest" -) - -// ComponentManifest represents xxx.json -type ComponentManifest struct { - // Signatures value - Signatures []v1manifest.Signature `json:"signatures"` - // Signed value; any value here must have the SignedBase base. - Signed v1manifest.Component `json:"signed"` -} - -// RootManifest represents root.json -type RootManifest struct { - // Signatures value - Signatures []v1manifest.Signature `json:"signatures"` - // Signed value; any value here must have the SignedBase base. - Signed v1manifest.Root `json:"signed"` -} - -// IndexManifest represents index.json -type IndexManifest struct { - // Signatures value - Signatures []v1manifest.Signature `json:"signatures"` - // Signed value; any value here must have the SignedBase base. - Signed v1manifest.Index `json:"signed"` -} - -// KeyOwner returns the owner or nil for a given keyid -func (m *IndexManifest) KeyOwner(keyid string) (string, *v1manifest.Owner) { - for on := range m.Signed.Owners { - for k := range m.Signed.Owners[on].Keys { - if k == keyid { - o := m.Signed.Owners[on] - return on, &o - } - } - } - return "", nil -} - -// SnapshotManifest represents snapshot.json -type SnapshotManifest struct { - // Signatures value - Signatures []v1manifest.Signature `json:"signatures"` - // Signed value; any value here must have the SignedBase base. - Signed v1manifest.Snapshot `json:"signed"` -} - -// TimestampManifest represents timestamp.json -type TimestampManifest struct { - // Signatures value - Signatures []v1manifest.Signature `json:"signatures"` - // Signed value; any value here must have the SignedBase base. - Signed v1manifest.Timestamp `json:"signed"` -} diff --git a/server/router.go b/server/router.go index 67b3025b6a..393b7b5af4 100644 --- a/server/router.go +++ b/server/router.go @@ -47,8 +47,8 @@ func (s *server) router() http.Handler { r := mux.NewRouter() r.Handle("/api/v1/tarball/{sid}", handler.UploadTarbal(s.sm)) - r.Handle("/api/v1/component/{sid}/{name}", handler.SignComponent(s.sm, s.keys)) - r.PathPrefix("/").Handler(s.static("/", s.root, s.upstream)) + r.Handle("/api/v1/component/{sid}/{name}", handler.SignComponent(s.sm, s.mirror)) + r.PathPrefix("/").Handler(s.static("/", s.mirror.Source(), s.upstream)) return httpRequestMiddleware(r) } diff --git a/server/server.go b/server/server.go index d47879aee0..8631a06dd1 100644 --- a/server/server.go +++ b/server/server.go @@ -14,48 +14,30 @@ package main import ( - "encoding/json" "fmt" "net/http" - "os" - "sync" - "github.com/pingcap/tiup/pkg/repository/v1manifest" + "github.com/pingcap/tiup/pkg/repository" "github.com/pingcap/tiup/server/session" - "github.com/pingcap/tiup/server/store" ) type server struct { - root string - upstream string - keys map[string]*v1manifest.KeyInfo + mirror repository.Mirror sm session.Manager + upstream string } // NewServer returns a pointer to server -func newServer(rootDir, upstream, indexKey, snapshotKey, timestampKey string) (*server, error) { - s := &server{ - root: rootDir, - upstream: upstream, - keys: make(map[string]*v1manifest.KeyInfo), - sm: session.New(store.NewStore(rootDir, upstream), new(sync.Map)), - } - - kmap := map[string]string{ - v1manifest.ManifestTypeIndex: indexKey, - v1manifest.ManifestTypeSnapshot: snapshotKey, - v1manifest.ManifestTypeTimestamp: timestampKey, +func newServer(rootDir, upstream string) (*server, error) { + mirror := repository.NewMirror(rootDir, repository.MirrorOptions{Upstream: upstream}) + if err := mirror.Open(); err != nil { + return nil, err } - for ty, kfile := range kmap { - if kfile == "" { - continue - } - k, err := loadPrivateKey(kfile) - if err != nil { - return nil, err - } - s.keys[ty] = k + s := &server{ + mirror: mirror, + sm: session.New(), + upstream: upstream, } return s, nil @@ -65,23 +47,3 @@ func (s *server) run(addr string) error { fmt.Println(addr) return http.ListenAndServe(addr, s.router()) } - -func loadPrivateKey(keyFile string) (*v1manifest.KeyInfo, error) { - var key v1manifest.KeyInfo - f, err := os.Open(keyFile) - if err != nil { - return nil, err - } - defer f.Close() - if err := json.NewDecoder(f).Decode(&key); err != nil { - return nil, err - } - - // Check if key is valid - _, err = key.ID() - if err != nil { - return nil, err - } - - return &key, nil -} diff --git a/server/session/session.go b/server/session/session.go index 5741edc071..41f184d7f1 100644 --- a/server/session/session.go +++ b/server/session/session.go @@ -14,12 +14,15 @@ package session import ( - "errors" + "io" + "os" + "path" "sync" "time" + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/server/store" ) // Max alive time of a session @@ -32,64 +35,100 @@ var ( // Manager provide methods to operates on upload sessions type Manager interface { - Begin(id string) error - Load(id string) store.FsTxn + Write(id string, name string, reader io.Reader) error + Read(id string) (string, io.ReadCloser, error) Delete(id string) } type sessionManager struct { - store store.Store - txns *sync.Map + m *sync.Map } // New returns a session manager -func New(store store.Store, txns *sync.Map) Manager { +func New() Manager { return &sessionManager{ - store: store, - txns: txns, + m: &sync.Map{}, } } -// Begin start a new session and returns the session id -func (s *sessionManager) Begin(id string) error { - if s.Load(id) != nil { +// Write start a new session +func (s *sessionManager) Write(id, name string, reader io.Reader) error { + if _, ok := s.m.Load(id); ok { return ErrorSessionConflict } log.Debugf("Begin new session: %s", id) - txn, err := s.store.Begin() + s.m.Store(id, name) + go s.gc(id) + + dataDir := os.Getenv(localdata.EnvNameComponentDataDir) + if dataDir == "" { + return errors.Errorf("cannot read environment variable %s", localdata.EnvNameComponentDataDir) + } + + pkgDir := path.Join(dataDir, "packages") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + return errors.Annotate(err, "create package dir") + } + + filePath := path.Join(pkgDir, id+"_"+name) + file, err := os.Create(filePath) if err != nil { - return err + return errors.Annotate(err, "create tar file") } - s.txns.Store(id, txn) - go s.gc(id) + defer file.Close() + + if _, err := io.Copy(file, reader); err != nil { + return errors.Annotate(err, "write tar file") + } + return nil } -func (s *sessionManager) gc(id string) { - time.Sleep(maxAliveTime) +// Read returns the tar file of given session +func (s *sessionManager) Read(id string) (string, io.ReadCloser, error) { + n, ok := s.m.Load(id) + if !ok { + return "", nil, nil + } + name := n.(string) - txn := s.Load(id) - if txn == nil { - return + dataDir := os.Getenv(localdata.EnvNameComponentDataDir) + if dataDir == "" { + return "", nil, errors.Errorf("cannot read environment variable %s", localdata.EnvNameComponentDataDir) } - s.Delete(id) - if err := txn.Rollback(); err != nil { - log.Errorf("Rollback: %s", err.Error()) + pkgDir := path.Join(dataDir, "packages") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + return "", nil, errors.Annotate(err, "create package dir") } -} -// Get returns the txn of given session -func (s *sessionManager) Load(id string) store.FsTxn { - txn, ok := s.txns.Load(id) - if ok { - return txn.(store.FsTxn) + filePath := path.Join(pkgDir, id+"_"+name) + + file, err := os.Open(filePath) + if err != nil { + return "", nil, errors.Annotate(err, "open tar file") } - return nil + return name, file, nil } // Delele delete a session func (s *sessionManager) Delete(id string) { log.Debugf("Delete session: %s", id) - s.txns.Delete(id) + n, ok := s.m.Load(id) + if !ok { + return + } + name := n.(string) + os.Remove(path.Join(os.Getenv(localdata.EnvNameComponentDataDir), "packages", id+"_"+name)) + s.m.Delete(id) +} + +func (s *sessionManager) gc(id string) { + time.Sleep(maxAliveTime) + + if _, ok := s.m.Load(id); !ok { + return + } + + s.Delete(id) } diff --git a/server/store/qcloud.go b/server/store/qcloud.go deleted file mode 100644 index c2e4910893..0000000000 --- a/server/store/qcloud.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path" - "sync" - "time" - - cjson "github.com/gibson042/canonicaljson-go" - "github.com/google/uuid" - "github.com/pingcap/tiup/pkg/logger/log" - "github.com/pingcap/tiup/pkg/utils" -) - -const ( - // EnvQCloudScriptPath enable the qcloud uplaod (will update CDN if specified) - EnvQCloudScriptPath = "QCLOUD_UPLOAD_SCRIPT" -) - -var ( - // ErrorFsCommitConflict indicates concurrent writing file - ErrorFsCommitConflict = errors.New("conflict on fs commit") -) - -type qcloudStore struct { - mux sync.Mutex - root string - upstream string - modified map[string]*time.Time -} - -func newQCloudStore(root, upstream string) *qcloudStore { - if err := os.MkdirAll(root, 0755); err != nil { - log.Errorf("Create store directory: %s", err.Error()) - } - return &qcloudStore{ - root: root, - upstream: upstream, - modified: make(map[string]*time.Time), - } -} - -func (s *qcloudStore) Begin() (FsTxn, error) { - return newQCloudTxn(s) -} - -func (s *qcloudStore) path(filename string) string { - return path.Join(s.root, filename) -} - -func (s *qcloudStore) modify(filename string, t *time.Time) { - s.modified[filename] = t -} - -// Returns the last modify time -func (s *qcloudStore) last(filename string) *time.Time { - return s.modified[filename] -} - -func (s *qcloudStore) lock() { - s.mux.Lock() -} - -func (s *qcloudStore) unlock() { - s.mux.Unlock() -} - -type qcloudTxn struct { - syncer Syncer - store *qcloudStore - root string - begin time.Time - accessed map[string]*time.Time -} - -func newQCloudTxn(store *qcloudStore) (*qcloudTxn, error) { - syncer := newFsSyncer(path.Join(store.root, "commits")) - if script := os.Getenv(EnvQCloudScriptPath); script != "" { - syncer = combine(syncer, newQcloudSyncer(script)) - } - txn := &qcloudTxn{ - syncer: syncer, - store: store, - root: path.Join("/tmp", uuid.New().String()), - begin: time.Now(), - accessed: make(map[string]*time.Time), - } - - if err := txn.require(); err != nil { - return nil, err - } - return txn, nil -} - -func (t *qcloudTxn) Write(filename string, reader io.Reader) error { - filepath := path.Join(t.root, filename) - file, err := os.Create(filepath) - if err != nil { - return err - } - defer file.Close() - - _, err = io.Copy(file, reader) - return err -} - -func (t *qcloudTxn) Read(filename string) (io.ReadCloser, error) { - filepath := t.store.path(filename) - if utils.IsExist(path.Join(t.root, filename)) { - filepath = path.Join(t.root, filename) - } - - return os.Open(filepath) -} - -func (t *qcloudTxn) WriteManifest(filename string, manifest interface{}) error { - t.access(filename) - filepath := path.Join(t.root, filename) - file, err := os.Create(filepath) - if err != nil { - return err - } - defer file.Close() - - bytes, err := cjson.Marshal(manifest) - if err != nil { - return err - } - - if _, err = file.Write(bytes); err != nil { - return err - } - - return nil -} - -func (t *qcloudTxn) ReadManifest(filename string, manifest interface{}) error { - t.access(filename) - filepath := t.store.path(filename) - if utils.IsExist(path.Join(t.root, filename)) { - filepath = path.Join(t.root, filename) - } - var wc io.ReadCloser - if file, err := os.Open(filepath); err == nil { - wc = file - } else if os.IsNotExist(err) && t.store.upstream != "" { - if resp, err := http.Get(fmt.Sprintf("%s/%s", t.store.upstream, filename)); err == nil { - wc = resp.Body - } else { - return err - } - } else { - log.Errorf("Error on read manifest: %s, upstream: %s", err.Error(), t.store.upstream) - return err - } - defer wc.Close() - - bytes, err := ioutil.ReadAll(wc) - if err != nil { - return err - } - - return cjson.Unmarshal(bytes, manifest) -} - -func (t *qcloudTxn) ResetManifest() error { - for file := range t.accessed { - fp := path.Join(t.root, file) - if utils.IsExist(fp) { - if err := os.Remove(fp); err != nil { - return err - } - } - } - t.begin = time.Now() - return nil -} - -func (t *qcloudTxn) Stat(filename string) (os.FileInfo, error) { - t.access(filename) - filepath := t.store.path(filename) - if utils.IsExist(path.Join(t.root, filename)) { - filepath = path.Join(t.root, filename) - } - return os.Stat(filepath) -} - -func (t *qcloudTxn) access(filename string) { - // Use the earliest time - if t.accessed[filename] != nil { - return - } - - at := time.Now() - t.accessed[filename] = &at -} - -// Returns the first access time -func (t *qcloudTxn) first(filename string) *time.Time { - return t.accessed[filename] -} - -func (t *qcloudTxn) Commit() error { - t.store.lock() - defer t.store.unlock() - - if err := t.checkConflict(); err != nil { - return err - } - - files, err := ioutil.ReadDir(t.root) - if err != nil { - return err - } - - for _, f := range files { - if err := utils.Copy(path.Join(t.root, f.Name()), t.store.path(f.Name())); err != nil { - return err - } - } - - at := time.Now() - for _, f := range files { - t.store.modify(f.Name(), &at) - } - - if err := t.syncer.Sync(t.root); err != nil { - return err - } - - return t.release() -} - -func (t *qcloudTxn) checkConflict() error { - for file := range t.accessed { - if t.store.last(file) != nil && t.store.last(file).After(*t.first(file)) { - return ErrorFsCommitConflict - } - } - return nil -} - -func (t *qcloudTxn) Rollback() error { - return t.release() -} - -func (t *qcloudTxn) require() (err error) { - return os.MkdirAll(t.root, 0755) -} - -func (t *qcloudTxn) release() error { - return os.RemoveAll(t.root) -} diff --git a/server/store/qcloud_test.go b/server/store/qcloud_test.go deleted file mode 100644 index 7ab8c91854..0000000000 --- a/server/store/qcloud_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - . "github.com/pingcap/check" - "github.com/pingcap/tiup/pkg/repository/v1manifest" -) - -var _ = Suite(&TestQCloudStoreSuite{}) - -type TestQCloudStoreSuite struct{} - -func (s *TestQCloudStoreSuite) TestEmptyCommit(c *C) { - store := NewStore("/tmp/store", "") - txn, err := store.Begin() - c.Assert(err, IsNil) - c.Assert(txn.Commit(), IsNil) -} - -func (s *TestQCloudStoreSuite) TestSingleWrite(c *C) { - store := NewStore("/tmp/store", "") - txn, err := store.Begin() - c.Assert(err, IsNil) - c.Assert(txn.WriteManifest("test.json", &v1manifest.Manifest{}), IsNil) - c.Assert(txn.ReadManifest("test.json", &v1manifest.Manifest{}), IsNil) - c.Assert(txn.Commit(), IsNil) -} - -func (s *TestQCloudStoreSuite) TestWrite(c *C) { - store := NewStore("/tmp/store", "") - txn1, err := store.Begin() - c.Assert(err, IsNil) - txn2, err := store.Begin() - c.Assert(err, IsNil) - c.Assert(txn1.WriteManifest("test.json", &v1manifest.Manifest{}), IsNil) - c.Assert(txn2.WriteManifest("test.json", &v1manifest.Manifest{}), IsNil) - c.Assert(txn1.Commit(), IsNil) - c.Assert(txn2.Commit(), NotNil) -} diff --git a/server/store/store_test.go b/server/store/store_test.go deleted file mode 100644 index ce9776014a..0000000000 --- a/server/store/store_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - "testing" - - . "github.com/pingcap/check" -) - -func TestUtils(t *testing.T) { - TestingT(t) -} diff --git a/tests/tiup/test_tiup.sh b/tests/tiup/test_tiup.sh index 2a21df9e68..4b958f3a1d 100755 --- a/tests/tiup/test_tiup.sh +++ b/tests/tiup/test_tiup.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -eu + TEST_DIR=$(cd "$(dirname "$0")"; pwd) TMP_DIR=$TEST_DIR/_tmp @@ -30,12 +32,39 @@ tiup tiup help tiup install tidb:v3.0.13 tiup update tidb -tiup update --self tiup status tiup clean --all tiup help tidb tiup env TIUP_SSHPASS_PROMPT="password" tiup env TIUP_SSHPASS_PROMPT | grep password + +# test mirror +cat > /tmp/hello.sh << EOF +#! /bin/sh + +echo "hello, TiDB" +EOF +chmod 755 /tmp/hello.sh +tar -C /tmp -czf /tmp/hello.tar.gz hello.sh + +tiup mirror genkey + +tiup mirror init /tmp/test-mirror-a +tiup mirror set /tmp/test-mirror-a +tiup mirror grant pingcap +echo "should fail" +! tiup mirror grant pingcap # this should failed +tiup mirror publish hello v0.0.1 /tmp/hello.tar.gz hello.sh +tiup hello:v0.0.1 | grep TiDB + +tiup mirror init /tmp/test-mirror-b +tiup mirror set /tmp/test-mirror-b +tiup mirror grant pingcap +tiup mirror publish hello v0.0.2 /tmp/hello.tar.gz hello.sh +tiup mirror set /tmp/test-mirror-a +tiup mirror merge /tmp/test-mirror-b +tiup hello:v0.0.2 | grep TiDB + tiup uninstall tiup uninstall tidb:v3.0.13 tiup uninstall tidb --all diff --git a/tools/migrate/main.go b/tools/migrate/main.go index 64df61ae3c..a28431a852 100644 --- a/tools/migrate/main.go +++ b/tools/migrate/main.go @@ -206,7 +206,6 @@ func migrate(srcDir, dstDir string, rehash bool) error { return errors.Trace(err) } index = i.Signed.(*v1manifest.Index) - index.Base().Version++ v1manifest.RenewManifest(index, initTime) } else { root = v1manifest.NewRoot(initTime)